@pictor-sdk/video-android 0.1.0

package/android/src/main/java/dev/pictor/video/PreviewTrace.kt

@@ -0,0 +1,478 @@
+package dev.pictor.video
+
+import android.os.Handler
+import android.os.HandlerThread
+import java.util.concurrent.atomic.AtomicLong
+import org.json.JSONArray
+import org.json.JSONObject
+
+/**
+ * One monotonic record of what both players were doing.
+ *
+ * ## Why a buffer and not a log
+ *
+ * The question is whether the picture and the sound agree, and answering it
+ * needs samples close enough together to see a 40 ms disagreement. Logging that
+ * from JavaScript would put a bridge hop and a string format inside the
+ * measurement, and `uiautomator` timing — already ruled out — is coarser still
+ * than either. So events are written into a fixed ring on the thread that
+ * observes them, and drained afterwards.
+ *
+ * Everything is stamped with `System.nanoTime()`, which is the clock
+ * `PlaybackAnchor` publishes on and the clock `VideoFrameMetadataListener`
+ * reports frame release times on. One timebase, so a comparison between two
+ * players is a subtraction rather than an argument.
+ *
+ * ## Bounded, and it says when it overflowed
+ *
+ * A ring drops the oldest events when it wraps, and a trace that silently lost
+ * its middle would be worse than no trace: the gap is exactly where a
+ * discontinuity would have been. The sequence number is monotonic and the
+ * drained result reports how many were dropped, so a harness can tell a quiet
+ * period from a lost one.
+ *
+ * ## Off unless armed
+ *
+ * Compile-time gated by [Diagnostics] and then armed explicitly, because
+ * sampling two players at 50 ms is work that a preview should not be doing for
+ * anybody who is not measuring it.
+ */
+internal object PreviewTrace {
+
+    /** Enough for roughly two minutes at the sampling interval, plus events. */
+    private const val CAPACITY = 8192
+
+    private val ring = arrayOfNulls<Event>(CAPACITY)
+
+    /** Total events ever written. The index into the ring is this modulo it. */
+    private val written = AtomicLong(0)
+
+    /** Armed by the harness; never on in a build without diagnostics. */
+    @Volatile var armed = false
+        internal set
+
+    /**
+     * Why the run stopped being trustworthy, or null.
+     *
+     * A sampler that threw and quietly disarmed left a trace that looked
+     * complete: the events before the fault were real, the summary computed
+     * happily, and nothing said the recording had stopped early. That is how a
+     * partial run becomes successful evidence. Set before disarming, cleared
+     * only by a fresh arm, and reported by every drain — a caller that ignores
+     * it is reading a truncated trace as a whole one.
+     */
+    @Volatile var faulted: String? = null
+        private set
+
+    class Event(val seq: Long, val atNs: Long, val type: String, val payload: JSONObject)
+
+    fun arm(on: Boolean) {
+        if (!Diagnostics.ENABLED) return
+        armed = on
+        if (on) {
+            faulted = null
+            reset()
+        }
+    }
+
+    fun reset() {
+        written.set(0)
+        for (i in ring.indices) ring[i] = null
+    }
+
+    /**
+     * Record one event.
+     *
+     * The payload is built by the caller only when armed, so an unarmed build
+     * pays for a volatile read and nothing else.
+     */
+    inline fun record(type: String, build: () -> JSONObject) {
+        if (!armed) return
+        put(type, build())
+    }
+
+    /** Record with no payload. */
+    fun mark(type: String) {
+        if (!armed) return
+        put(type, JSONObject())
+    }
+
+    fun put(type: String, payload: JSONObject) {
+        if (!armed) return
+        val seq = written.getAndIncrement()
+        ring[(seq % CAPACITY).toInt()] = Event(seq, System.nanoTime(), type, payload)
+    }
+
+    /**
+     * Everything recorded since `fromSeq`, oldest first.
+     *
+     * `dropped` counts events that were overwritten before this call could see
+     * them — a harness that ignores it is reading a trace with a hole in it.
+     */
+    fun drain(fromSeq: Long, limit: Int): JSONObject {
+        val total = written.get()
+        val oldestHeld = maxOf(0L, total - CAPACITY)
+        val start = maxOf(fromSeq, oldestHeld)
+        val dropped = maxOf(0L, oldestHeld - fromSeq)
+
+        val out = JSONArray()
+        var seq = start
+        while (seq < total && out.length() < limit) {
+            val event = ring[(seq % CAPACITY).toInt()]
+            // A slot can be null only before the ring has filled that far.
+            if (event != null && event.seq == seq) {
+                out.put(
+                    JSONObject()
+                        .put("seq", event.seq)
+                        .put("atNs", event.atNs)
+                        .put("type", event.type)
+                        .put("v", event.payload),
+                )
+            }
+            seq += 1
+        }
+        return JSONObject()
+            // `ok` is about the drain call. `valid` is about the run: a trace
+            // whose sampler threw is complete up to the fault and silent after
+            // it, and the difference is invisible in the events themselves.
+            .put("ok", true)
+            .put("valid", faulted == null)
+            .put("fault", faulted ?: JSONObject.NULL)
+            .put("events", out)
+            .put("nextSeq", seq)
+            .put("total", total)
+            .put("dropped", dropped)
+    }
+
+    // ── the paired sampler ────────────────────────────────────────────────
+
+    private var samplerThread: HandlerThread? = null
+    private var samplerHandler: Handler? = null
+
+    /**
+     * Both players, read at the same instant, on the same clock.
+     *
+     * Neither read blocks: the video player publishes [PlaybackAnchor] and the
+     * mix publishes its own, both from their own threads. So a sample is two
+     * volatile reads and a subtraction, and the interval between them is
+     * nanoseconds rather than a thread hop.
+     *
+     * The video position is *extrapolated* to the sample instant through
+     * `positionAt`, because `currentPosition` does not advance continuously and
+     * comparing a stale value against a fresh one reports the staleness as
+     * drift. The anchor's own age is recorded alongside, so a reader can see
+     * how much of any offset is extrapolation.
+     */
+    private val sampler = object : Runnable {
+        override fun run() {
+            if (!armed) return
+            /*
+              Nothing here may kill the process.
+
+              It did: `FrameClock.pipeline()` was shortened from ten fields to
+              eight and a reader was left at index eight, so the sampler threw
+              `ArrayIndexOutOfBoundsException` on its first tick and Android took
+              the app down. From the outside that is indistinguishable from
+              another application taking the foreground — the previous one
+              surfaces — and it was twice reported as exactly that.
+
+              An instrument that can crash what it measures is worse than no
+              instrument, so the whole tick is guarded and a failure disarms
+              rather than propagates.
+            */
+            try {
+                sample()
+            } catch (e: Throwable) {
+                // Marked *before* disarming. A run that stopped recording must
+                // not be readable as one that simply had nothing more to say.
+                faulted = e.javaClass.simpleName + (e.message?.let { ": $it" } ?: "")
+                armed = false
+                android.util.Log.e("PictorTrace", "sampler failed; run invalid", e)
+                return
+            }
+            samplerHandler?.postDelayed(this, intervalMs)
+        }
+
+        private fun sample() {
+            val atNs = System.nanoTime()
+            val video = PlaybackAnchor.latest()
+            val audio = PreviewAudioPlayer.current()?.anchor
+
+            /*
+              Converted to project time, not read as-is.
+
+              `player.currentPosition` is position within the current window, so
+              comparing it against a mix that covers the whole timeline made
+              every seam look like a nine-second desynchronisation. The local
+              value is extrapolated first and converted second, because speed is
+              a property of the window.
+            */
+            val localUs = PlaybackAnchor.positionAt(atNs)
+            val videoUs = Media3Player.globalPositionUs(video.windowIndex, localUs)
+
+            val payload = JSONObject()
+                .put("videoUs", videoUs)
+                // The frame on screen, which is not the decoder's position: a
+                // stall shows as the two separating, and only one of them is
+                // what a person sees.
+                .put("presentedUs", PreviewSync.presentedUs() ?: -1)
+                // The pipeline behind that number, so a frozen reading can be
+                // attributed rather than guessed at.
+                .also { p ->
+                    val c = FrameClock.pipeline()
+                    p.put("fcOffered", c[0]).put("fcAvailable", c[1]).put("fcTaken", c[2])
+                        .put("fcOrphaned", c[3]).put("fcUnpaired", c[4]).put("fcStale", c[5])
+                        .put("fcCoalesced", c[6]).put("fcGeneration", c[7])
+                    /*
+                      The presented frame's identity, not just its time.
+
+                      `id` advancing with an unchanged `pts` is a repeated
+                      picture; `id` unchanged is a stale record; both advancing
+                      with a wrong global value is a mapping fault. The position
+                      alone cannot distinguish them, and a frozen reading was
+                      previously reported without saying which it was.
+                    */
+                    PreviewSync.presentedRecord()?.let { r ->
+                        p.put("pfId", r[0]).put("pfGen", r[1]).put("pfWindow", r[2])
+                            .put("pfPtsUs", r[3]).put("pfGlobalUs", r[4])
+                            .put("pfConsumedNs", r[5])
+                            .put("pfAgeNs", atNs - r[5])
+                            .put("pfFresh", r[0] != lastPresentedId)
+                        lastPresentedId = r[0]
+                    }
+                }
+                .put("videoLocalUs", localUs)
+                .put("videoWindow", video.windowIndex)
+                .put("videoRawUs", video.positionUs)
+                .put("videoAnchorAgeNs", atNs - video.monotonicNs)
+                .put("videoPlaying", video.playing)
+                .put("videoBuffering", video.buffering)
+                .put("videoGeneration", video.generation)
+                .put("videoRate", video.rate.toDouble())
+                // The other factor in the conversion. A project clock running
+                // at the wrong rate is either the player's speed or this one,
+                // and only both together say which.
+                .put("windowSpeed", Media3Player.windowSpeedFor(video.windowIndex).toDouble())
+                .put("sourceVolume", Media3Player.currentVolumeOrNull() ?: -1.0)
+
+            if (audio != null) {
+                // Extrapolated the same way, and only while it is actually
+                // playing: a paused player's position does not advance, so
+                // projecting it forward would invent drift.
+                val projected =
+                    if (audio.playing) {
+                        // At the player's own rate. Extrapolating at 1.0 while
+                        // it runs at anything else puts the error into the
+                        // measurement instead of the thing measured.
+                        audio.positionUs +
+                            ((atNs - audio.monotonicNs) / 1000.0 * audio.rate).toLong()
+                    } else {
+                        audio.positionUs
+                    }
+                payload
+                    .put("audioUs", projected)
+                    .put("audioRawUs", audio.positionUs)
+                    .put("audioAnchorAgeNs", atNs - audio.monotonicNs)
+                    .put("audioPlaying", audio.playing)
+                    .put("audioReady", audio.ready)
+                    .put("audioEpoch", audio.epoch)
+                    .put("audioRate", audio.rate.toDouble())
+                    .put("offsetUs", projected - videoUs)
+            } else {
+                payload.put("audioUs", JSONObject.NULL)
+            }
+
+            /*
+              Transitions, recorded with positions.
+
+              A summary of offsets cannot say whether an excursion came from a
+              late pause or an early resume. These are the two instants that
+              distinguish them.
+            */
+            val playingNow = audio?.playing ?: false
+            if (playingNow != lastAudioPlaying) {
+                lastAudioPlaying = playingNow
+                put(
+                    if (playingNow) "audio.advancing" else "audio.stopped",
+                    JSONObject()
+                        .put("audioUs", audio?.positionUs ?: -1)
+                        .put("videoUs", videoUs)
+                        .put("presentedUs", PreviewSync.presentedUs() ?: -1),
+                )
+            }
+
+            put("sample", payload)
+        }
+    }
+
+    @Volatile private var intervalMs: Long = 25
+
+    /** For detecting the instant the mix actually stops or starts advancing. */
+    @Volatile private var lastAudioPlaying = false
+
+    /** The last presented frame's sequence, so freshness is a comparison. */
+    @Volatile private var lastPresentedId = Long.MIN_VALUE
+
+    fun startSampling(everyMs: Long) {
+        if (!Diagnostics.ENABLED) return
+        stopSampling()
+        intervalMs = everyMs.coerceIn(10, 1000)
+        val thread = HandlerThread("pictor-preview-trace").also { it.start() }
+        samplerThread = thread
+        samplerHandler = Handler(thread.looper).also { it.post(sampler) }
+    }
+
+    fun stopSampling() {
+        samplerHandler?.removeCallbacksAndMessages(null)
+        samplerHandler = null
+        samplerThread?.quitSafely()
+        samplerThread = null
+    }
+
+    /**
+     * Compare the two players and correct the mix if it has drifted too far.
+     *
+     * Done here, natively, because both anchors are already published on one
+     * clock and neither read blocks. The alternative — asking from JavaScript —
+     * puts a bridge hop between the two positions being compared and gives the
+     * caller no nanosecond clock to stamp them with, so the measurement would
+     * carry its own scheduling jitter into the answer.
+     *
+     * The picture is authoritative: the composed-frame clock is what the rest
+     * of the editor follows, so the mix moves to meet it.
+     *
+     * Correction only past the tolerance. Every seek is a discontinuity in the
+     * sound, so a monitor that nudges on every tick produces exactly the
+     * artefact it exists to prevent.
+     */
+    /**
+     * How long a correcting seek takes to take effect, learned.
+     *
+     * A seek is not instantaneous: the mix re-buffers, and the picture keeps
+     * moving while it does. Measured on a Pixel 7, seeking to the picture's
+     * current position landed the mix about 195 ms behind it — and the monitor
+     * then chased that back the other way, oscillating +264, -194, -207, -178
+     * for as long as playback continued. Correcting to where the picture *will
+     * be* is what makes it converge.
+     *
+     * Learned rather than fixed: it is a property of the device, the file and
+     * how much of it is buffered, none of which this can know in advance.
+     * Clamped, because a wild estimate would be worse than none.
+     */
+    @Volatile private var seekLatencyUs: Long = 0
+    private const val MAX_LATENCY_US = 400_000L
+
+    /** When the last correction was issued, and what it aimed at. */
+    @Volatile private var lastCorrectionNs: Long = 0
+    @Volatile private var lastCorrectionTargetUs: Long = 0
+
+    /**
+     * How long to leave a correction alone before judging it.
+     *
+     * A measurement taken mid-seek reads the seek, not the drift. Without this
+     * the monitor corrected every tick and never let anything settle.
+     */
+    private const val SETTLE_NS = 900_000_000L
+
+    fun driftCheck(toleranceUs: Long): JSONObject {
+        val out = JSONObject()
+        val player = PreviewAudioPlayer.current()
+            ?: return out.put("ok", false).put("reason", "no mixed player")
+        val audio = player.anchor
+        if (!audio.playing) return out.put("ok", true).put("acted", false).put("reason", "not playing")
+
+        val video = PlaybackAnchor.latest()
+        if (!video.playing) return out.put("ok", true).put("acted", false).put("reason", "video paused")
+        if (PreviewSync.stalled) {
+            // The coordinator owns this. A correction now would aim at a
+            // position the picture is about to leave, which is how the drift
+            // monitor previously turned a stall into an oscillation.
+            return out.put("ok", true).put("acted", false).put("reason", "stall held")
+        }
+        if (video.buffering) {
+            // The picture is stalled and the sound is not. Correcting now would
+            // aim at a position the video is about to leave; this is the
+            // condition that creates the drift, not a moment to measure it.
+            return out.put("ok", true).put("acted", false).put("reason", "video buffering")
+        }
+
+        val atNs = System.nanoTime()
+        val videoUs = Media3Player.globalPositionUs(
+            video.windowIndex,
+            PlaybackAnchor.positionAt(atNs),
+        )
+        val audioUs = audio.positionUs + (atNs - audio.monotonicNs) / 1000
+        val driftUs = audioUs - videoUs
+
+        out.put("ok", true)
+            .put("driftUs", driftUs)
+            .put("videoUs", videoUs)
+            .put("audioUs", audioUs)
+            .put("toleranceUs", toleranceUs)
+            .put("latencyUs", seekLatencyUs)
+
+        val sinceCorrection = atNs - lastCorrectionNs
+        if (lastCorrectionNs != 0L && sinceCorrection < SETTLE_NS) {
+            return out.put("acted", false).put("reason", "settling")
+        }
+
+        /*
+          Learn from the last correction before deciding on this one.
+
+          If the previous seek left the mix behind, it undershot by exactly the
+          time the seek took. That residual is the latency, smoothed so a single
+          noisy sample cannot swing the next target.
+        */
+        if (lastCorrectionNs != 0L) {
+            val residual = -driftUs
+            run {
+                /*
+                  Most of the residual, immediately, and in both directions.
+
+                  The residual *is* the error the last correction left, so the
+                  estimate that would have been right is `latency + residual`.
+                  Learning only when the mix landed *behind* made the estimate
+                  ratchet upward — measured, it ran to 344 ms and then overshot
+                  the other way by 140 ms with no path back down.
+                */
+                seekLatencyUs = (seekLatencyUs + residual * 4 / 5)
+                    .coerceIn(0L, MAX_LATENCY_US)
+            }
+        }
+
+        if (Math.abs(driftUs) <= toleranceUs) return out.put("acted", false)
+        /*
+          Measured either way, acted on only when enabled.
+
+          The comparison needs the excursion recorded identically in both arms;
+          returning before the drift is computed would make the disabled arm
+          look like a run with no drift rather than a run with no correction.
+        */
+        if (!PreviewConfig.driftCorrection) {
+            record("drift.suppressed") {
+                JSONObject().put("driftUs", driftUs).put("toleranceUs", toleranceUs)
+            }
+            return out.put("acted", false).put("reason", "correction disabled")
+        }
+
+        // Where the picture will be once the seek lands, not where it is now.
+        val target = videoUs + seekLatencyUs
+        lastCorrectionNs = atNs
+        lastCorrectionTargetUs = target
+        // Stamped with the epoch the mix is on, so a correction computed for a
+        // mix that has since been replaced is dropped by the player.
+        player.seekTo(target, audio.epoch)
+        record("drift.correct") {
+            JSONObject().put("driftUs", driftUs).put("toUs", target)
+                .put("latencyUs", seekLatencyUs).put("epoch", audio.epoch)
+        }
+        return out.put("acted", true).put("targetUs", target)
+    }
+
+    /** Forget what was learned; a new mix is a new file with its own latency. */
+    fun resetDrift() {
+        seekLatencyUs = 0
+        lastCorrectionNs = 0
+        lastCorrectionTargetUs = 0
+    }
+}