@pictor-sdk/video-android 0.1.0

package/android/src/main/java/dev/pictor/video/FrameClock.kt

@@ -0,0 +1,591 @@
+package dev.pictor.video
+
+/**
+ * One canonical sample per displayed frame — the only source of timing truth.
+ *
+ * Everything reported about synchronisation comes from here: the percentiles
+ * and the on-screen readout read the *same* sample, matched by [Sample.ID]. An
+ * independent readout calculation was how a 22 ms disagreement went unnoticed
+ * between two views of one quantity.
+ *
+ * ## What is paired with what
+ *
+ * `VideoFrameMetadataListener` fires just before a frame is released to the
+ * surface; `onFrameAvailable` follows once it is queued, and **always after**
+ * the metadata for that frame. `updateTexImage()` then latches the *newest*
+ * buffer, discarding anything behind it.
+ *
+ * **The pairing happens when availability fires, not when the renderer ticks.**
+ * `onFrameAvailable` coalesces: several queued frames can raise one callback.
+ * Draining the two queues independently — one metadata per availability —
+ * therefore leaves metadata behind whenever that happens, and the surplus never
+ * clears. The reported PTS falls further behind with every coalesced callback,
+ * which measured as a *growing* offset: 21 ms after twenty-five seconds and
+ * 616 ms after five minutes, while the picture stayed correct.
+ *
+ * So availability claims **all** metadata queued at that moment and keeps the
+ * newest as the pending pair. That is sound precisely because metadata always
+ * precedes its own availability: everything queued when the callback fires
+ * belongs to a frame already handed to the surface. The renderer then consumes
+ * one complete pair, and metadata cannot accumulate.
+ *
+ * ## Time bases
+ *
+ * Every nanosecond value here is `System.nanoTime`. `releaseTimeNs` from Media3
+ * is documented on that clock, `onFrameAvailable` is stamped with it, and the
+ * anchor is published with it, so differences between them are meaningful. The
+ * only microsecond values are media times. `TIME_UNSET` never reaches this
+ * class; the player maps it to zero before publishing.
+ *
+ * **Every method is `@JvmStatic`.** A Kotlin `object`'s members are instance
+ * methods on `INSTANCE`, so `GetStaticMethodID` returns null without it and JNI
+ * aborts the process with `JNI DETECTED ERROR IN APPLICATION: mid == null` — no
+ * Java stack, and a message naming the JNI call rather than the annotation.
+ */
+internal object FrameClock {
+
+    /** A few frames of pipeline depth, and no more. */
+    const val CAPACITY = 32
+
+    /** Fields of one canonical sample, in the order [take] returns them. */
+    const val FIELDS = 18
+
+    /** Indices into the returned array. Mirrored in `pictor_videosurface.cpp`. */
+    object Field {
+        const val ID = 0
+        const val GENERATION = 1
+        const val PTS_US = 2
+        const val METADATA_NS = 3
+        const val RELEASE_NS = 4
+        const val AVAILABLE_NS = 5
+        const val CONSUMED_NS = 6
+        const val ANCHOR_POSITION_US = 7
+        const val ANCHOR_NS = 8
+        const val RATE_MILLI = 9
+        const val POSITION_AT_RELEASE_US = 10
+        const val POSITION_AT_CONSUMPTION_US = 11
+        const val FLAGS = 12
+        const val ANCHOR_AGE_NS = 13
+        const val COALESCED = 14
+        /**
+         * Which playlist window the frame came from.
+         *
+         * Carried per frame rather than read from player state, because state
+         * is published on a timer and a seam is exactly where a frame and a
+         * timer-sampled window index disagree. A consumer uses this to refuse a
+         * frame belonging to a clip the playhead has already left.
+         */
+        const val WINDOW_INDEX = 15
+
+        /**
+         * The acquired buffer's own timestamp, after `updateTexImage`.
+         *
+         * Not media time. Measured, this is a `System.nanoTime`-based value in
+         * the same domain as `RELEASE_NS`, so it identifies *which buffer* was
+         * acquired rather than which frame it depicts.
+         *
+         * The metadata deque is coalesced to its newest entry on every
+         * availability callback, so the pairing between "the sample we kept"
+         * and "the buffer `updateTexImage` took" is an assumption. This is the
+         * one fact that can check it: `SurfaceTexture.getTimestamp()` is the
+         * acquired buffer's own presentation time. A disagreement means the
+         * frame on screen is not the frame the record describes, which is the
+         * difference between a late frame and a mislabelled one.
+         */
+        const val TEXTURE_PTS_US = 16
+
+        /**
+         * Whether this sample's pixels and its timestamps are the same frame.
+         *
+         * 1 when the latched buffer's own timestamp matched a metadata entry
+         * inside the pairing window, and matched exactly one. 0 otherwise: no
+         * timestamp from the producer, no candidate close enough, or two
+         * candidates equally close.
+         *
+         * A 0 is not a small error. It means the pair is a guess — the claimed
+         * metadata with whatever buffer happened to be latched — which is the
+         * shape that put frame 7's pixels under frame 2's presentation time.
+         * Nothing may gate readiness or judge frame identity on one.
+         */
+        const val ASSOCIATION = 17
+    }
+
+    private val lock = Any()
+    private val metadata = ArrayDeque<LongArray>()
+
+    /**
+     * The newest complete pair, waiting for the renderer.
+     *
+     * One slot, not a queue: `updateTexImage` shows the newest buffer, so an
+     * older pending pair describes a frame nobody will see. A newer pair
+     * replaces it and the replaced one counts as [coalesced].
+     */
+    private var pending: LongArray? = null
+
+    @Volatile
+    var generation: Int = 0
+        private set
+
+    @Volatile
+    private var nextId: Long = 1
+
+    /** The sample the renderer most recently consumed. Readout reads this. */
+    @Volatile
+    private var latest: LongArray? = null
+
+    @Volatile var overflowed: Long = 0; private set
+    @Volatile var unpaired: Long = 0; private set
+    @Volatile var stale: Long = 0; private set
+    @Volatile var coalesced: Long = 0; private set
+    @Volatile var highWater: Int = 0; private set
+
+    /** Everything queued belongs to the clip that has just been replaced. */
+    /**
+     * Whether the transport is in continuous playback.
+     *
+     * Set by whoever owns the transport, not inferred. A gap between frame
+     * callbacks is not a transition: a clip seam produces one and playback is
+     * continuous across it, so pacing must stay engaged. What ends a stream is
+     * a pause, a seek or a start — and only the transport knows which of those
+     * just happened.
+     */
+    @Volatile private var streaming = false
+
+    /**
+     * Whether the latch enforces the intended presentation time.
+     *
+     * Off, and measured rather than assumed: enforcing it at the latch costs a
+     * frame per deferral. `markAvailable` coalesces the metadata deque whenever
+     * something is already pending, so a frame held for its due time has its
+     * record discarded by the next arrival and `updateTexImage` takes the newer
+     * buffer regardless. The displayed cadence in a 1x clip went from 33.3 ms
+     * to 66.7 ms — half the frame rate — to remove a 34 ms systematic lead.
+     *
+     * That is the wrong trade, so the enforcement stays available and stays
+     * off. Honouring the schedule without dropping frames needs the previous
+     * picture retained, and the external texture cannot do that: one
+     * `updateTexImage` overwrites it, and a recorded `SkPicture` samples the
+     * texture at draw time rather than holding its pixels. Retaining it means
+     * an owned copy — one 1080x1920 RGBA texture, about 8.3 MB, plus a
+     * full-frame blit per held frame.
+     */
+    @Volatile var enforcePresentationSchedule = false
+
+    @JvmStatic
+    fun setStreaming(on: Boolean) {
+        streaming = on
+    }
+
+    @JvmStatic
+    fun isStreaming(): Boolean = streaming
+
+    @JvmStatic
+    fun bump() {
+        // A discontinuity ends the stream: the frames in flight belong to the
+        // position being left, and their release times describe it.
+        streaming = false
+        synchronized(lock) {
+            generation += 1
+            stale += metadata.size + (if (pending == null) 0 else 1)
+            metadata.clear()
+            pending = null
+            latest = null
+        }
+    }
+
+    /**
+     * Which playlist window is playing, as the player last reported it.
+     *
+     * Lives here rather than being passed in, because the only place frames are
+     * offered is a factory that builds the player before the instance holding
+     * that number exists. Set on the player's own thread at a media-item
+     * transition — which is the instant it changes — and read on the playback
+     * thread when a frame is offered.
+     */
+    @Volatile
+    @JvmStatic
+    var windowIndex: Int = 0
+
+    /** Metadata callbacks so far. With [available], separates the two halves. */
+    @Volatile var offered: Long = 0; private set
+
+    /** From `VideoFrameMetadataListener`, before the frame is released. */
+    @JvmStatic
+    fun offer(ptsUs: Long, scheduledReleaseNs: Long) {
+        val now = PlaybackAnchor.nowNs()
+        /*
+          How far ahead the renderer scheduled this frame.
+
+          `releaseOutputBuffer(index, releaseTimeNs)` hands the buffer to the
+          surface's frame scheduler with a desired present time. A lead of a
+          frame or two is normal; a lead measured in seconds means the frame was
+          handed over but is not going to be presented for a very long time, and
+          from outside that is indistinguishable from a producer that stopped.
+        */
+        lastReleaseLeadNs = scheduledReleaseNs - now
+        lastOfferedPtsUs = ptsUs
+        if (lastReleaseLeadNs > 1_000_000_000L) farFutureOffers += 1
+        if (lastReleaseLeadNs < -1_000_000_000L) farPastOffers += 1
+        val anchor = PlaybackAnchor.latest()
+        offered += 1
+        val sample = LongArray(FIELDS)
+        sample[Field.GENERATION] = generation.toLong()
+        sample[Field.PTS_US] = ptsUs
+        sample[Field.METADATA_NS] = now
+        sample[Field.RELEASE_NS] = scheduledReleaseNs
+        sample[Field.ANCHOR_POSITION_US] = anchor.positionUs
+        sample[Field.ANCHOR_NS] = anchor.monotonicNs
+        sample[Field.RATE_MILLI] = (anchor.rate * 1000).toLong()
+        sample[Field.POSITION_AT_RELEASE_US] = PlaybackAnchor.positionAt(scheduledReleaseNs)
+        sample[Field.FLAGS] = stateBits(anchor)
+        sample[Field.ANCHOR_AGE_NS] = PlaybackAnchor.ageNs(scheduledReleaseNs)
+        sample[Field.WINDOW_INDEX] = windowIndex.toLong()
+        synchronized(lock) {
+            if (metadata.size >= CAPACITY) {
+                metadata.removeFirst()
+                overflowed += 1
+            }
+            metadata.addLast(sample)
+            if (metadata.size > highWater) highWater = metadata.size
+        }
+    }
+
+    /**
+     * From `SurfaceTexture.OnFrameAvailableListener`, once queued.
+     *
+     * Claims every metadata event queued at this instant and keeps the newest.
+     * Anything older describes a buffer the next `updateTexImage` will skip.
+     */
+    /** Every availability callback, paired or not. A liveness counter. */
+    @Volatile var available: Long = 0; private set
+
+    @JvmStatic
+    fun markAvailable() {
+        val now = PlaybackAnchor.nowNs()
+        available += 1
+        synchronized(lock) {
+            if (metadata.isEmpty()) {
+                // Availability with nothing to claim. Not fatal — a repeat
+                // callback, or a frame whose metadata was discarded by a
+                // generation change — but worth counting.
+                orphaned += 1
+                return
+            }
+            var dropped = 0
+            while (metadata.size > 1) {
+                metadata.removeFirst()
+                dropped += 1
+            }
+            if (pending != null) dropped += 1
+            coalesced += dropped.toLong()
+            val sample = metadata.removeFirst()
+            sample[Field.AVAILABLE_NS] = now
+            sample[Field.COALESCED] = dropped.toLong()
+            pending = sample
+        }
+    }
+
+    /** Availability callbacks that found no metadata to claim. */
+    @Volatile var orphaned: Long = 0; private set
+
+    @JvmStatic
+    fun pendingFrames(): Int = synchronized(lock) { if (pending == null) 0 else 1 }
+
+    private fun stateBits(anchor: PlaybackAnchor.Sample): Long {
+        var bits = 0L
+        if (anchor.playing) bits = bits or 1L
+        if (anchor.buffering) bits = bits or 2L
+        if (anchor.discontinuity) bits = bits or 4L
+        return bits
+    }
+
+    /**
+     * Complete the sample for the frame the renderer is about to show.
+     *
+     * Drains to the newest pending frame, because that is the one
+     * `updateTexImage` will produce. Returns null when nothing of the current
+     * generation is pending — the honest answer after a seek, and better than
+     * a stale pairing.
+     */
+    /**
+     * Every entry to [take], counted before the lock is attempted.
+     *
+     * The one measurement that separates a consumer which has stopped ticking
+     * from one blocked inside the latch. `taken` and `unpaired` are both
+     * recorded *after* the lock is held, so a consumer waiting on it looks
+     * exactly like a consumer that never arrived — and a frozen picture with
+     * `unpaired` at zero could mean either.
+     */
+    @Volatile var takeCalls: Long = 0; private set
+
+    @JvmStatic
+    /** Acquisitions whose texture timestamp did not match the paired metadata. */
+    @Volatile var mismatched: Long = 0; private set
+
+    /** Acquisitions where the texture reported no usable timestamp. */
+    @Volatile var unverified: Long = 0; private set
+
+    /**
+     * How far the texture's timestamp may sit from the paired metadata.
+     *
+     * Half a frame at 30 fps. The two describe the same buffer, so they should
+     * agree exactly; the tolerance is for the rounding between microseconds and
+     * nanoseconds and for producers that offset their timestamps.
+     */
+    private const val IDENTITY_TOLERANCE_US = 16_000L
+
+    @JvmStatic
+    fun take(consumedNs: Long): LongArray? = take(consumedNs, 0L)
+
+    /**
+     * How far a buffer's timestamp may sit from a metadata entry's release time
+     * and still be the same buffer.
+     *
+     * Half a frame at 30 fps. Both are `System.nanoTime` values describing the
+     * same handover, so they agree closely; the tolerance covers the scheduler's
+     * own rounding.
+     */
+    private const val PAIRING_TOLERANCE_NS = 16_000_000L
+
+    /**
+     * How much closer the best candidate must be than the next.
+     *
+     * Below this the two are indistinguishable and neither can be claimed. A
+     * quarter of the pairing window: wide enough that ordinary jitter still
+     * resolves, narrow enough that two frames a whole interval apart do.
+     */
+    private const val AMBIGUITY_MARGIN_NS = 4_000_000L
+
+    /** Latches whose pixels could not be tied to any metadata entry. */
+    @Volatile var unassociated: Long = 0; private set
+
+    /** Latches where the buffer was not the one the claimed metadata described. */
+    @Volatile var repaired: Long = 0; private set
+
+    @JvmStatic
+    fun take(consumedNs: Long, texturePtsNs: Long): LongArray? {
+        takeCalls += 1
+        synchronized(lock) {
+            /*
+              Pair with the buffer that was actually latched, not the one that
+              was claimed when availability first fired.
+
+              `markAvailable` claims the newest metadata *at that instant* and
+              `updateTexImage` takes the newest buffer *at its instant*, and
+              between the two more frames can arrive. Measured at startup: the
+              renderer released frames 2 to 7 in a burst, availability claimed
+              frame 2, the latch took frame 7's buffer, and the pair went out
+              as frame 7's pixels labelled 33.3 ms — a wrong picture that every
+              timestamp in the system agreed was right.
+
+              The texture's own timestamp settles it. It is a `System.nanoTime`
+              value on this device, the same domain as `RELEASE_NS`, so the
+              entry describing the latched buffer is the one whose release time
+              it matches. Without a usable timestamp this falls back to the
+              claimed pair, which is the old behaviour.
+            */
+            var sample = pending
+            var associated = false
+            if (texturePtsNs > 0L) {
+                var best: LongArray? = null
+                var bestSkew = Long.MAX_VALUE
+                var runnerUp = Long.MAX_VALUE
+                val claimed = pending
+                if (claimed != null) {
+                    bestSkew = Math.abs(claimed[Field.RELEASE_NS] - texturePtsNs)
+                    best = claimed
+                }
+                for (entry in metadata) {
+                    val skew = Math.abs(entry[Field.RELEASE_NS] - texturePtsNs)
+                    if (skew < bestSkew) {
+                        runnerUp = bestSkew
+                        bestSkew = skew
+                        best = entry
+                    } else if (skew < runnerUp) {
+                        runnerUp = skew
+                    }
+                }
+                /*
+                  Unique, or it is not an association.
+
+                  Two entries equally close to the latched timestamp cannot both
+                  describe it, and picking the nearer by a nanosecond is a coin
+                  toss dressed as a measurement. Ambiguity is reported as
+                  unverified, like absence.
+                */
+                if (best != null && bestSkew <= PAIRING_TOLERANCE_NS && runnerUp - bestSkew >= AMBIGUITY_MARGIN_NS) {
+                    associated = true
+                }
+                if (associated && best != null && best !== pending) {
+                    /*
+                      The match is ahead of the claimed pair. Everything between
+                      describes a buffer this latch skipped, so it is dropped and
+                      counted rather than shown later out of order.
+                    */
+                    var dropped = if (pending != null) 1 else 0
+                    while (metadata.isNotEmpty() && metadata.first() !== best) {
+                        metadata.removeFirst()
+                        dropped += 1
+                    }
+                    if (metadata.isNotEmpty() && metadata.first() === best) metadata.removeFirst()
+                    coalesced += dropped.toLong()
+                    best!![Field.AVAILABLE_NS] = consumedNs
+                    best[Field.COALESCED] = dropped.toLong()
+                    pending = best
+                    sample = best
+                    repaired += 1
+                }
+            }
+            if (sample == null) {
+                unpaired += 1
+                return null
+            }
+            if (sample[Field.GENERATION] != generation.toLong()) {
+                // The clip moved on between pairing and consumption.
+                pending = null
+                stale += 1
+                unpaired += 1
+                return null
+            }
+            pending = null
+            taken += 1
+            /*
+              Stamped on every sample, including the ones that fell back.
+
+              The fallback keeps the claimed pair so a producer without
+              timestamps still gets a picture, but it is a guess and has to
+              travel as one: readiness will not release on it and the identity
+              gates will not count it.
+            */
+            sample[Field.ASSOCIATION] = if (associated) 1L else 0L
+            if (!associated) unassociated += 1
+            sample[Field.ID] = nextId++
+            sample[Field.CONSUMED_NS] = consumedNs
+            sample[Field.POSITION_AT_CONSUMPTION_US] = PlaybackAnchor.positionAt(consumedNs)
+            /*
+              The acquired texture, labelled with what it actually holds.
+
+              Recorded rather than assumed, and the disagreement is counted
+              rather than smoothed: a sample carrying another frame's timestamp
+              would make a correctly-delivered frame look late, and there is no
+              way to tell those apart afterwards from the numbers alone.
+            */
+            if (texturePtsNs > 0L) {
+                sample[Field.TEXTURE_PTS_US] = texturePtsNs / 1000
+                /*
+                  Compared against the release time, not the media timestamp.
+
+                  Measured on a Pixel 7, `SurfaceTexture.getTimestamp` returns a
+                  `System.nanoTime`-based value here — 175,110,822,173 us, about
+                  forty-nine hours of uptime — and not the buffer's presentation
+                  time. Comparing it to `PTS_US` counted every single frame as
+                  mismatched, which is a broken check reporting a broken
+                  pipeline. Against `RELEASE_NS` the two share a domain, and the
+                  question it answers is the useful one: is the buffer that was
+                  acquired the one whose release we paired?
+                */
+                val skewUs = Math.abs(texturePtsNs - sample[Field.RELEASE_NS]) / 1000
+                if (skewUs > IDENTITY_TOLERANCE_US) mismatched += 1
+            } else {
+                sample[Field.TEXTURE_PTS_US] = -1
+                unverified += 1
+            }
+            latest = sample
+            return sample
+        }
+    }
+
+    /** Samples actually consumed by the renderer. A liveness counter. */
+    @Volatile var lastReleaseLeadNs: Long = 0; private set
+    @Volatile var lastOfferedPtsUs: Long = 0; private set
+    @Volatile var farFutureOffers: Long = 0; private set
+    @Volatile var farPastOffers: Long = 0; private set
+    @Volatile var taken: Long = 0; private set
+
+    /** The most recent canonical sample, for a readout. Never recomputed. */
+    @JvmStatic
+    fun latestSample(): LongArray? = latest
+
+    /**
+     * Everything needed to say *which stage* stopped, in one read.
+     *
+     * `latest` freezing can mean the picture froze, the renderer stopped
+     * consuming, or metadata stopped arriving — and an offset summary cannot
+     * tell those apart. These are the counters that can.
+     *
+     * **Volatiles only, and deliberately no `depth` or `pendingFrames`.** Those
+     * take [lock], which `offer`, `markAvailable` and `take` all hold on the
+     * paths that decode and draw. A sampler reading them at 40 Hz contended
+     * with the renderer badly enough to stop it consuming frames entirely —
+     * the instrument froze the picture it was measuring. Anything here must be
+     * readable without waiting for a frame.
+     */
+    /**
+     * When the pending frame was asked to be presented, or 0 if none.
+     *
+     * Exposed so the consumer can honour that time instead of latching the
+     * instant a buffer is queued. Measured, it latched about 34 ms early:
+     * `releaseOutputBuffer(index, releaseTimeNs)` queues immediately with a
+     * desired present time in the future, and nothing downstream was enforcing
+     * it — so the picture ran ahead of the sound by that much and the
+     * comparison against the intended time described an intention nobody kept.
+     */
+    /**
+     * When the pending frame was asked to be presented, or 0 if none.
+     *
+     * Zero while the transport is not streaming, which is the whole of the
+     * first-frame readiness path: a start or a seek needs its first frame
+     * latched as soon as it exists, and there is nothing yet for it to be early
+     * relative to. Pacing resumes when playback does.
+     */
+    @JvmStatic
+    fun pendingDueNs(): Long =
+        if (!streaming || !enforcePresentationSchedule) 0L
+        else synchronized(lock) { pending?.get(Field.RELEASE_NS) ?: 0L }
+
+    @JvmStatic
+    fun pipeline(): LongArray = longArrayOf(
+        offered, available, taken, orphaned, unpaired, stale, coalesced,
+        generation.toLong(), takeCalls,
+    )
+
+    @JvmStatic
+    fun depth(): Int = synchronized(lock) { metadata.size }
+
+    /** overflowed, unpaired, stale, highWater, generation, coalesced, orphaned. */
+    @JvmStatic
+    fun counters(): LongArray = longArrayOf(
+        overflowed, unpaired, stale, highWater.toLong(), generation.toLong(), coalesced,
+        orphaned,
+    )
+
+    @JvmStatic
+    fun reset() {
+        synchronized(lock) {
+            generation = 0
+            nextId = 1
+            overflowed = 0
+            available = 0
+            offered = 0
+            unpaired = 0
+            stale = 0
+            coalesced = 0
+            orphaned = 0
+            // Identity counters too: a static tally that survives a reset makes
+            // every test after the first read another test's evidence.
+            mismatched = 0
+            repaired = 0
+            unassociated = 0
+            unverified = 0
+            taken = 0
+            takeCalls = 0
+            streaming = false
+            enforcePresentationSchedule = false
+            highWater = 0
+            metadata.clear()
+            pending = null
+            latest = null
+        }
+    }
+
+}