@pictor-sdk/video-android 0.1.0

package/android/src/main/java/dev/pictor/video/SurfaceAttachment.kt

@@ -0,0 +1,110 @@
+package dev.pictor.video
+
+/**
+ * Which presentation surface is live, and which one the player is attached to.
+ *
+ * A player attaches to a `Surface` once and keeps it for life. When the surface
+ * is replaced — a remounted editor, a second `create` — the player goes on
+ * rendering into the one it was born with: it decodes, it releases frames, it
+ * reports no error, and the live texture receives nothing. On a Pixel 7 that
+ * read as metadata offered climbing 2018 → 2164 over five seconds while
+ * availability sat at 180 and the renderer pulled zero frames, with a black
+ * preview and playback state READY.
+ *
+ * Nothing in that is an error condition, so it needs a comparison rather than a
+ * try/catch. Two facts are enough: which surface is live, and which one an
+ * attachment was made against.
+ *
+ * **Attachment is the adapter's job, not the host's.** A caller that has to
+ * remember to re-attach will forget, and the symptom is a black rectangle with
+ * healthy counters. So creating a surface attaches it, and this type only
+ * decides *whether* an attachment is needed.
+ *
+ * Deliberately free of Android types and of any player, so the whole decision is
+ * testable on the JVM without a device, a mock, or a codec.
+ */
+internal class SurfaceAttachment {
+
+    /** No surface, and no attachment. Distinct from any real generation. */
+    companion object {
+        const val NONE = 0
+    }
+
+    /** The generation of the surface that is currently live. */
+    @Volatile
+    var live: Int = NONE
+        private set
+
+    /** The generation the player is attached to, or [NONE]. */
+    @Volatile
+    var attached: Int = NONE
+        private set
+
+    /** Attachments actually performed. A gate on needless output rebuilds. */
+    @Volatile
+    var attachments: Int = 0
+        private set
+
+    /** Surfaces that have ever been live. Only ever increases. */
+    @Volatile
+    var created: Int = 0
+        private set
+
+    /**
+     * A new surface is live. Returns its generation.
+     *
+     * Never reuses a number, so a teardown arriving late can always be told
+     * apart from the surface it is trying to retire.
+     */
+    fun surfaceLive(): Int {
+        created += 1
+        live = created
+        return live
+    }
+
+    /** Whether an attachment made at [generation] is still the live surface. */
+    fun isStale(generation: Int): Boolean = generation == NONE || generation != live
+
+    /** Whether the player needs attaching to the live surface right now. */
+    fun needsAttach(): Boolean = live != NONE && isStale(attached)
+
+    /**
+     * Record an attachment to whatever is live. Returns whether it acted.
+     *
+     * Idempotent: re-attaching an unchanged surface makes Media3 drop and
+     * rebuild its output, which costs frames at exactly the wrong moment.
+     */
+    fun attach(): Boolean {
+        if (!needsAttach()) return false
+        attached = live
+        attachments += 1
+        return true
+    }
+
+    /**
+     * Retire the surface of [generation]. Returns whether it was the live one.
+     *
+     * **Generation-checked, and this is the point.** Teardown of an old surface
+     * can arrive after a newer one is already live and attached — the old
+     * screen's cleanup running after the new screen's setup, which is the
+     * ordinary case rather than a race. Clearing unconditionally there would
+     * detach the surface that is actually on screen and reproduce the very
+     * defect this exists to prevent.
+     *
+     * Idempotent: retiring something already retired changes nothing.
+     */
+    fun retire(generation: Int): Boolean {
+        if (generation == NONE || generation != live) return false
+        live = NONE
+        attached = NONE
+        return true
+    }
+
+    /** Test seam. Production never rewinds. */
+    fun resetForTest() {
+        live = NONE
+        attached = NONE
+        attachments = 0
+        created = 0
+    }
+}

package/android/src/main/java/dev/pictor/video/ThumbnailDecoder.kt

@@ -0,0 +1,171 @@
+package dev.pictor.video
+
+import android.graphics.Bitmap
+import android.media.MediaMetadataRetriever
+import android.net.Uri
+import java.io.ByteArrayOutputStream
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.Executors
+import java.util.concurrent.Future
+import java.util.concurrent.atomic.AtomicLong
+
+/**
+ * Still frames for the timeline strip.
+ *
+ * `MediaMetadataRetriever`, not a second `ExoPlayer`: a retriever opens a source
+ * once and seeks around it cheaply, while a player instance per strip would
+ * hold a codec, a surface and a buffer pool for something that draws a 64-pixel
+ * square. The playback path is untouched by any of this — one retriever per
+ * source, on a background pool, and the presentation surface never hears about
+ * it.
+ *
+ * **Cancellation is real, not advisory.** A strip that scrolls fast abandons
+ * far more requests than it uses. Each request is a `Future` that the JS side
+ * can cancel by token, and a cancelled one stops before it decodes rather than
+ * decoding and discarding.
+ *
+ * Retrievers are pooled per source and closed explicitly. Nothing here is
+ * reachable from a finalizer.
+ */
+internal object ThumbnailDecoder {
+
+    /**
+     * Two threads.
+     *
+     * Enough to keep decoding while one request waits on I/O, few enough that a
+     * scroll cannot start twenty concurrent seeks and starve the player's own
+     * threads — which share the same cores as the thing the user is watching.
+     */
+    private val pool = Executors.newFixedThreadPool(2) { runnable ->
+        Thread(runnable, "pictor-thumbs").apply { isDaemon = true }
+    }
+
+    private val retrievers = ConcurrentHashMap<String, MediaMetadataRetriever>()
+    private val running = ConcurrentHashMap<Long, Future<*>>()
+    private val results = ConcurrentHashMap<Long, String>()
+    private val failures = ConcurrentHashMap<Long, String>()
+    private val nextToken = AtomicLong(1)
+
+    @Volatile var decoded: Long = 0; private set
+    @Volatile var cancelled: Long = 0; private set
+    @Volatile var failed: Long = 0; private set
+
+    /**
+     * Start a decode. Returns a token to poll or cancel with.
+     *
+     * Deliberately not a Promise across the bridge. A strip issues and abandons
+     * requests in bursts, and a rejected Promise per abandoned request is both
+     * noisy and hard to cancel; a token the caller owns makes "stop this one"
+     * expressible.
+     */
+    @JvmStatic
+    fun request(uri: String, sourceTimeUs: Long, widthPx: Int, context: android.content.Context): Long {
+        val token = nextToken.getAndIncrement()
+        val future = pool.submit {
+            if (Thread.currentThread().isInterrupted) return@submit
+            val bitmap = runCatching { decode(uri, sourceTimeUs, widthPx, context) }
+                .getOrElse { error ->
+                    failures[token] = error.message ?: error.toString()
+                    failed += 1
+                    null
+                }
+            if (bitmap == null) {
+                running.remove(token)
+                return@submit
+            }
+            if (Thread.currentThread().isInterrupted) {
+                bitmap.recycle()
+                running.remove(token)
+                return@submit
+            }
+            // A data URI, because the renderer builds its own image from bytes
+            // and this way the adapter never holds a decoded frame the JS side
+            // might also be holding. The strip's images are small by
+            // construction — a 64px thumbnail is a few kilobytes.
+            results[token] = encode(bitmap)
+            bitmap.recycle()
+            decoded += 1
+            running.remove(token)
+        }
+        running[token] = future
+        return token
+    }
+
+    /** Stop a request. Interrupts it if it has started. */
+    @JvmStatic
+    fun cancel(token: Long) {
+        val future = running.remove(token) ?: return
+        if (future.cancel(true)) cancelled += 1
+        results.remove(token)
+        failures.remove(token)
+    }
+
+    /** The result, or null while it is still running. Consumes it. */
+    @JvmStatic
+    fun take(token: Long): String? = results.remove(token)
+
+    /** Why a request failed, or null. Consumes it. */
+    @JvmStatic
+    fun takeError(token: Long): String? = failures.remove(token)
+
+    /** Whether a token is still being worked on. */
+    @JvmStatic
+    fun isRunning(token: Long): Boolean = running.containsKey(token)
+
+    private fun decode(
+        uri: String,
+        sourceTimeUs: Long,
+        widthPx: Int,
+        context: android.content.Context,
+    ): Bitmap? {
+        val retriever = retrievers.getOrPut(uri) {
+            MediaMetadataRetriever().apply {
+                val parsed = Uri.parse(uri)
+                if (parsed.scheme == "http" || parsed.scheme == "https") {
+                    setDataSource(uri, HashMap())
+                } else {
+                    setDataSource(context, parsed)
+                }
+            }
+        }
+        // OPTION_CLOSEST_SYNC, not OPTION_CLOSEST: a strip wants speed and
+        // approximate times, and exact-frame extraction decodes every frame
+        // from the previous keyframe forward — seconds of work for one 64px
+        // square.
+        val full = retriever.getFrameAtTime(sourceTimeUs, MediaMetadataRetriever.OPTION_CLOSEST_SYNC)
+            ?: return null
+        if (full.width <= widthPx) return full
+        val height = (full.height.toLong() * widthPx / full.width).toInt().coerceAtLeast(1)
+        val scaled = Bitmap.createScaledBitmap(full, widthPx, height, true)
+        if (scaled !== full) full.recycle()
+        return scaled
+    }
+
+    private fun encode(bitmap: Bitmap): String {
+        val bytes = ByteArrayOutputStream()
+        bitmap.compress(Bitmap.CompressFormat.JPEG, 70, bytes)
+        return android.util.Base64.encodeToString(bytes.toByteArray(), android.util.Base64.NO_WRAP)
+    }
+
+    /** Close the retriever for one source. Explicit; never from a finalizer. */
+    @JvmStatic
+    fun releaseSource(uri: String) {
+        retrievers.remove(uri)?.let { runCatching { it.release() } }
+    }
+
+    /** Close everything. Called when the editor goes away. */
+    @JvmStatic
+    fun releaseAll() {
+        for (future in running.values) future.cancel(true)
+        running.clear()
+        results.clear()
+        failures.clear()
+        for (retriever in retrievers.values) runCatching { retriever.release() }
+        retrievers.clear()
+    }
+
+    /** decoded, cancelled, failed, running, open retrievers. */
+    @JvmStatic
+    fun counters(): LongArray =
+        longArrayOf(decoded, cancelled, failed, running.size.toLong(), retrievers.size.toLong())
+}

package/android/src/main/java/dev/pictor/video/VideoSurface.kt

@@ -0,0 +1,394 @@
+package dev.pictor.video
+
+import android.graphics.SurfaceTexture
+import android.os.Handler
+import android.os.HandlerThread
+import android.view.Surface
+
+/**
+ * The Java half of the persistent presentation surface.
+ *
+ * A `SurfaceTexture` binds to the *calling thread's* current GL context, so
+ * every method here is called from native code running on the UI runtime with
+ * Skia's context current — the arrangement the context probe verified. Nothing
+ * here may be called from anywhere else.
+ *
+ * The `Surface` is registered so a player can render into it without any of
+ * this crossing Pictor's public API: the adapter hands Media3 an Android
+ * `Surface`, and Pictor sees a texture id and a transform.
+ */
+internal object VideoSurface {
+
+    /**
+     * The surface the current session renders into.
+     *
+     * Set from native once the texture exists, read by the player when it
+     * attaches. Volatile because those are different threads.
+     */
+    @Volatile
+    private var current: Surface? = null
+
+    /** Which surface is live and which one the player holds. */
+    @JvmStatic
+    val attachment = SurfaceAttachment()
+
+    /**
+     * Whether a frame is waiting, as a level rather than an edge.
+     *
+     * `onFrameAvailable` fires once and is gone; a consumer that was disarmed at
+     * that instant — backgrounded for the system picker — never learns it
+     * happened, and a paused player produces no second edge. See [FramePending].
+     */
+    @JvmStatic
+    val framePending = FramePending()
+
+    /**
+     * The texture the live generation belongs to.
+     *
+     * The listener resolves its generation *through* this, at callback time. A
+     * generation captured when the listener was registered is a snapshot of a
+     * value that moves: if `watch` were ever called before `register`, or the
+     * surface replaced between them, every subsequent frame would be tagged
+     * with a number that was already wrong — and `mark` would reject them all,
+     * silently, as belonging to a retired generation.
+     *
+     * Resolving from the texture also gets the late case right for free. A
+     * listener detached a moment too late fires on a texture that is no longer
+     * the live one, so it is rejected — which is what should happen, and what
+     * reading a bare "current generation" at callback time would get backwards
+     * by crediting the frame to whatever replaced it.
+     */
+    @Volatile
+    private var liveTexture: SurfaceTexture? = null
+
+    /**
+     * Notified the instant a surface becomes live.
+     *
+     * This is what makes attachment the adapter's responsibility rather than a
+     * step a host has to remember. A caller that must call `attachSurface()`
+     * after every `create` will eventually not, and the symptom is a black
+     * rectangle with entirely healthy counters — no exception, no error state,
+     * nothing to catch. Set once, when the player is built.
+     */
+    @Volatile
+    @JvmStatic
+    var onSurfaceLive: (() -> Unit)? = null
+
+    /** The live surface's generation, or [SurfaceAttachment.NONE]. */
+    @JvmStatic
+    val generation: Int
+        get() = attachment.live
+
+    /**
+     * A new surface is live; attach it.
+     *
+     * Returns the generation, which the caller keeps so a later teardown can
+     * prove it is retiring its own surface rather than someone else's.
+     */
+    @JvmStatic
+    fun register(surface: Surface): Int {
+        current = surface
+        val generation = attachment.surfaceLive()
+        framePending.adopt(generation)
+        // Attach here, not at the next playlist change: between the two the
+        // player would be rendering into the previous texture.
+        onSurfaceLive?.invoke()
+        return generation
+    }
+
+    @JvmStatic
+    fun currentSurface(): Surface? = current
+
+    /** Whether an attachment made at [attachedGeneration] is still live. */
+    @JvmStatic
+    fun isStale(attachedGeneration: Int): Boolean = attachment.isStale(attachedGeneration)
+
+    @JvmStatic
+    fun clear() {
+        current = null
+    }
+
+    /** Attach a `SurfaceTexture` to an existing external texture. */
+    @JvmStatic
+    fun create(textureId: Int, width: Int, height: Int): SurfaceTexture {
+        val texture = SurfaceTexture(textureId)
+        texture.setDefaultBufferSize(width, height)
+        return texture
+    }
+
+    @JvmStatic
+    fun surfaceOf(texture: SurfaceTexture): Surface = Surface(texture)
+
+    /**
+     * Pull the newest frame into the texture. Returns whether it actually did.
+     *
+     * `updateTexImage` does **not** always throw when it fails. If Skia's GL
+     * context is not current on this thread the platform logs
+     * `LegacySurfaceTexture is not attached` and returns quietly, and a caller
+     * that had already consumed the availability signal has then lost the frame
+     * entirely — the picture holds until the *next* frame arrives, which at 30
+     * fps is a 33 ms stutter with no error to explain it.
+     *
+     * So success is measured rather than assumed: the texture's timestamp
+     * advances only when a buffer was really latched. The caller puts the
+     * availability signal back when this returns false, turning a dropped frame
+     * into a one-tick retry.
+     */
+    @JvmStatic
+    fun update(texture: SurfaceTexture, generation: Int): Boolean {
+        // One owner, and it stops owning the moment its surface is retired or
+        // replaced. A worklet ticking an old host object would otherwise latch
+        // into a texture nobody is drawing — the invalid call, removed rather
+        // than caught.
+        if (attachment.isStale(generation)) {
+            updatesSkipped += 1
+            return false
+        }
+        updatesAttempted += 1
+        // Read before the acquisition, not after: `updateTexImage` can run for
+        // long enough that a frame is announced during it, and that frame is
+        // not one this latch took.
+        val mark = framePending.observe(generation)
+        val before = texture.timestamp
+        return try {
+            texture.updateTexImage()
+            // Success is measured, not assumed: updateTexImage returns quietly
+            // when the context is not current, and the timestamp only advances
+            // when a buffer was really latched.
+            val after = texture.timestamp
+            val latched = after != before
+            if (latched) {
+                updatesSucceeded += 1
+                // Coalesced to the newest this call could have taken: one
+                // buffer was taken and everything it skipped is gone, but a
+                // frame announced *during* the call is still waiting.
+                framePending.consumeUpTo(generation, mark)
+            } else {
+                // Nothing was taken, so nothing is accounted for. Leaving the
+                // counter alone is the restore.
+                updatesFailed += 1
+            }
+            latched
+        } catch (error: RuntimeException) {
+            updatesFailed += 1
+            false
+        }
+    }
+
+    /**
+     * What the one owner did, so a failure stays visible.
+     *
+     * `failed` must remain reportable rather than swallowed: a latch that
+     * silently does nothing looks exactly like a paused video.
+     */
+    @Volatile @JvmStatic var updatesAttempted: Long = 0; private set
+    @Volatile @JvmStatic var updatesSucceeded: Long = 0; private set
+    @Volatile @JvmStatic var updatesSkipped: Long = 0; private set
+    @Volatile @JvmStatic var updatesFailed: Long = 0; private set
+
+    /** Retained name for the failure count, which is what a missed latch is. */
+    @JvmStatic
+    val missedLatches: Long
+        get() = updatesFailed
+
+    /** The 4×4 texture transform: crop, flip and rotation, in that encoding. */
+    @JvmStatic
+    fun transform(texture: SurfaceTexture): FloatArray {
+        val matrix = FloatArray(16)
+        texture.getTransformMatrix(matrix)
+        return matrix
+    }
+
+    @JvmStatic
+    fun timestamp(texture: SurfaceTexture): Long = texture.timestamp
+
+    /** Explicit, in order, on the thread that owns them. Never from a finalizer. */
+    @JvmStatic
+    fun release(texture: SurfaceTexture, surface: Surface, generation: Int) {
+        releases += 1
+        // Generation-checked. A screen's teardown routinely runs *after* the
+        // next screen's setup, so an old surface can be released while a newer
+        // one is already live and attached. Clearing unconditionally there
+        // would detach what is actually on screen — the very defect this
+        // machinery exists to prevent. Idempotent for the same reason.
+        if (attachment.retire(generation)) current = null
+        // The only thing permitted to discard a pending frame: the texture it
+        // belongs to is going away, so latching it would pull from one nobody
+        // is drawing. Disarming the consumer must never do this.
+        framePending.retire(generation)
+        // Generation-checked like everything above it. A screen's teardown
+        // routinely runs after the next screen's setup, and this thread is
+        // shared by every session — quitting it while retiring an *old* texture
+        // leaves the live listener with a dead looper, which stops
+        // `onFrameAvailable` for good with no exception anywhere.
+        val wasLive = texture === liveTexture
+        if (wasLive) liveTexture = null
+        // Listener first: nothing may arrive once the surface is gone.
+        runCatching { texture.setOnFrameAvailableListener(null) }
+        runCatching { surface.release() }
+        runCatching { texture.release() }
+        if (wasLive) {
+            runCatching { frames?.quitSafely() }
+            threadQuits += 1
+            frames = null
+        }
+    }
+
+    /**
+     * Raise a flag when a frame lands, rather than doing work on the producer's
+     * thread. The UI runtime consumes it on its own tick.
+     */
+    /**
+     * One thread for the surface's whole life, not one per session.
+     *
+     * `onFrameAvailable` used to arrive on the main looper, which put UI
+     * scheduling delay inside the delivery measurement and did avoidable work
+     * on the thread that draws. This thread is created with the surface and
+     * quits with it, so sessions come and go without touching it.
+     */
+    @Volatile
+    private var frames: HandlerThread? = null
+
+    /*
+      Delivery bookkeeping.
+
+      `available` says frames stopped arriving; it cannot say whether the
+      listener was cleared, the thread that carries it was quit, or the producer
+      simply stopped. Those need opposite fixes, and the difference is three
+      counters and one liveness read.
+    */
+    @Volatile var watches: Long = 0
+        private set
+    @Volatile var releases: Long = 0
+        private set
+    @Volatile var threadQuits: Long = 0
+        private set
+
+    /** Whether the thread `onFrameAvailable` is delivered on can still run. */
+    @JvmStatic
+    fun deliveryAlive(): Boolean = frames?.isAlive == true
+
+    @JvmStatic
+    fun delivery(): LongArray =
+        longArrayOf(
+            watches,
+            releases,
+            threadQuits,
+            if (deliveryAlive()) 1 else 0,
+            // The acquisition side of the boundary, separated from the
+            // notification side. `announced` counts callbacks the listener
+            // actually received; `attempted`/`succeeded` count what the GL
+            // thread did with them. A gap between announced and attempted is a
+            // bookkeeping fault; a gap between attempted and succeeded is an
+            // acquisition fault; equality with a flat `announced` puts the
+            // fault before the callback.
+            framePending.announced,
+            framePending.rejected,
+            framePending.depth(),
+            updatesAttempted,
+            updatesSucceeded,
+            updatesFailed,
+            updatesSkipped,
+        )
+
+    @JvmStatic
+    fun watch(texture: SurfaceTexture, flag: java.util.concurrent.atomic.AtomicBoolean) {
+        val thread = frames ?: HandlerThread("pictor-frame-available")
+            .also { it.start(); frames = it }
+        liveTexture = texture
+        watches += 1
+        texture.setOnFrameAvailableListener(
+            { arrived ->
+                // Records availability and signals; nothing more. updateTexImage
+                // and every Skia call stay on the UI/GL runtime, because that is
+                // where the context lives.
+                FrameClock.markAvailable()
+                // Resolved now, from the texture the frame actually arrived on —
+                // never from a number captured when the listener was registered.
+                framePending.mark(generationOf(arrived))
+                flag.set(true)
+            },
+            Handler(thread.looper),
+        )
+    }
+
+    /**
+     * Which generation a texture belongs to, at the moment of asking.
+     *
+     * [SurfaceAttachment.NONE] for anything that is not the live texture, which
+     * is how a late listener's frame is rejected rather than credited to the
+     * surface that replaced it.
+     */
+    @JvmStatic
+    fun generationOf(texture: SurfaceTexture): Int =
+        if (texture === liveTexture) attachment.live else SurfaceAttachment.NONE
+
+    /**
+     * Stable identity of the producer side, for proving who is talking to whom.
+     *
+     * A consumer holding a stale object cannot be detected by comparing
+     * *values*: a captured surface that was `undefined` and a live one both
+     * answer "no frame pending", and every counter stays consistent while the
+     * two halves talk past each other. Identity is the only thing that
+     * distinguishes them, so it is published and recorded on both sides.
+     */
+    @JvmStatic
+    val controllerId: Int
+        get() = System.identityHashCode(this)
+
+    /** Identity of the pending counters the listener writes to. */
+    @JvmStatic
+    val pendingId: Int
+        get() = System.identityHashCode(framePending)
+
+    /** Whether a frame is waiting for [generation]. Level-triggered. */
+    @JvmStatic
+    fun hasPendingFrame(generation: Int): Boolean = framePending.pending(generation)
+
+    /** Pending-ness, depth and generation as one reading. See [FramePending.snapshot]. */
+    @JvmStatic
+    fun pendingSnapshot(generation: Int): Long = framePending.snapshot(generation)
+
+    /** The pending frame's intended presentation time, for scheduling the latch. */
+    @JvmStatic
+    fun pendingDueNs(): Long = FrameClock.pendingDueNs()
+
+    /**
+     * The generation the pending counters are keeping for.
+     *
+     * Published so a consumer can see *why* it is idle: pending for generation 2
+     * while the worklet asks about 3 is a completely different fault from
+     * nothing having arrived, and the two are indistinguishable from a single
+     * idle count.
+     */
+    @JvmStatic
+    val pendingGeneration: Int
+        get() = framePending.generation
+
+    /** Frames announced, ever. For the overlay. */
+    @JvmStatic
+    val framesAnnounced: Long
+        get() = framePending.announced
+
+    /** Announcements dropped because their generation was already retired. */
+    @JvmStatic
+    val framesRejected: Long
+        get() = framePending.rejected
+
+    /** Frames waiting right now. Usually 0 or 1; larger after a background. */
+    @JvmStatic
+    val pendingDepth: Long
+        get() = framePending.depth()
+
+    /**
+     * Stop callbacks before anything they touch goes away.
+     *
+     * Detaching is not optional on source replacement: a callback that lands
+     * after the generation moved would stamp availability for a frame of a clip
+     * that no longer exists.
+     */
+    @JvmStatic
+    fun unwatch(texture: SurfaceTexture) {
+        texture.setOnFrameAvailableListener(null)
+    }
+}