@pictor-sdk/video-android 0.1.0

package/android/src/main/java/dev/pictor/video/PreviewMix.kt

@@ -0,0 +1,396 @@
+package dev.pictor.video
+
+import android.content.Context
+import java.io.File
+import java.security.MessageDigest
+import java.util.concurrent.Executors
+import java.util.concurrent.atomic.AtomicLong
+import org.json.JSONArray
+import org.json.JSONObject
+
+/**
+ * The audio a preview plays, prepared once and reused.
+ *
+ * ## Why a file, and why the same mixer
+ *
+ * Preview and export must agree about what a project sounds like, and the only
+ * way to guarantee that is for both to run the same code over the same plan.
+ * [AudioMixer] already sums spans, applies gains and ramps, stretches by clip
+ * speed and saturates once — so preview asks it for exactly what an export
+ * asks, and plays the result. Reimplementing any of that in the player, or in
+ * React, would be a second definition of the sound.
+ *
+ * The output is an app-private file rather than a stream because the mix is not
+ * realtime: it decodes every contributing source, re-times it and encodes once.
+ * See the latency table in the JS wrapper — a caller has to know this is
+ * preparation, not playback.
+ *
+ * ## Keyed by what the sound depends on, and nothing else
+ *
+ * The cache key is the digest of the compiled mix program, the resolved source
+ * URIs and the mixer's own configuration. That is precisely the set of things
+ * that can change the samples: selecting a clip, opening a sheet or scrolling
+ * the timeline change none of them, so they cannot invalidate a mix. A trim, a
+ * volume, a fade, a speed, a mute or a different source all change the program,
+ * so they all do.
+ *
+ * ## One job, and the newest wins
+ *
+ * A newer request cancels the one in flight. The old job unwinds through
+ * [AudioMixer.Cancelled], deletes its partial output and reports `superseded` —
+ * which the caller must not publish. Generation numbers rather than flags: a
+ * job compares the generation it was started with against the current one, so a
+ * completion that arrives late can tell that it is late.
+ */
+object PreviewMix {
+
+    private const val DIR = "pictor-preview"
+    private const val SUFFIX = ".m4a"
+    private const val PART_SUFFIX = ".part"
+
+    /**
+     * The mixer configuration that reaches the key.
+     *
+     * Bumped when anything about how the samples are produced changes, so a
+     * cached file from an older rule is never played. It is not a schema
+     * version: the program carries its own.
+     *
+     * `v3` is the priming shift. `v1` never corrected the delay at all and plays
+     * about 43 ms late; `v2` corrected it by dropping access units and lost the
+     * first 43 ms of any sound starting at zero. Both are silently wrong rather
+     * than merely slow, which is the worse thing for a cache to be.
+     */
+    private const val MIXER_CONFIG = "48000/2/sum-then-saturate/v3-primingshifted"
+
+    private val executor = Executors.newSingleThreadExecutor { r ->
+        Thread(r, "pictor-preview-mix").apply { isDaemon = true }
+    }
+
+    /** The generation a job must still match to be allowed to publish. */
+    private val generation = AtomicLong(0)
+
+    /** Bumped by [cancel] and by every new request. */
+    @Volatile private var currentJob: String? = null
+
+    /** Preparations started and finished, so an abandoned one is visible. */
+    @Volatile private var jobsStarted = 0L
+    @Volatile private var jobsFinished = 0L
+
+    fun jobCounters(): JSONObject =
+        JSONObject()
+            .put("started", jobsStarted)
+            .put("finished", jobsFinished)
+            .put("live", jobsStarted - jobsFinished)
+            .put("current", currentJob ?: "")
+
+    private fun dir(context: Context): File {
+        // `filesDir`: a cached mix that vanished mid-session would silently
+        // become "Preparing audio" again with nothing to explain it.
+        val d = File(context.filesDir, DIR)
+        if (!d.exists()) d.mkdirs()
+        return d
+    }
+
+    /**
+     * The name a given sound has.
+     *
+     * Content-addressed like a managed source, and for one of the same reasons:
+     * a partial file can never wear the name of a complete one, because the
+     * name is decided before the work and the file only takes it after a
+     * successful rename.
+     */
+    fun keyFor(programJson: String, urisJson: String): String {
+        val digest = MessageDigest.getInstance("SHA-256")
+        digest.update(MIXER_CONFIG.toByteArray())
+        digest.update(programJson.toByteArray())
+        digest.update(urisJson.toByteArray())
+        return digest.digest().joinToString("") { "%02x".format(it) }
+    }
+
+    /** Where a prepared mix lives, if it is there. */
+    fun cached(context: Context, key: String): File? {
+        val file = File(dir(context), key + SUFFIX)
+        return if (file.exists() && file.length() > 0) file else null
+    }
+
+    /**
+     * Prepare the mix for this program, or return the one already prepared.
+     *
+     * Blocking: the caller runs it on [executor] via the module. Returns
+     * `{ok, uri, key, cached, durationUs, byteLength}`, or `{ok:false, ...}`
+     * with `superseded` or `cancelled` set when it stopped rather than failed.
+     */
+    fun prepare(context: Context, jobId: String, programJson: String, urisJson: String): String =
+        try {
+            prepareInner(context, jobId, programJson, urisJson)
+        } finally {
+            /*
+              Every exit, not just the cancelled one.
+
+              The first version of this counter only closed a job in `cancel`,
+              so a run of sixteen rebuilds reported sixteen live jobs and no
+              finished ones — a measurement artefact that reads exactly like a
+              leak. `finally` is what makes "started" and "finished" answer the
+              same question.
+            */
+            jobsFinished += 1
+        }
+
+    private fun prepareInner(
+        context: Context,
+        jobId: String,
+        programJson: String,
+        urisJson: String,
+    ): String {
+        val out = JSONObject()
+        val key = keyFor(programJson, urisJson)
+        val mine = generation.incrementAndGet()
+        currentJob = jobId
+        jobsStarted += 1
+        PreviewTrace.record("mix.start") {
+            JSONObject().put("job", jobId).put("key", key.take(12)).put("gen", mine)
+        }
+
+        cached(context, key)?.let { file ->
+            // Already prepared. Still checked against the generation: a request
+            // superseded before it got here must not publish, even instantly.
+            if (generation.get() != mine) {
+                return out.put("ok", false).put("superseded", true).put("key", key).toString()
+            }
+            PreviewTrace.record("mix.cached") {
+                JSONObject().put("key", key.take(12)).put("gen", mine)
+            }
+            return out
+                .put("ok", true)
+                .put("uri", android.net.Uri.fromFile(file).toString())
+                .put("key", key)
+                .put("cached", true)
+                .put("byteLength", file.length())
+                .toString()
+        }
+
+        val part = File(dir(context), "$key$SUFFIX$PART_SUFFIX")
+        CodecLedger.currentJob = jobId
+        try {
+            val result = JSONObject(
+                AudioMixer.mix(
+                    context,
+                    programJson,
+                    urisJson,
+                    part,
+                    // Cancelled the moment a newer request arrives. Checked in
+                    // every loop that can run for seconds, so superseding does
+                    // not wait out a decode and an encode.
+                    keepGoing = { generation.get() == mine },
+                ),
+            )
+            if (!result.optBoolean("ok", false)) {
+                runCatching { part.delete() }
+                return out
+                    .put("ok", false)
+                    .put("key", key)
+                    .put("error", result.optString("error", "mix failed"))
+                    .toString()
+            }
+        } catch (e: AudioMixer.Cancelled) {
+            PreviewTrace.record("mix.superseded") {
+                JSONObject().put("key", key.take(12)).put("gen", mine).put("at", "mix")
+            }
+            // Superseded or cancelled. The partial goes with it: a half-encoded
+            // file under a name nothing will ask for is storage nobody can
+            // account for.
+            runCatching { part.delete() }
+            return out.put("ok", false).put("superseded", true).put("key", key).toString()
+        } catch (e: Throwable) {
+            runCatching { part.delete() }
+            return out
+                .put("ok", false)
+                .put("key", key)
+                .put("error", e.javaClass.simpleName + (e.message?.let { ": $it" } ?: ""))
+                .toString()
+        }
+
+        if (generation.get() != mine) {
+            // Finished, but the answer is to a question nobody is asking any
+            // more. Never published, and never left behind.
+            PreviewTrace.record("mix.superseded") {
+                JSONObject().put("key", key.take(12)).put("gen", mine).put("at", "afterMix")
+            }
+            runCatching { part.delete() }
+            return out.put("ok", false).put("superseded", true).put("key", key).toString()
+        }
+
+        /*
+          Preparation gives its codecs back before anything adopts the file.
+
+          Every codec a job creates is its own to release, and a hardware
+          instance that outlives the job is one the video decoder cannot have.
+          Reported rather than enforced with a wait: a publication that blocked
+          on a stuck codec would turn a resource leak into a hang, and the
+          count is what identifies the defect either way.
+        */
+        CodecLedger.trimCompleted()
+        val codecs = CodecLedger.snapshot()
+        val liveAfterMix = codecs.optInt("live", -1)
+        val profile = AudioMixer.profile.json()
+
+        /*
+          The start moves before anything adopts the file.
+
+          `AudioMixer.encode` is left exactly as it was — its output is the
+          frozen baseline the encoder rewrite was checked against, and moving
+          the trim inside it would invalidate that and change what an export
+          mixes from. So the raw mix is written, trimmed once into the name the
+          cache will use, and the raw copy removed. Corrected exactly once, at
+          the point of publication, by the same code the export mux uses.
+        */
+        val trimAt = System.nanoTime()
+        val target = File(dir(context), key + SUFFIX)
+        /*
+          The compiled end reaches the publication, so preview follows the same
+          tail policy as an export.
+
+          Without it the mix ran 643 frames — 13.4 ms — past the timeline: the
+          encoder pads its last frame and nothing was trimming it. Marker
+          alignment says nothing about the end, which is why the duration is
+          checked separately from the offsets.
+        */
+        val programEndUs = try {
+            org.json.JSONObject(programJson).optLong("durationUs", 0)
+        } catch (e: Exception) {
+            0L
+        }
+        val trimmed = PrimingTrim.shiftAudioFile(
+            context, part, target, programEndUs,
+            // Superseding still wins during the trim: it is short, but a rapid
+            // edit must not have to wait out a remux either.
+            keepGoing = { generation.get() == mine },
+        )
+        val trimMs = (System.nanoTime() - trimAt) / 1_000_000
+        runCatching { part.delete() }
+        if (!trimmed.optBoolean("ok", false)) {
+            // Both outputs go: the raw mix above, and whatever the trim managed
+            // to write under the published name.
+            runCatching { target.delete() }
+            if (trimmed.optBoolean("cancelled", false)) {
+                return out.put("ok", false).put("superseded", true).put("key", key).toString()
+            }
+            return out
+                .put("ok", false)
+                .put("key", key)
+                .put("error", trimmed.optString("error", "the mix could not be corrected"))
+                .toString()
+        }
+        if (generation.get() != mine) {
+            runCatching { target.delete() }
+            return out.put("ok", false).put("superseded", true).put("key", key).toString()
+        }
+
+        PreviewTrace.record("mix.published") {
+            JSONObject().put("key", key.take(12)).put("gen", mine)
+                .put("bytes", target.length()).put("trimMs", trimMs)
+        }
+        return out
+            .put("ok", true)
+            .put("uri", android.net.Uri.fromFile(target).toString())
+            .put("key", key)
+            .put("cached", false)
+            .put("byteLength", target.length())
+            // Where the time went. A multi-second preparation is a product
+            // problem, and the first thing anyone will do about it is guess.
+            .put("profile", profile)
+            .put("trimMs", trimMs)
+            .put("codecsLiveAfterMix", liveAfterMix)
+            .put("codecsPeakLive", codecs.optInt("peakLive", -1))
+            .put("priming", trimmed)
+            .toString()
+    }
+
+    /**
+     * Stop whatever is preparing.
+     *
+     * Bumping the generation is the whole mechanism: the running job's
+     * `keepGoing` closure compares against it and unwinds at its next
+     * checkpoint.
+     */
+    fun cancel(): String {
+        PreviewTrace.record("mix.cancel") {
+            JSONObject().put("job", currentJob ?: "").put("gen", generation.get())
+        }
+        generation.incrementAndGet()
+        val was = currentJob
+        currentJob = null
+        return JSONObject().put("ok", true).put("cancelled", was ?: "").toString()
+    }
+
+    /** Drop one prepared mix. */
+    fun release(context: Context, key: String): String {
+        if (!safeKey(key)) {
+            return JSONObject().put("ok", false).put("error", "unsafe key").toString()
+        }
+        val file = File(dir(context), key + SUFFIX)
+        val removed = file.exists() && file.delete()
+        return JSONObject().put("ok", true).put("removed", removed).toString()
+    }
+
+    /**
+     * Remove every prepared mix except the ones named, and every partial.
+     *
+     * The bound on the cache. A caller keeps the key it is playing and lets the
+     * rest go; nothing here decides on its own that a file is stale, for the
+     * same reason nothing deletes a managed source implicitly.
+     */
+    fun sweep(context: Context, keep: JSONArray): String {
+        val keepSet = HashSet<String>(keep.length())
+        for (i in 0 until keep.length()) keepSet.add(keep.optString(i))
+
+        var removed = 0
+        var kept = 0
+        var partials = 0
+        var skipped = 0
+        for (file in dir(context).listFiles() ?: emptyArray()) {
+            val name = file.name
+            when {
+                name.endsWith(PART_SUFFIX) -> if (file.delete()) partials += 1
+                name.endsWith(SUFFIX) && safeKey(name.removeSuffix(SUFFIX)) -> {
+                    if (keepSet.contains(name.removeSuffix(SUFFIX))) kept += 1
+                    else if (file.delete()) removed += 1
+                }
+                // Not ours, and therefore not ours to delete — the rule the
+                // managed-source sweep had to learn the hard way.
+                else -> skipped += 1
+            }
+        }
+        return JSONObject()
+            .put("ok", true)
+            .put("removed", removed)
+            .put("kept", kept)
+            .put("partials", partials)
+            .put("skipped", skipped)
+            .toString()
+    }
+
+    /** What is prepared, for assertions. */
+    fun ledger(context: Context): String {
+        val items = JSONArray()
+        var bytes = 0L
+        for (file in (dir(context).listFiles() ?: emptyArray()).sortedBy { it.name }) {
+            bytes += file.length()
+            items.put(JSONObject().put("name", file.name).put("byteLength", file.length()))
+        }
+        return JSONObject()
+            .put("ok", true)
+            .put("count", items.length())
+            .put("bytes", bytes)
+            .put("items", items)
+            .toString()
+    }
+
+    /** The executor the module runs [prepare] on. */
+    fun submit(task: () -> Unit) {
+        executor.execute(task)
+    }
+
+    private fun safeKey(key: String): Boolean =
+        key.length == 64 && key.all { it in '0'..'9' || it in 'a'..'f' }
+}