@pictor-sdk/video-android 0.1.0 → 0.1.3

package/android/build.gradle

@@ -22,15 +22,26 @@ android {
     // itself (26). None of that code exists any more. What the Media3 + external-OES
     // path actually needs is far lower — SurfaceTexture and GL_TEXTURE_EXTERNAL_OES
     // have been present since API 11 and 8, Media3 1.9.4's own floor is 23, and
-    // React Native 0.76 requires 24 — so 24 is the real requirement.
+    // React Native 0.76 requires 24, Media3 1.9.4 declares 23, and lint's NewApi
+    // check is what keeps every call above 24 behind a runtime branch. Nothing
+    // in the decode, texture or export path needs more.
     //
-    // It stays at 29 because nothing below it has been *run*. The only device here
-    // is API 36 and the one installed system image is incomplete, so 24 would be a
-    // support claim resting on reading API levels rather than on a passing test.
-    // Lower it when an older device or a complete emulator image can prove it.
-    // Installing this package raises a host's minSdk; not installing it leaves
-    // image-only Pictor consumers exactly where they were.
-    minSdk 29
+    // 29 was held here for a long time because nothing below it had been *run*.
+    // It cost every consumer the 24..28 range for a floor that was never a
+    // technical requirement — a host installing this adapter had to drop
+    // Android 7.0 through 9.0 for its whole app. See `docs/research/min-sdk-24.md`
+    // for what was audited and what is verified.
+    minSdk 24
+
+    /*
+      Shipped to the consumer, because the names are resolved from C++.
+
+      A host that minifies cannot know which of this package's classes are
+      reached by `FindClass`, and R8 cannot see a reference that exists only as
+      a string. Without these the app aborts inside the adapter's first native
+      call. See `consumer-rules.pro` for what is kept and why it is only that.
+    */
+    consumerProguardFiles "consumer-rules.pro"
     externalNativeBuild {
       cmake {
         cppFlags "-std=c++17"
@@ -102,7 +113,23 @@ repositories { google(); mavenCentral() }
  * adapter's consumers pay for; they are here because the gate needs them, not
  * in anticipation of a slice that has not started.
  */
-ext.media3Version = "1.9.4"
+/*
+  Overridable by the host, defaulting to the version this adapter is verified on.
+
+  A host that already has another Media3 consumer can be forced into a version
+  conflict it cannot resolve: Gradle unifies the *runtime* classpath upward, but
+  each module compiles against its own, and a module that vendors a Media3 class
+  (expo-video forks `DefaultLoadControl`) then implements an interface contract
+  that no longer matches. The failure is an `AbstractMethodError` on a background
+  thread at startup, naming neither package.
+
+  The default is unchanged and stays the verified one. Overriding it is a host
+  saying "I have measured that this combination works for me" — this adapter has
+  not, for any value but the default.
+*/
+ext.media3Version = rootProject.hasProperty("pictorMedia3Version")
+  ? rootProject.property("pictorMedia3Version")
+  : "1.9.4"
 
 dependencies {
   implementation "com.facebook.react:react-android"

package/android/consumer-rules.pro

@@ -0,0 +1,47 @@
+# Names this adapter's native code resolves at runtime.
+#
+# `pictor_videosurface.cpp` reaches Kotlin through `FindClass` and
+# `GetStaticMethodID`, which look names up as strings. R8 has no way to see
+# those references, so in a consumer that minifies it renames the classes and
+# the lookup returns null. That is not a soft failure: the JNI call that follows
+# aborts the process with
+#
+#     JNI DETECTED ERROR IN APPLICATION: java_class == null
+#
+# attributed to whichever thread happened to be running, which names neither
+# this package nor the class that went missing. Measured in a consumer with
+# `enableProguardInReleaseBuilds`: `dev.pictor.video.VideoSurface` had become
+# `oc.P0`, and the string `dev/pictor/video/VideoSurface` appeared in none of
+# the APK's dex files.
+#
+# Scoped to exactly the two classes native names and the members it calls.
+# Nothing here keeps React Native, Worklets, or anything of the consumer's:
+# an adapter that widened a host's keep rules to protect itself would be
+# trading the host's app size for its own convenience.
+#
+# `VideoSurface` and `FrameClock` are `internal object`s reached only from
+# native, so without these rules R8 is right to rename them — the reference it
+# cannot see is the one that matters.
+
+-keep class dev.pictor.video.VideoSurface {
+    public static *** create(...);
+    public static *** surfaceOf(...);
+    public static *** register(...);
+    public static *** watch(...);
+    public static *** release(...);
+    public static *** update(...);
+    public static *** hasPendingFrame(...);
+    public static *** pendingSnapshot(...);
+    public static *** pendingDueNs(...);
+    public static *** getControllerId(...);
+    public static *** getPendingId(...);
+    public static *** getPendingGeneration(...);
+    public static *** getPendingDepth(...);
+    public static *** timestamp(...);
+    public static *** transform(...);
+    public static *** controllerId(...);
+}
+
+-keep class dev.pictor.video.FrameClock {
+    public static *** take(...);
+}

package/android/src/main/AndroidManifest.xml

@@ -1 +1,15 @@
-<manifest xmlns:android="http://schemas.android.com/apk/res/android" />
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+  <!--
+    Only for API 24..28, where saving to the gallery means writing a real file
+    to shared storage. From API 29 the media store owns the write and no
+    permission is involved, so this is capped rather than merely unused: left
+    uncapped it would show up in every modern consumer's manifest for a path
+    none of their devices can reach.
+
+    Nothing else is declared here. Reading the source media is the host's, and
+    the decoder, the surface and the external texture need no permission at all.
+  -->
+  <uses-permission
+    android:name="android.permission.WRITE_EXTERNAL_STORAGE"
+    android:maxSdkVersion="28" />
+</manifest>

package/android/src/main/cpp/pictor_videosurface.cpp

@@ -16,12 +16,73 @@
 #include <EGL/egl.h>
 #include <GLES2/gl2.h>
 #include <GLES2/gl2ext.h>
+#include <android/log.h>
 #include <atomic>
 #include <jni.h>
 #include <mutex>
 #include <jsi/jsi.h>
 
 #include <ctime>
+
+/*
+  Named lookups that say what is missing instead of aborting.
+
+  `FindClass` and `GetStaticMethodID` answer with `nullptr` and a pending
+  exception. Calling on through that is what turns a renamed class into
+
+      JNI DETECTED ERROR IN APPLICATION: java_class == null
+
+  a process abort attributed to whichever thread was running — naming neither
+  this package nor the symbol that went missing. That cost a full investigation
+  once; these make the same failure say `dev/pictor/video/VideoSurface`.
+
+  A consumer that minifies without this package's `consumer-rules.pro` is the
+  case that hits it. Reporting is better than aborting even so: a host can see
+  the message, and a preview that does not start beats a process that dies.
+*/
+namespace {
+
+constexpr const char *kPictorJniTag = "PictorVideo";
+
+/** Clear anything pending so a later call is not blamed for this one. */
+void pictorClearPending(JNIEnv *e) {
+  if (e->ExceptionCheck()) {
+    e->ExceptionDescribe();
+    e->ExceptionClear();
+  }
+}
+
+/** `FindClass`, or `nullptr` with the name logged. Never leaves an exception. */
+jclass pictorFindClass(JNIEnv *e, const char *name) {
+  jclass cls = e->FindClass(name);
+  pictorClearPending(e);
+  if (cls == nullptr) {
+    __android_log_print(
+        ANDROID_LOG_ERROR, kPictorJniTag,
+        "class not found: %s — a minified build needs this package's "
+        "consumer-rules.pro (consumerProguardFiles), which keeps the names "
+        "this library resolves from native code",
+        name);
+  }
+  return cls;
+}
+
+/** `GetStaticMethodID`, or `nullptr` with the member logged. */
+jmethodID pictorStaticMethod(JNIEnv *e, jclass cls, const char *owner,
+                             const char *name, const char *sig) {
+  if (cls == nullptr) return nullptr;
+  jmethodID id = e->GetStaticMethodID(cls, name, sig);
+  pictorClearPending(e);
+  if (id == nullptr) {
+    __android_log_print(ANDROID_LOG_ERROR, kPictorJniTag,
+                        "method not found: %s.%s%s — kept the class but not "
+                        "the member; check consumer-rules.pro",
+                        owner, name, sig);
+  }
+  return id;
+}
+
+}  // namespace
 #include <memory>
 
 namespace jsi = facebook::jsi;
@@ -131,15 +192,29 @@ public:
                             GL_CLAMP_TO_EDGE);
             const GLenum error = glGetError();
 
-            jclass cls = e->FindClass("dev/pictor/video/VideoSurface");
-            jmethodID create = e->GetStaticMethodID(
-                cls, "create", "(III)Landroid/graphics/SurfaceTexture;");
+            jclass cls = pictorFindClass(e, "dev/pictor/video/VideoSurface");
+            jmethodID create = pictorStaticMethod(
+                e, cls, "VideoSurface", "create",
+                "(III)Landroid/graphics/SurfaceTexture;");
+            jmethodID surfaceOf = pictorStaticMethod(
+                e, cls, "VideoSurface", "surfaceOf",
+                "(Landroid/graphics/SurfaceTexture;)Landroid/view/Surface;");
+            if (cls == nullptr || create == nullptr || surfaceOf == nullptr) {
+              // No surface, and the log above says which name is missing. The
+              // host sees a preview that never starts rather than a crash it
+              // cannot attribute.
+              return jsi::Value(false);
+            }
             jobject texture = e->CallStaticObjectMethod(
                 cls, create, (jint)self->texture_, (jint)width, (jint)height);
-            jmethodID surfaceOf = e->GetStaticMethodID(
-                cls, "surfaceOf",
-                "(Landroid/graphics/SurfaceTexture;)Landroid/view/Surface;");
+            pictorClearPending(e);
             jobject surface = e->CallStaticObjectMethod(cls, surfaceOf, texture);
+            pictorClearPending(e);
+            if (texture == nullptr || surface == nullptr) {
+              __android_log_print(ANDROID_LOG_ERROR, kPictorJniTag,
+                                  "VideoSurface.create/surfaceOf returned null");
+              return jsi::Value(false);
+            }
 
             self->surfaceTexture_ = e->NewGlobalRef(texture);
             self->surface_ = e->NewGlobalRef(surface);
@@ -487,8 +562,13 @@ public:
                     : e->CallStaticLongMethod(self->cls_, stamp, self->surfaceTexture_);
             if (e->ExceptionCheck()) e->ExceptionClear();
 
-            jclass clock = e->FindClass("dev/pictor/video/FrameClock");
-            jmethodID take = e->GetStaticMethodID(clock, "take", "(JJ)[J");
+            jclass clock = pictorFindClass(e, "dev/pictor/video/FrameClock");
+            jmethodID take =
+                pictorStaticMethod(e, clock, "FrameClock", "take", "(JJ)[J");
+            if (clock == nullptr || take == nullptr) {
+              self->ptsUs_ = -1;
+              return jsi::Value(-1.0);
+            }
             auto paired = (jlongArray)e->CallStaticObjectMethod(
                 clock, take, (jlong)consumedNs, texturePtsNs);
             if (paired == nullptr) {

package/android/src/main/java/dev/pictor/video/AudioMixer.kt

@@ -276,6 +276,15 @@ internal object AudioMixer {
             val mixed = FloatArray((totalSamples * CHANNELS).toInt())
 
             val contributed = JSONObject()
+            /*
+              Spans that turned out to have no audio at all.
+
+              Reported rather than merely tolerated: "the mix is silent because
+              the clips have no sound" and "the mix is silent because something
+              went wrong" are different, and only one of them is worth anybody's
+              time.
+            */
+            val silentForNoTrack = mutableListOf<String>()
             for (span in spans) {
                 if (span.silent) {
                     // An empty contribution. Nothing is generated and nothing is
@@ -283,10 +292,34 @@ internal object AudioMixer {
                     contributed.put(span.id, 0)
                     continue
                 }
-                val decoded = profile.time("decode.total") {
-                    decodeToOutputFormat(
-                        context, span.uri!!, span.sourceInUs, span.sourceOutUs, span.speed,
-                    )
+                /*
+                  A source with no audio track contributes silence.
+
+                  It is not a failure. A clip recorded with the microphone off,
+                  a screen capture, an animation exported without sound — all
+                  are ordinary things to put on a timeline, and the mix's job is
+                  to place what audio exists, not to insist that every clip has
+                  some. Refusing here failed the whole export with `no audio
+                  track in <file>` and left the person with a video they could
+                  only fix by adding sound they did not want.
+
+                  Narrow on purpose: only `find-track` is treated this way.
+                  Every other decode failure — a corrupt file, an unsupported
+                  encoding, a decoder that never finishes — still fails the mix,
+                  because those are cases where audio was expected and could not
+                  be read, which is a different thing from there being none.
+                */
+                val decoded = try {
+                    profile.time("decode.total") {
+                        decodeToOutputFormat(
+                            context, span.uri!!, span.sourceInUs, span.sourceOutUs, span.speed,
+                        )
+                    }
+                } catch (e: DecodeFailure) {
+                    if (e.stage != "find-track") throw e
+                    contributed.put(span.id, 0)
+                    silentForNoTrack.add(span.id)
+                    continue
                 }
                 profile.source(
                     span.id,
@@ -376,6 +409,8 @@ internal object AudioMixer {
             if (!encoded.optBoolean("ok", false)) return encoded.toString()
 
             out.put("ok", true)
+                // Named, so a silent mix can be explained rather than guessed at.
+                .put("silentForNoTrack", org.json.JSONArray(silentForNoTrack))
                 .put("uri", output.absolutePath)
                 .put("sampleRate", SAMPLE_RATE)
                 .put("channels", CHANNELS)

package/android/src/main/java/dev/pictor/video/BoundedWait.kt

@@ -0,0 +1,89 @@
+package dev.pictor.video
+
+/**
+ * Waiting for a run to settle, with an end that a cancel can bring forward.
+ *
+ * ## What this replaces, and what it cost
+ *
+ * The wait used to be a `Boolean`: settled, or not. A cancel released the
+ * latch, so "not settled" could only mean the deadline had expired — and that
+ * reasoning was only sound while the release was guaranteed. It is not. A
+ * cancel arriving before a Transformer has published, or after its run has
+ * already been cleared, has no latch to count down; it sets the cancelled flag
+ * and returns. The waiter then sat out its whole deadline with the flag true
+ * beside it, and **logged that flag every four seconds while doing so**.
+ *
+ * Measured, six times: a wedged shard reported its stall in about seven seconds
+ * and the export settled 600 010 ms later, because 600 s was the deadline it
+ * had been given. A person watching an export that is already known to have
+ * failed waits ten minutes.
+ *
+ * So the flag is now part of the exit condition rather than a decoration on the
+ * log line, and the answer is three-valued: a deadline that genuinely expired
+ * and a run that was cancelled are different failures, and only one of them
+ * means the platform stopped answering.
+ *
+ * ## Settling outranks cancelling
+ *
+ * Checked in that order deliberately. A run that finished while the cancel was
+ * in flight has a real result, and reporting it as cancelled would throw away
+ * an export that succeeded.
+ *
+ * ## No Android, on purpose
+ *
+ * Time, waiting and the cancel flag all arrive as functions, so the orderings
+ * that caused this can be tested on the JVM instead of being restated in a
+ * comment. `RunRegistry` is factored the same way and for the same reason.
+ */
+internal object BoundedWait {
+
+    enum class Outcome {
+        /** The run answered: a result or a failure is waiting to be read. */
+        Settled,
+
+        /** A cancel claimed the run before it answered. */
+        Cancelled,
+
+        /** The deadline expired with no answer and no cancel. */
+        TimedOut,
+    }
+
+    /**
+     * Wait in slices until the run settles, a cancel claims it, or time runs out.
+     *
+     * @param awaitSlice waits up to that many milliseconds, true if it settled.
+     * @param cancelled read between slices, and once before the first.
+     * @param onSlice called with the total waited so far, for the progress log.
+     */
+    fun await(
+        timeoutMs: Long,
+        sliceMs: Long,
+        nowMs: () -> Long,
+        awaitSlice: (Long) -> Boolean,
+        cancelled: () -> Boolean,
+        onSlice: (Long) -> Unit,
+    ): Outcome {
+        val deadlineMs = nowMs() + timeoutMs
+        /*
+          Before the first slice, not only between them.
+
+          A cancel that lands while the composition is still being built is the
+          common case — it is exactly the one with no published run to claim —
+          and waiting a whole slice to notice it would put the floor for a
+          cancelled export at the reporting interval.
+        */
+        if (cancelled()) return Outcome.Cancelled
+        var waitedMs = 0L
+        while (true) {
+            val remainingMs = deadlineMs - nowMs()
+            if (remainingMs <= 0) return Outcome.TimedOut
+            val slice = minOf(sliceMs, remainingMs)
+            if (awaitSlice(slice)) return Outcome.Settled
+            // The run may have settled and been cancelled in the same slice;
+            // `awaitSlice` answered first, so this only runs when it did not.
+            if (cancelled()) return Outcome.Cancelled
+            waitedMs += slice
+            onSlice(waitedMs)
+        }
+    }
+}

package/android/src/main/java/dev/pictor/video/CadenceShaderProgram.kt

@@ -67,10 +67,25 @@ internal class CadenceEffect(
     val glDeleted = AtomicLong(0)
     val peakPending = AtomicLong(0)
     val stalls = AtomicLong(0)
+    /**
+     * End-of-stream, in and out.
+     *
+     * A frame counter that reaches its expected total says the *effect* did its
+     * work. It says nothing about whether the stream was ever declared over,
+     * which is the signal the encoder and the muxer are actually waiting on —
+     * and mistaking one for the other is how a shard that "finished" was
+     * reported as complete while nothing downstream had finished at all.
+     */
+    val endIn = AtomicLong(0)
+    val endOut = AtomicLong(0)
+    /** Outstanding copies the moment the stream was declared over. */
+    val pendingAtEnd = AtomicLong(-1)
     private val problems = mutableListOf<String>()
 
     fun trace(): String = synchronized(problems) {
         "inputs=" + inputsSeen.get() + " emitted=" + emitted.get() +
+            " endIn=" + endIn.get() + " endOut=" + endOut.get() +
+            " pendingAtEnd=" + pendingAtEnd.get() +
             " glCreated=" + glCreated.get() + " glDeleted=" + glDeleted.get() +
             " peakPending=" + peakPending.get() + " stalls=" + stalls.get() +
             (if (problems.isEmpty()) "" else " problems=" + problems.joinToString("|"))
@@ -147,6 +162,18 @@ internal class CadenceEffect(
                     */
                     inputListener?.onInputFrameProcessed(input)
                     inputListener?.onReadyToAcceptInputFrame()
+                    /*
+                      Ask, even though nothing was emitted.
+
+                      Every other path that finishes with an input ends by
+                      asking whether the stream is over; this one returned
+                      early. Above 1× most frames take this path, so if the
+                      end-of-stream signal arrives while one is in flight, the
+                      condition it was waiting for becomes true here and
+                      nowhere else — and the guard that would have noticed is
+                      the one being skipped.
+                    */
+                    maybeSignalEnd()
                     return
                 }
                 heldInput = input
@@ -253,6 +280,7 @@ internal class CadenceEffect(
         }
 
         override fun signalEndOfCurrentInputStream() {
+            endIn.incrementAndGet()
             endOfStreamSignalled = true
             maybeSignalEnd()
         }
@@ -268,6 +296,7 @@ internal class CadenceEffect(
          */
         private fun maybeSignalEnd() {
             if (endOfStreamSignalled && heldInput == null && owed == 0) {
+                if (endOut.getAndIncrement() == 0L) pendingAtEnd.set(pending.toLong())
                 outputListener?.onCurrentOutputStreamEnded()
             }
         }