@pictor-sdk/video-android 0.1.0 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/android/build.gradle +36 -9
- package/android/consumer-rules.pro +47 -0
- package/android/src/main/AndroidManifest.xml +15 -1
- package/android/src/main/cpp/pictor_videosurface.cpp +88 -8
- package/android/src/main/java/dev/pictor/video/AudioMixer.kt +39 -4
- package/android/src/main/java/dev/pictor/video/BoundedWait.kt +89 -0
- package/android/src/main/java/dev/pictor/video/CadenceShaderProgram.kt +29 -0
- package/android/src/main/java/dev/pictor/video/CompositionSeek.kt +8 -0
- package/android/src/main/java/dev/pictor/video/CompositionSpike.kt +23 -13
- package/android/src/main/java/dev/pictor/video/CompositionVideoItem.kt +16 -0
- package/android/src/main/java/dev/pictor/video/CompositionWindow.kt +17 -0
- package/android/src/main/java/dev/pictor/video/ExportGate.kt +421 -44
- package/android/src/main/java/dev/pictor/video/FrameClock.kt +6 -1
- package/android/src/main/java/dev/pictor/video/GateLadder.kt +405 -0
- package/android/src/main/java/dev/pictor/video/MediaStoreSave.kt +216 -9
- package/android/src/main/java/dev/pictor/video/PictorVideoModule.kt +13 -0
- package/android/src/main/java/dev/pictor/video/RunRegistry.kt +108 -0
- package/android/src/main/java/dev/pictor/video/SkiaPassThroughEffect.kt +4 -0
- package/dist/index.d.mts +27 -0
- package/dist/index.d.ts +27 -0
- package/dist/index.js +15 -3
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +15 -3
- package/dist/index.mjs.map +1 -1
- package/package.json +3 -3
- package/plugin/gradleProperties.js +9 -6
- package/src/NativePictorVideo.ts +11 -0
- package/src/compositionSpike.ts +9 -0
- package/src/exporter.ts +41 -3
package/android/build.gradle
CHANGED
|
@@ -22,15 +22,26 @@ android {
|
|
|
22
22
|
// itself (26). None of that code exists any more. What the Media3 + external-OES
|
|
23
23
|
// path actually needs is far lower — SurfaceTexture and GL_TEXTURE_EXTERNAL_OES
|
|
24
24
|
// have been present since API 11 and 8, Media3 1.9.4's own floor is 23, and
|
|
25
|
-
// React Native 0.76 requires 24
|
|
25
|
+
// React Native 0.76 requires 24, Media3 1.9.4 declares 23, and lint's NewApi
|
|
26
|
+
// check is what keeps every call above 24 behind a runtime branch. Nothing
|
|
27
|
+
// in the decode, texture or export path needs more.
|
|
26
28
|
//
|
|
27
|
-
//
|
|
28
|
-
//
|
|
29
|
-
//
|
|
30
|
-
//
|
|
31
|
-
//
|
|
32
|
-
|
|
33
|
-
|
|
29
|
+
// 29 was held here for a long time because nothing below it had been *run*.
|
|
30
|
+
// It cost every consumer the 24..28 range for a floor that was never a
|
|
31
|
+
// technical requirement — a host installing this adapter had to drop
|
|
32
|
+
// Android 7.0 through 9.0 for its whole app. See `docs/research/min-sdk-24.md`
|
|
33
|
+
// for what was audited and what is verified.
|
|
34
|
+
minSdk 24
|
|
35
|
+
|
|
36
|
+
/*
|
|
37
|
+
Shipped to the consumer, because the names are resolved from C++.
|
|
38
|
+
|
|
39
|
+
A host that minifies cannot know which of this package's classes are
|
|
40
|
+
reached by `FindClass`, and R8 cannot see a reference that exists only as
|
|
41
|
+
a string. Without these the app aborts inside the adapter's first native
|
|
42
|
+
call. See `consumer-rules.pro` for what is kept and why it is only that.
|
|
43
|
+
*/
|
|
44
|
+
consumerProguardFiles "consumer-rules.pro"
|
|
34
45
|
externalNativeBuild {
|
|
35
46
|
cmake {
|
|
36
47
|
cppFlags "-std=c++17"
|
|
@@ -102,7 +113,23 @@ repositories { google(); mavenCentral() }
|
|
|
102
113
|
* adapter's consumers pay for; they are here because the gate needs them, not
|
|
103
114
|
* in anticipation of a slice that has not started.
|
|
104
115
|
*/
|
|
105
|
-
|
|
116
|
+
/*
|
|
117
|
+
Overridable by the host, defaulting to the version this adapter is verified on.
|
|
118
|
+
|
|
119
|
+
A host that already has another Media3 consumer can be forced into a version
|
|
120
|
+
conflict it cannot resolve: Gradle unifies the *runtime* classpath upward, but
|
|
121
|
+
each module compiles against its own, and a module that vendors a Media3 class
|
|
122
|
+
(expo-video forks `DefaultLoadControl`) then implements an interface contract
|
|
123
|
+
that no longer matches. The failure is an `AbstractMethodError` on a background
|
|
124
|
+
thread at startup, naming neither package.
|
|
125
|
+
|
|
126
|
+
The default is unchanged and stays the verified one. Overriding it is a host
|
|
127
|
+
saying "I have measured that this combination works for me" — this adapter has
|
|
128
|
+
not, for any value but the default.
|
|
129
|
+
*/
|
|
130
|
+
ext.media3Version = rootProject.hasProperty("pictorMedia3Version")
|
|
131
|
+
? rootProject.property("pictorMedia3Version")
|
|
132
|
+
: "1.9.4"
|
|
106
133
|
|
|
107
134
|
dependencies {
|
|
108
135
|
implementation "com.facebook.react:react-android"
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# Names this adapter's native code resolves at runtime.
|
|
2
|
+
#
|
|
3
|
+
# `pictor_videosurface.cpp` reaches Kotlin through `FindClass` and
|
|
4
|
+
# `GetStaticMethodID`, which look names up as strings. R8 has no way to see
|
|
5
|
+
# those references, so in a consumer that minifies it renames the classes and
|
|
6
|
+
# the lookup returns null. That is not a soft failure: the JNI call that follows
|
|
7
|
+
# aborts the process with
|
|
8
|
+
#
|
|
9
|
+
# JNI DETECTED ERROR IN APPLICATION: java_class == null
|
|
10
|
+
#
|
|
11
|
+
# attributed to whichever thread happened to be running, which names neither
|
|
12
|
+
# this package nor the class that went missing. Measured in a consumer with
|
|
13
|
+
# `enableProguardInReleaseBuilds`: `dev.pictor.video.VideoSurface` had become
|
|
14
|
+
# `oc.P0`, and the string `dev/pictor/video/VideoSurface` appeared in none of
|
|
15
|
+
# the APK's dex files.
|
|
16
|
+
#
|
|
17
|
+
# Scoped to exactly the two classes native names and the members it calls.
|
|
18
|
+
# Nothing here keeps React Native, Worklets, or anything of the consumer's:
|
|
19
|
+
# an adapter that widened a host's keep rules to protect itself would be
|
|
20
|
+
# trading the host's app size for its own convenience.
|
|
21
|
+
#
|
|
22
|
+
# `VideoSurface` and `FrameClock` are `internal object`s reached only from
|
|
23
|
+
# native, so without these rules R8 is right to rename them — the reference it
|
|
24
|
+
# cannot see is the one that matters.
|
|
25
|
+
|
|
26
|
+
-keep class dev.pictor.video.VideoSurface {
|
|
27
|
+
public static *** create(...);
|
|
28
|
+
public static *** surfaceOf(...);
|
|
29
|
+
public static *** register(...);
|
|
30
|
+
public static *** watch(...);
|
|
31
|
+
public static *** release(...);
|
|
32
|
+
public static *** update(...);
|
|
33
|
+
public static *** hasPendingFrame(...);
|
|
34
|
+
public static *** pendingSnapshot(...);
|
|
35
|
+
public static *** pendingDueNs(...);
|
|
36
|
+
public static *** getControllerId(...);
|
|
37
|
+
public static *** getPendingId(...);
|
|
38
|
+
public static *** getPendingGeneration(...);
|
|
39
|
+
public static *** getPendingDepth(...);
|
|
40
|
+
public static *** timestamp(...);
|
|
41
|
+
public static *** transform(...);
|
|
42
|
+
public static *** controllerId(...);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
-keep class dev.pictor.video.FrameClock {
|
|
46
|
+
public static *** take(...);
|
|
47
|
+
}
|
|
@@ -1 +1,15 @@
|
|
|
1
|
-
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
|
|
1
|
+
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
|
|
2
|
+
<!--
|
|
3
|
+
Only for API 24..28, where saving to the gallery means writing a real file
|
|
4
|
+
to shared storage. From API 29 the media store owns the write and no
|
|
5
|
+
permission is involved, so this is capped rather than merely unused: left
|
|
6
|
+
uncapped it would show up in every modern consumer's manifest for a path
|
|
7
|
+
none of their devices can reach.
|
|
8
|
+
|
|
9
|
+
Nothing else is declared here. Reading the source media is the host's, and
|
|
10
|
+
the decoder, the surface and the external texture need no permission at all.
|
|
11
|
+
-->
|
|
12
|
+
<uses-permission
|
|
13
|
+
android:name="android.permission.WRITE_EXTERNAL_STORAGE"
|
|
14
|
+
android:maxSdkVersion="28" />
|
|
15
|
+
</manifest>
|
|
@@ -16,12 +16,73 @@
|
|
|
16
16
|
#include <EGL/egl.h>
|
|
17
17
|
#include <GLES2/gl2.h>
|
|
18
18
|
#include <GLES2/gl2ext.h>
|
|
19
|
+
#include <android/log.h>
|
|
19
20
|
#include <atomic>
|
|
20
21
|
#include <jni.h>
|
|
21
22
|
#include <mutex>
|
|
22
23
|
#include <jsi/jsi.h>
|
|
23
24
|
|
|
24
25
|
#include <ctime>
|
|
26
|
+
|
|
27
|
+
/*
|
|
28
|
+
Named lookups that say what is missing instead of aborting.
|
|
29
|
+
|
|
30
|
+
`FindClass` and `GetStaticMethodID` answer with `nullptr` and a pending
|
|
31
|
+
exception. Calling on through that is what turns a renamed class into
|
|
32
|
+
|
|
33
|
+
JNI DETECTED ERROR IN APPLICATION: java_class == null
|
|
34
|
+
|
|
35
|
+
a process abort attributed to whichever thread was running — naming neither
|
|
36
|
+
this package nor the symbol that went missing. That cost a full investigation
|
|
37
|
+
once; these make the same failure say `dev/pictor/video/VideoSurface`.
|
|
38
|
+
|
|
39
|
+
A consumer that minifies without this package's `consumer-rules.pro` is the
|
|
40
|
+
case that hits it. Reporting is better than aborting even so: a host can see
|
|
41
|
+
the message, and a preview that does not start beats a process that dies.
|
|
42
|
+
*/
|
|
43
|
+
namespace {
|
|
44
|
+
|
|
45
|
+
constexpr const char *kPictorJniTag = "PictorVideo";
|
|
46
|
+
|
|
47
|
+
/** Clear anything pending so a later call is not blamed for this one. */
|
|
48
|
+
void pictorClearPending(JNIEnv *e) {
|
|
49
|
+
if (e->ExceptionCheck()) {
|
|
50
|
+
e->ExceptionDescribe();
|
|
51
|
+
e->ExceptionClear();
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** `FindClass`, or `nullptr` with the name logged. Never leaves an exception. */
|
|
56
|
+
jclass pictorFindClass(JNIEnv *e, const char *name) {
|
|
57
|
+
jclass cls = e->FindClass(name);
|
|
58
|
+
pictorClearPending(e);
|
|
59
|
+
if (cls == nullptr) {
|
|
60
|
+
__android_log_print(
|
|
61
|
+
ANDROID_LOG_ERROR, kPictorJniTag,
|
|
62
|
+
"class not found: %s — a minified build needs this package's "
|
|
63
|
+
"consumer-rules.pro (consumerProguardFiles), which keeps the names "
|
|
64
|
+
"this library resolves from native code",
|
|
65
|
+
name);
|
|
66
|
+
}
|
|
67
|
+
return cls;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** `GetStaticMethodID`, or `nullptr` with the member logged. */
|
|
71
|
+
jmethodID pictorStaticMethod(JNIEnv *e, jclass cls, const char *owner,
|
|
72
|
+
const char *name, const char *sig) {
|
|
73
|
+
if (cls == nullptr) return nullptr;
|
|
74
|
+
jmethodID id = e->GetStaticMethodID(cls, name, sig);
|
|
75
|
+
pictorClearPending(e);
|
|
76
|
+
if (id == nullptr) {
|
|
77
|
+
__android_log_print(ANDROID_LOG_ERROR, kPictorJniTag,
|
|
78
|
+
"method not found: %s.%s%s — kept the class but not "
|
|
79
|
+
"the member; check consumer-rules.pro",
|
|
80
|
+
owner, name, sig);
|
|
81
|
+
}
|
|
82
|
+
return id;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
} // namespace
|
|
25
86
|
#include <memory>
|
|
26
87
|
|
|
27
88
|
namespace jsi = facebook::jsi;
|
|
@@ -131,15 +192,29 @@ public:
|
|
|
131
192
|
GL_CLAMP_TO_EDGE);
|
|
132
193
|
const GLenum error = glGetError();
|
|
133
194
|
|
|
134
|
-
jclass cls = e
|
|
135
|
-
jmethodID create =
|
|
136
|
-
cls, "
|
|
195
|
+
jclass cls = pictorFindClass(e, "dev/pictor/video/VideoSurface");
|
|
196
|
+
jmethodID create = pictorStaticMethod(
|
|
197
|
+
e, cls, "VideoSurface", "create",
|
|
198
|
+
"(III)Landroid/graphics/SurfaceTexture;");
|
|
199
|
+
jmethodID surfaceOf = pictorStaticMethod(
|
|
200
|
+
e, cls, "VideoSurface", "surfaceOf",
|
|
201
|
+
"(Landroid/graphics/SurfaceTexture;)Landroid/view/Surface;");
|
|
202
|
+
if (cls == nullptr || create == nullptr || surfaceOf == nullptr) {
|
|
203
|
+
// No surface, and the log above says which name is missing. The
|
|
204
|
+
// host sees a preview that never starts rather than a crash it
|
|
205
|
+
// cannot attribute.
|
|
206
|
+
return jsi::Value(false);
|
|
207
|
+
}
|
|
137
208
|
jobject texture = e->CallStaticObjectMethod(
|
|
138
209
|
cls, create, (jint)self->texture_, (jint)width, (jint)height);
|
|
139
|
-
|
|
140
|
-
cls, "surfaceOf",
|
|
141
|
-
"(Landroid/graphics/SurfaceTexture;)Landroid/view/Surface;");
|
|
210
|
+
pictorClearPending(e);
|
|
142
211
|
jobject surface = e->CallStaticObjectMethod(cls, surfaceOf, texture);
|
|
212
|
+
pictorClearPending(e);
|
|
213
|
+
if (texture == nullptr || surface == nullptr) {
|
|
214
|
+
__android_log_print(ANDROID_LOG_ERROR, kPictorJniTag,
|
|
215
|
+
"VideoSurface.create/surfaceOf returned null");
|
|
216
|
+
return jsi::Value(false);
|
|
217
|
+
}
|
|
143
218
|
|
|
144
219
|
self->surfaceTexture_ = e->NewGlobalRef(texture);
|
|
145
220
|
self->surface_ = e->NewGlobalRef(surface);
|
|
@@ -487,8 +562,13 @@ public:
|
|
|
487
562
|
: e->CallStaticLongMethod(self->cls_, stamp, self->surfaceTexture_);
|
|
488
563
|
if (e->ExceptionCheck()) e->ExceptionClear();
|
|
489
564
|
|
|
490
|
-
jclass clock = e
|
|
491
|
-
jmethodID take =
|
|
565
|
+
jclass clock = pictorFindClass(e, "dev/pictor/video/FrameClock");
|
|
566
|
+
jmethodID take =
|
|
567
|
+
pictorStaticMethod(e, clock, "FrameClock", "take", "(JJ)[J");
|
|
568
|
+
if (clock == nullptr || take == nullptr) {
|
|
569
|
+
self->ptsUs_ = -1;
|
|
570
|
+
return jsi::Value(-1.0);
|
|
571
|
+
}
|
|
492
572
|
auto paired = (jlongArray)e->CallStaticObjectMethod(
|
|
493
573
|
clock, take, (jlong)consumedNs, texturePtsNs);
|
|
494
574
|
if (paired == nullptr) {
|
|
@@ -276,6 +276,15 @@ internal object AudioMixer {
|
|
|
276
276
|
val mixed = FloatArray((totalSamples * CHANNELS).toInt())
|
|
277
277
|
|
|
278
278
|
val contributed = JSONObject()
|
|
279
|
+
/*
|
|
280
|
+
Spans that turned out to have no audio at all.
|
|
281
|
+
|
|
282
|
+
Reported rather than merely tolerated: "the mix is silent because
|
|
283
|
+
the clips have no sound" and "the mix is silent because something
|
|
284
|
+
went wrong" are different, and only one of them is worth anybody's
|
|
285
|
+
time.
|
|
286
|
+
*/
|
|
287
|
+
val silentForNoTrack = mutableListOf<String>()
|
|
279
288
|
for (span in spans) {
|
|
280
289
|
if (span.silent) {
|
|
281
290
|
// An empty contribution. Nothing is generated and nothing is
|
|
@@ -283,10 +292,34 @@ internal object AudioMixer {
|
|
|
283
292
|
contributed.put(span.id, 0)
|
|
284
293
|
continue
|
|
285
294
|
}
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
295
|
+
/*
|
|
296
|
+
A source with no audio track contributes silence.
|
|
297
|
+
|
|
298
|
+
It is not a failure. A clip recorded with the microphone off,
|
|
299
|
+
a screen capture, an animation exported without sound — all
|
|
300
|
+
are ordinary things to put on a timeline, and the mix's job is
|
|
301
|
+
to place what audio exists, not to insist that every clip has
|
|
302
|
+
some. Refusing here failed the whole export with `no audio
|
|
303
|
+
track in <file>` and left the person with a video they could
|
|
304
|
+
only fix by adding sound they did not want.
|
|
305
|
+
|
|
306
|
+
Narrow on purpose: only `find-track` is treated this way.
|
|
307
|
+
Every other decode failure — a corrupt file, an unsupported
|
|
308
|
+
encoding, a decoder that never finishes — still fails the mix,
|
|
309
|
+
because those are cases where audio was expected and could not
|
|
310
|
+
be read, which is a different thing from there being none.
|
|
311
|
+
*/
|
|
312
|
+
val decoded = try {
|
|
313
|
+
profile.time("decode.total") {
|
|
314
|
+
decodeToOutputFormat(
|
|
315
|
+
context, span.uri!!, span.sourceInUs, span.sourceOutUs, span.speed,
|
|
316
|
+
)
|
|
317
|
+
}
|
|
318
|
+
} catch (e: DecodeFailure) {
|
|
319
|
+
if (e.stage != "find-track") throw e
|
|
320
|
+
contributed.put(span.id, 0)
|
|
321
|
+
silentForNoTrack.add(span.id)
|
|
322
|
+
continue
|
|
290
323
|
}
|
|
291
324
|
profile.source(
|
|
292
325
|
span.id,
|
|
@@ -376,6 +409,8 @@ internal object AudioMixer {
|
|
|
376
409
|
if (!encoded.optBoolean("ok", false)) return encoded.toString()
|
|
377
410
|
|
|
378
411
|
out.put("ok", true)
|
|
412
|
+
// Named, so a silent mix can be explained rather than guessed at.
|
|
413
|
+
.put("silentForNoTrack", org.json.JSONArray(silentForNoTrack))
|
|
379
414
|
.put("uri", output.absolutePath)
|
|
380
415
|
.put("sampleRate", SAMPLE_RATE)
|
|
381
416
|
.put("channels", CHANNELS)
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
package dev.pictor.video
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Waiting for a run to settle, with an end that a cancel can bring forward.
|
|
5
|
+
*
|
|
6
|
+
* ## What this replaces, and what it cost
|
|
7
|
+
*
|
|
8
|
+
* The wait used to be a `Boolean`: settled, or not. A cancel released the
|
|
9
|
+
* latch, so "not settled" could only mean the deadline had expired — and that
|
|
10
|
+
* reasoning was only sound while the release was guaranteed. It is not. A
|
|
11
|
+
* cancel arriving before a Transformer has published, or after its run has
|
|
12
|
+
* already been cleared, has no latch to count down; it sets the cancelled flag
|
|
13
|
+
* and returns. The waiter then sat out its whole deadline with the flag true
|
|
14
|
+
* beside it, and **logged that flag every four seconds while doing so**.
|
|
15
|
+
*
|
|
16
|
+
* Measured, six times: a wedged shard reported its stall in about seven seconds
|
|
17
|
+
* and the export settled 600 010 ms later, because 600 s was the deadline it
|
|
18
|
+
* had been given. A person watching an export that is already known to have
|
|
19
|
+
* failed waits ten minutes.
|
|
20
|
+
*
|
|
21
|
+
* So the flag is now part of the exit condition rather than a decoration on the
|
|
22
|
+
* log line, and the answer is three-valued: a deadline that genuinely expired
|
|
23
|
+
* and a run that was cancelled are different failures, and only one of them
|
|
24
|
+
* means the platform stopped answering.
|
|
25
|
+
*
|
|
26
|
+
* ## Settling outranks cancelling
|
|
27
|
+
*
|
|
28
|
+
* Checked in that order deliberately. A run that finished while the cancel was
|
|
29
|
+
* in flight has a real result, and reporting it as cancelled would throw away
|
|
30
|
+
* an export that succeeded.
|
|
31
|
+
*
|
|
32
|
+
* ## No Android, on purpose
|
|
33
|
+
*
|
|
34
|
+
* Time, waiting and the cancel flag all arrive as functions, so the orderings
|
|
35
|
+
* that caused this can be tested on the JVM instead of being restated in a
|
|
36
|
+
* comment. `RunRegistry` is factored the same way and for the same reason.
|
|
37
|
+
*/
|
|
38
|
+
internal object BoundedWait {
|
|
39
|
+
|
|
40
|
+
enum class Outcome {
|
|
41
|
+
/** The run answered: a result or a failure is waiting to be read. */
|
|
42
|
+
Settled,
|
|
43
|
+
|
|
44
|
+
/** A cancel claimed the run before it answered. */
|
|
45
|
+
Cancelled,
|
|
46
|
+
|
|
47
|
+
/** The deadline expired with no answer and no cancel. */
|
|
48
|
+
TimedOut,
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Wait in slices until the run settles, a cancel claims it, or time runs out.
|
|
53
|
+
*
|
|
54
|
+
* @param awaitSlice waits up to that many milliseconds, true if it settled.
|
|
55
|
+
* @param cancelled read between slices, and once before the first.
|
|
56
|
+
* @param onSlice called with the total waited so far, for the progress log.
|
|
57
|
+
*/
|
|
58
|
+
fun await(
|
|
59
|
+
timeoutMs: Long,
|
|
60
|
+
sliceMs: Long,
|
|
61
|
+
nowMs: () -> Long,
|
|
62
|
+
awaitSlice: (Long) -> Boolean,
|
|
63
|
+
cancelled: () -> Boolean,
|
|
64
|
+
onSlice: (Long) -> Unit,
|
|
65
|
+
): Outcome {
|
|
66
|
+
val deadlineMs = nowMs() + timeoutMs
|
|
67
|
+
/*
|
|
68
|
+
Before the first slice, not only between them.
|
|
69
|
+
|
|
70
|
+
A cancel that lands while the composition is still being built is the
|
|
71
|
+
common case — it is exactly the one with no published run to claim —
|
|
72
|
+
and waiting a whole slice to notice it would put the floor for a
|
|
73
|
+
cancelled export at the reporting interval.
|
|
74
|
+
*/
|
|
75
|
+
if (cancelled()) return Outcome.Cancelled
|
|
76
|
+
var waitedMs = 0L
|
|
77
|
+
while (true) {
|
|
78
|
+
val remainingMs = deadlineMs - nowMs()
|
|
79
|
+
if (remainingMs <= 0) return Outcome.TimedOut
|
|
80
|
+
val slice = minOf(sliceMs, remainingMs)
|
|
81
|
+
if (awaitSlice(slice)) return Outcome.Settled
|
|
82
|
+
// The run may have settled and been cancelled in the same slice;
|
|
83
|
+
// `awaitSlice` answered first, so this only runs when it did not.
|
|
84
|
+
if (cancelled()) return Outcome.Cancelled
|
|
85
|
+
waitedMs += slice
|
|
86
|
+
onSlice(waitedMs)
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
@@ -67,10 +67,25 @@ internal class CadenceEffect(
|
|
|
67
67
|
val glDeleted = AtomicLong(0)
|
|
68
68
|
val peakPending = AtomicLong(0)
|
|
69
69
|
val stalls = AtomicLong(0)
|
|
70
|
+
/**
|
|
71
|
+
* End-of-stream, in and out.
|
|
72
|
+
*
|
|
73
|
+
* A frame counter that reaches its expected total says the *effect* did its
|
|
74
|
+
* work. It says nothing about whether the stream was ever declared over,
|
|
75
|
+
* which is the signal the encoder and the muxer are actually waiting on —
|
|
76
|
+
* and mistaking one for the other is how a shard that "finished" was
|
|
77
|
+
* reported as complete while nothing downstream had finished at all.
|
|
78
|
+
*/
|
|
79
|
+
val endIn = AtomicLong(0)
|
|
80
|
+
val endOut = AtomicLong(0)
|
|
81
|
+
/** Outstanding copies the moment the stream was declared over. */
|
|
82
|
+
val pendingAtEnd = AtomicLong(-1)
|
|
70
83
|
private val problems = mutableListOf<String>()
|
|
71
84
|
|
|
72
85
|
fun trace(): String = synchronized(problems) {
|
|
73
86
|
"inputs=" + inputsSeen.get() + " emitted=" + emitted.get() +
|
|
87
|
+
" endIn=" + endIn.get() + " endOut=" + endOut.get() +
|
|
88
|
+
" pendingAtEnd=" + pendingAtEnd.get() +
|
|
74
89
|
" glCreated=" + glCreated.get() + " glDeleted=" + glDeleted.get() +
|
|
75
90
|
" peakPending=" + peakPending.get() + " stalls=" + stalls.get() +
|
|
76
91
|
(if (problems.isEmpty()) "" else " problems=" + problems.joinToString("|"))
|
|
@@ -147,6 +162,18 @@ internal class CadenceEffect(
|
|
|
147
162
|
*/
|
|
148
163
|
inputListener?.onInputFrameProcessed(input)
|
|
149
164
|
inputListener?.onReadyToAcceptInputFrame()
|
|
165
|
+
/*
|
|
166
|
+
Ask, even though nothing was emitted.
|
|
167
|
+
|
|
168
|
+
Every other path that finishes with an input ends by
|
|
169
|
+
asking whether the stream is over; this one returned
|
|
170
|
+
early. Above 1× most frames take this path, so if the
|
|
171
|
+
end-of-stream signal arrives while one is in flight, the
|
|
172
|
+
condition it was waiting for becomes true here and
|
|
173
|
+
nowhere else — and the guard that would have noticed is
|
|
174
|
+
the one being skipped.
|
|
175
|
+
*/
|
|
176
|
+
maybeSignalEnd()
|
|
150
177
|
return
|
|
151
178
|
}
|
|
152
179
|
heldInput = input
|
|
@@ -253,6 +280,7 @@ internal class CadenceEffect(
|
|
|
253
280
|
}
|
|
254
281
|
|
|
255
282
|
override fun signalEndOfCurrentInputStream() {
|
|
283
|
+
endIn.incrementAndGet()
|
|
256
284
|
endOfStreamSignalled = true
|
|
257
285
|
maybeSignalEnd()
|
|
258
286
|
}
|
|
@@ -268,6 +296,7 @@ internal class CadenceEffect(
|
|
|
268
296
|
*/
|
|
269
297
|
private fun maybeSignalEnd() {
|
|
270
298
|
if (endOfStreamSignalled && heldInput == null && owed == 0) {
|
|
299
|
+
if (endOut.getAndIncrement() == 0L) pendingAtEnd.set(pending.toLong())
|
|
271
300
|
outputListener?.onCurrentOutputStreamEnded()
|
|
272
301
|
}
|
|
273
302
|
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
package dev.pictor.video
|
|
2
|
+
|
|
3
|
+
import androidx.media3.common.Player
|
|
4
|
+
|
|
5
|
+
/** Media3 seeks in milliseconds. A duplicate must not erase its latched frame. */
|
|
6
|
+
internal fun isRedundantCompositionSeek(positionMs: Long, targetUs: Long, state: Int): Boolean =
|
|
7
|
+
(state == Player.STATE_READY || state == Player.STATE_BUFFERING) &&
|
|
8
|
+
positionMs == targetUs / 1000
|
|
@@ -145,6 +145,7 @@ internal object CompositionSpike {
|
|
|
145
145
|
val playbackState: Int,
|
|
146
146
|
val durationUs: Long,
|
|
147
147
|
val presentedUs: Long,
|
|
148
|
+
val presentedWindowIndex: Int,
|
|
148
149
|
val presentedAgeNs: Long,
|
|
149
150
|
/** How long the read itself took, which bounds the pairing error. */
|
|
150
151
|
val readSpanNs: Long,
|
|
@@ -606,6 +607,7 @@ internal object CompositionSpike {
|
|
|
606
607
|
playbackState = playbackState,
|
|
607
608
|
durationUs = duration,
|
|
608
609
|
presentedUs = frame?.get(FrameClock.Field.PTS_US) ?: -1,
|
|
610
|
+
presentedWindowIndex = frame?.get(FrameClock.Field.WINDOW_INDEX)?.toInt() ?: -1,
|
|
609
611
|
/*
|
|
610
612
|
Aged from the *scheduled release*, not from the metadata callback.
|
|
611
613
|
|
|
@@ -699,6 +701,7 @@ internal object CompositionSpike {
|
|
|
699
701
|
.put("playbackState", snapshot.playbackState)
|
|
700
702
|
.put("durationUs", snapshot.durationUs)
|
|
701
703
|
.put("presentedUs", if (snapshot.presentedUs < 0) JSONObject.NULL else snapshot.presentedUs)
|
|
704
|
+
.put("presentedWindowIndex", snapshot.presentedWindowIndex)
|
|
702
705
|
.put("presentedAgeNs", snapshot.presentedAgeNs)
|
|
703
706
|
.put("readSpanNs", snapshot.readSpanNs)
|
|
704
707
|
// How old this pairing is by the time it is read. Reported so a
|
|
@@ -761,11 +764,14 @@ internal object CompositionSpike {
|
|
|
761
764
|
nothing follows it to prepare.
|
|
762
765
|
*/
|
|
763
766
|
val seams = ArrayList<Long>(itemsJson.length())
|
|
767
|
+
val windowEndsUs = LongArray(itemsJson.length())
|
|
764
768
|
var at = 0L
|
|
765
769
|
for (i in 0 until itemsJson.length()) {
|
|
766
770
|
val item = itemsJson.getJSONObject(i)
|
|
767
771
|
val speed = item.optDouble("speed", 1.0).takeIf { it > 0 } ?: 1.0
|
|
768
|
-
at
|
|
772
|
+
at = item.optLong("timelineEndUs",
|
|
773
|
+
at + ((item.optLong("endUs") - item.optLong("startUs")) / speed).toLong())
|
|
774
|
+
windowEndsUs[i] = at
|
|
769
775
|
if (i < itemsJson.length() - 1) seams.add(at)
|
|
770
776
|
}
|
|
771
777
|
seamHoldEnabled = spec.optBoolean("seamHold", false)
|
|
@@ -804,7 +810,10 @@ internal object CompositionSpike {
|
|
|
804
810
|
// The same hook the two-player path uses, so the presented
|
|
805
811
|
// frame is measured by the machinery already trusted.
|
|
806
812
|
p.setVideoFrameMetadataListener { presentationTimeUs, releaseTimeNs, _, _ ->
|
|
807
|
-
FrameClock.
|
|
813
|
+
FrameClock.offerForWindow(
|
|
814
|
+
presentationTimeUs, releaseTimeNs,
|
|
815
|
+
compositionWindowAt(presentationTimeUs, windowEndsUs),
|
|
816
|
+
)
|
|
808
817
|
}
|
|
809
818
|
p.addListener(object : Player.Listener {
|
|
810
819
|
override fun onPlaybackStateChanged(state: Int) {
|
|
@@ -873,17 +882,8 @@ internal object CompositionSpike {
|
|
|
873
882
|
.build(),
|
|
874
883
|
)
|
|
875
884
|
.build()
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
No clip audio, ever.
|
|
879
|
-
|
|
880
|
-
The prepared mix already contains it — that is what makes it a
|
|
881
|
-
mix — and letting the clip's own track through would play the
|
|
882
|
-
whole timeline twice. This is the same rule the two-player path
|
|
883
|
-
enforces by muting the source player.
|
|
884
|
-
*/
|
|
885
|
-
.setRemoveAudio(true)
|
|
886
|
-
.setDurationUs(endUs - startUs)
|
|
885
|
+
// The helper removes clip audio: the prepared mix already contains it.
|
|
886
|
+
val builder = compositionVideoItem(media, o.optLong("sourceDurationUs", endUs))
|
|
887
887
|
if (speed != 1f) {
|
|
888
888
|
/*
|
|
889
889
|
Speed as a source mapping, not as a clock rate.
|
|
@@ -929,6 +929,16 @@ internal object CompositionSpike {
|
|
|
929
929
|
// A seek supersedes any hold: the boundary it was covering is no longer
|
|
930
930
|
// where playback is going.
|
|
931
931
|
cancelSeamHold()
|
|
932
|
+
// Scrubbing sends the last position again on release. Media3 may not
|
|
933
|
+
// emit another frame for a no-op seek; bumping FrameClock here would
|
|
934
|
+
// erase the only frame that can satisfy the following Play readiness
|
|
935
|
+
// gate. Keep pause/hold semantics, but don't invalidate that frame or
|
|
936
|
+
// the metadata for a seek to this position already in flight.
|
|
937
|
+
if (isRedundantCompositionSeek(it.currentPosition, positionUs, it.playbackState)) {
|
|
938
|
+
it.pause()
|
|
939
|
+
armRelease(positionUs)
|
|
940
|
+
return@onPlayer
|
|
941
|
+
}
|
|
932
942
|
seekStartedNs = System.nanoTime()
|
|
933
943
|
seekReadyNs = 0
|
|
934
944
|
// A new epoch: frames still in flight belong to the position being left.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
package dev.pictor.video
|
|
2
|
+
|
|
3
|
+
import androidx.media3.common.MediaItem
|
|
4
|
+
import androidx.media3.transformer.EditedMediaItem
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Media3 applies the MediaItem's trim itself. Its duration is the untrimmed
|
|
8
|
+
* source duration, NOT end - start: that would apply the trim twice and reject
|
|
9
|
+
* any end beyond that shortened duration. Speed is applied separately as well.
|
|
10
|
+
* Legacy callers without metadata supply endUs as the known source extent;
|
|
11
|
+
* with an explicit clipping end this gives the same presentation duration.
|
|
12
|
+
*/
|
|
13
|
+
internal fun compositionVideoItem(media: MediaItem, sourceDurationUs: Long): EditedMediaItem.Builder =
|
|
14
|
+
EditedMediaItem.Builder(media)
|
|
15
|
+
.setRemoveAudio(true)
|
|
16
|
+
.setDurationUs(sourceDurationUs)
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
package dev.pictor.video
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* CompositionPlayer emits project-time PTS, not playlist transition events.
|
|
5
|
+
* Tag the actual offered frame using compiled exclusive window ends, never
|
|
6
|
+
* the playhead: outgoing and incoming frames can be in flight at the seam.
|
|
7
|
+
*/
|
|
8
|
+
internal fun compositionWindowAt(presentationUs: Long, windowEndsUs: LongArray): Int {
|
|
9
|
+
if (presentationUs < 0 || windowEndsUs.isEmpty()) return -1
|
|
10
|
+
var low = 0
|
|
11
|
+
var high = windowEndsUs.size
|
|
12
|
+
while (low < high) {
|
|
13
|
+
val mid = (low + high) ushr 1
|
|
14
|
+
if (presentationUs < windowEndsUs[mid]) high = mid else low = mid + 1
|
|
15
|
+
}
|
|
16
|
+
return low.coerceAtMost(windowEndsUs.lastIndex)
|
|
17
|
+
}
|