@pictor-sdk/video-android 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +40 -0
  3. package/android/CMakeLists.txt +30 -0
  4. package/android/build.gradle +119 -0
  5. package/android/src/main/AndroidManifest.xml +1 -0
  6. package/android/src/main/cpp/pictor_overlaystage.cpp +372 -0
  7. package/android/src/main/cpp/pictor_publish.cpp +87 -0
  8. package/android/src/main/cpp/pictor_videosurface.cpp +735 -0
  9. package/android/src/main/java/dev/pictor/video/AacPriming.kt +261 -0
  10. package/android/src/main/java/dev/pictor/video/AudioMixer.kt +1112 -0
  11. package/android/src/main/java/dev/pictor/video/AudioProbe.kt +162 -0
  12. package/android/src/main/java/dev/pictor/video/CadenceShaderProgram.kt +337 -0
  13. package/android/src/main/java/dev/pictor/video/CodecLedger.kt +126 -0
  14. package/android/src/main/java/dev/pictor/video/CompositionSpike.kt +978 -0
  15. package/android/src/main/java/dev/pictor/video/Diagnostics.kt +54 -0
  16. package/android/src/main/java/dev/pictor/video/EncoderGate.kt +476 -0
  17. package/android/src/main/java/dev/pictor/video/ExportGate.kt +1974 -0
  18. package/android/src/main/java/dev/pictor/video/ExportHeartbeat.kt +72 -0
  19. package/android/src/main/java/dev/pictor/video/ExportProgress.kt +103 -0
  20. package/android/src/main/java/dev/pictor/video/FrameClock.kt +591 -0
  21. package/android/src/main/java/dev/pictor/video/FramePending.kt +181 -0
  22. package/android/src/main/java/dev/pictor/video/GateAudio.kt +456 -0
  23. package/android/src/main/java/dev/pictor/video/GateDestination.kt +445 -0
  24. package/android/src/main/java/dev/pictor/video/GateFrames.kt +662 -0
  25. package/android/src/main/java/dev/pictor/video/GateMux.kt +185 -0
  26. package/android/src/main/java/dev/pictor/video/ManagedSources.kt +333 -0
  27. package/android/src/main/java/dev/pictor/video/Media3Player.kt +934 -0
  28. package/android/src/main/java/dev/pictor/video/MediaStoreSave.kt +419 -0
  29. package/android/src/main/java/dev/pictor/video/OverlayResources.kt +169 -0
  30. package/android/src/main/java/dev/pictor/video/PictorVideoModule.kt +1612 -0
  31. package/android/src/main/java/dev/pictor/video/PictorVideoPackage.kt +26 -0
  32. package/android/src/main/java/dev/pictor/video/PlaybackAnchor.kt +161 -0
  33. package/android/src/main/java/dev/pictor/video/PreviewAudioPlayer.kt +291 -0
  34. package/android/src/main/java/dev/pictor/video/PreviewConfig.kt +96 -0
  35. package/android/src/main/java/dev/pictor/video/PreviewMix.kt +396 -0
  36. package/android/src/main/java/dev/pictor/video/PreviewSync.kt +374 -0
  37. package/android/src/main/java/dev/pictor/video/PreviewTrace.kt +478 -0
  38. package/android/src/main/java/dev/pictor/video/PrimingTrim.kt +337 -0
  39. package/android/src/main/java/dev/pictor/video/RepeatingEncoderFactory.kt +97 -0
  40. package/android/src/main/java/dev/pictor/video/SceneProgram.kt +169 -0
  41. package/android/src/main/java/dev/pictor/video/SceneProgramNative.kt +299 -0
  42. package/android/src/main/java/dev/pictor/video/SkiaPassThroughEffect.kt +695 -0
  43. package/android/src/main/java/dev/pictor/video/SurfaceAttachment.kt +110 -0
  44. package/android/src/main/java/dev/pictor/video/ThumbnailDecoder.kt +171 -0
  45. package/android/src/main/java/dev/pictor/video/VideoSurface.kt +394 -0
  46. package/app.plugin.js +43 -0
  47. package/dist/index.d.mts +3622 -0
  48. package/dist/index.d.ts +3622 -0
  49. package/dist/index.js +3737 -0
  50. package/dist/index.js.map +1 -0
  51. package/dist/index.mjs +3625 -0
  52. package/dist/index.mjs.map +1 -0
  53. package/package.json +82 -0
  54. package/plugin/gradleProperties.d.ts +19 -0
  55. package/plugin/gradleProperties.js +64 -0
  56. package/react-native.config.js +13 -0
  57. package/src/NativePictorVideo.ts +574 -0
  58. package/src/audioProbe.ts +72 -0
  59. package/src/compositionSpike.ts +305 -0
  60. package/src/convergence.ts +112 -0
  61. package/src/encoderGate.ts +66 -0
  62. package/src/exportDestination.ts +149 -0
  63. package/src/exportJob.ts +232 -0
  64. package/src/exportProgress.ts +391 -0
  65. package/src/exportProgressPump.ts +190 -0
  66. package/src/exporter.ts +655 -0
  67. package/src/index.ts +276 -0
  68. package/src/managedSources.ts +252 -0
  69. package/src/media3.ts +412 -0
  70. package/src/mediaStore.ts +177 -0
  71. package/src/memoryGuard.ts +118 -0
  72. package/src/mixSources.ts +82 -0
  73. package/src/overlayStaging.ts +216 -0
  74. package/src/playbackLifecycle.ts +137 -0
  75. package/src/player.ts +39 -0
  76. package/src/playlistSession.ts +125 -0
  77. package/src/previewAudioDiagnostics.ts +67 -0
  78. package/src/previewAudioSession.ts +168 -0
  79. package/src/previewCadence.ts +950 -0
  80. package/src/previewMix.ts +212 -0
  81. package/src/previewTrace.ts +348 -0
  82. package/src/spikeExcursions.ts +405 -0
  83. package/src/submissionScoring.ts +680 -0
  84. package/src/surfaceOwner.ts +264 -0
  85. package/src/surfaceRecovery.ts +121 -0
  86. package/src/syncTrace.ts +189 -0
  87. package/src/thumbnailProvider.ts +163 -0
  88. package/src/tracePhases.ts +217 -0
  89. package/src/useCompositionPlayback.ts +327 -0
  90. package/src/useMedia3Playback.ts +492 -0
  91. package/src/usePreviewAudio.ts +305 -0
  92. package/src/useVideoSurface.ts +282 -0
@@ -0,0 +1,1974 @@
1
+ package dev.pictor.video
2
+
3
+ import android.content.Context
4
+ import android.media.MediaExtractor
5
+ import android.media.MediaFormat
6
+ import android.os.Build
7
+ import android.os.Handler
8
+ import android.os.Looper
9
+ import androidx.media3.common.Effect
10
+ import androidx.media3.common.MediaItem
11
+ import androidx.media3.common.C
12
+ import androidx.media3.common.audio.SpeedProvider
13
+ import androidx.media3.effect.FrameDropEffect
14
+ import androidx.media3.common.util.UnstableApi
15
+ import androidx.media3.transformer.Composition
16
+ import androidx.media3.transformer.EditedMediaItem
17
+ import androidx.media3.transformer.EditedMediaItemSequence
18
+ import androidx.media3.transformer.ExportException
19
+ import androidx.media3.transformer.ExportResult
20
+ import androidx.media3.transformer.Effects
21
+ import androidx.media3.transformer.Transformer
22
+ import com.google.common.collect.ImmutableList
23
+ import java.io.File
24
+ import java.util.concurrent.CountDownLatch
25
+ import java.util.concurrent.TimeUnit
26
+ import java.util.concurrent.atomic.AtomicLong
27
+ import org.json.JSONObject
28
+
29
+ /**
30
+ * The V3.3 Transformer–Skia GL interoperability **gate**.
31
+ *
32
+ * Not an exporter. It encodes one media item to an adapter-private temporary
33
+ * file and reports what came out, so the question "can Skia draw inside
34
+ * Transformer's frame processor without changing the picture" can be answered
35
+ * with measurements instead of an opinion. Nothing here maps a `VideoProject`,
36
+ * and nothing here is reachable from `VideoExporter`.
37
+ *
38
+ * ## Two runs, in this order, always
39
+ *
40
+ * **Vanilla first.** A Transformer run with no effect establishes what the codec
41
+ * alone does to the picture — decode, re-encode and mux are lossy, and a
42
+ * comparison threshold chosen after seeing the Skia output would be a threshold
43
+ * chosen to pass. The floor is measured, frozen, and only then is the effect
44
+ * run evaluated against it.
45
+ *
46
+ * That run must be forced to **re-encode**. Left alone, Transformer transmuxes a
47
+ * source it does not need to touch — it finished in 400 ms, produced a 630 KB
48
+ * file against the effect run's 2.4 MB, and reported no colour information at
49
+ * all, because nothing was decoded. A transmux is a byte copy: its codec noise
50
+ * is zero, and a threshold frozen from it would declare every real difference a
51
+ * failure. `setTransmuxVideo(false)` is what makes the baseline a baseline.
52
+ *
53
+ * ## Audio
54
+ *
55
+ * **Audio is copied through unmodified**, by not asking Transformer to do
56
+ * anything with it. This gate is about video frames; speed, mixing and pitch are
57
+ * V3.5 and are deliberately absent. A run whose input has no audio track simply
58
+ * has none in its output, which the metadata check reports rather than assumes.
59
+ *
60
+ * ## Destination
61
+ *
62
+ * Adapter-private and temporary: `context.cacheDir`, deleted by
63
+ * [releaseTemporary]. `VideoExportSpec.destination` is a V3.6 decision and is
64
+ * not being pre-empted by a gate.
65
+ */
66
+ @UnstableApi
67
+ internal class ExportGate(private val context: Context) {
68
+
69
+ /** What one run produced. Numbers and strings only — Media3 stops here. */
70
+ data class Outcome(
71
+ val ok: Boolean,
72
+ val uri: String,
73
+ val error: String,
74
+ /** Wall time from `start` to the callback, in milliseconds. */
75
+ val elapsedMs: Long,
76
+ val fileBytes: Long,
77
+ val durationMs: Long,
78
+ val width: Int,
79
+ val height: Int,
80
+ /** Frames the encoder reported writing. */
81
+ val videoFrameCount: Int,
82
+ val hasAudio: Boolean,
83
+ val videoMimeType: String,
84
+ val audioMimeType: String,
85
+ val colorInfo: String,
86
+ /** `glGetError` values seen inside the effect, joined. Empty is the pass. */
87
+ val glErrors: String,
88
+ /** Frames the effect saw. Zero on a vanilla run. */
89
+ val effectFrames: Long,
90
+ /** Presentation timestamps in, joined — to compare against what came out. */
91
+ val effectTimestampsUs: String,
92
+ /** Where Transformer ran the effect: context, thread, and any changes. */
93
+ val boundary: String,
94
+ /** GL programs created and deleted, and on which threads. */
95
+ val lifecycle: String,
96
+ /** Segments the program selected, and any window mismatch. */
97
+ val programTrace: String,
98
+ /** What the final mux copied, against what produced it. */
99
+ val muxTrace: String = "",
100
+ /** Rasters uploaded, and textures freed, for leak accounting. */
101
+ val overlayUploads: Long,
102
+ val overlayDraws: Long,
103
+ val overlayTexturesFreed: Long,
104
+ /** Native heap allocated after the run, in bytes. */
105
+ val nativeHeapBytes: Long,
106
+ /** Java heap in use after the run, in bytes. */
107
+ val javaHeapBytes: Long,
108
+ /** The thread `run` was called on, so single-thread access is checkable. */
109
+ val callerThread: String,
110
+ )
111
+
112
+ private val main = Handler(Looper.getMainLooper())
113
+
114
+ /**
115
+ * Files this gate created, so a caller can prove it left none behind.
116
+ *
117
+ * A failed run must leave no partial file. Transformer writes to the path it
118
+ * was given as it goes, so the path is remembered before the run starts and
119
+ * deleted when the run fails.
120
+ */
121
+ private val created = mutableListOf<File>()
122
+
123
+ /*
124
+ Accounting, split three ways.
125
+
126
+ `runs == releases` on its own is ambiguous: a scene refused before
127
+ Transformer started allocated nothing, so counting it alongside a real
128
+ export makes a balanced tally that proves nothing about cleanup. What
129
+ matters is that every export that **started** also released, and refusals
130
+ are tracked separately so they cannot flatter that number.
131
+ */
132
+ /*
133
+ Cancellation, reaching the Transformer rather than only the promise.
134
+
135
+ A Transformer nobody is listening to still encodes, still writes and still
136
+ holds its codecs, so rejecting in JavaScript would leave a cancelled UI
137
+ over a running export. The handle is kept while one is in flight and
138
+ cleared the moment it finishes, and `cancelled` is checked between stages
139
+ so a cancel that arrives between two Transformer runs is not ignored until
140
+ the next one ends.
141
+ */
142
+ @Volatile private var currentJob: String? = null
143
+ @Volatile private var cancelled = false
144
+ @Volatile private var activeTransformer: Transformer? = null
145
+ /*
146
+ The run's latch, so a cancellation can end the wait.
147
+
148
+ `Transformer.cancel()` does not call the listener — traced: cancel entered
149
+ and exited on the main thread in 47 ms, and no `onCompleted` or `onError`
150
+ ever followed. The run's latch was therefore never counted down and the
151
+ export sat on `latch.await` until its timeout, which is where the seconds
152
+ came from once the cancel finally reached the Transformer at all.
153
+ */
154
+ @Volatile private var activeLatch: CountDownLatch? = null
155
+
156
+ /**
157
+ * Whether to hash every sample to prove the mux copied rather than encoded.
158
+ *
159
+ * Conformance work. Off in production, because an export should not pay to
160
+ * verify itself, and a watchdog calibrated with it present would be
161
+ * calibrated against work the user never runs.
162
+ */
163
+ /**
164
+ * Whether a conformance run has armed the gate controls *within a build that
165
+ * has them at all*.
166
+ *
167
+ * Scoping only. Whether they exist is [Diagnostics.ENABLED], decided at
168
+ * compile time — this cannot widen that, only narrow it, which is why every
169
+ * read below is the conjunction. A release build has the constant false, so
170
+ * no value here makes any difference.
171
+ */
172
+ @Volatile var armed: Boolean = false
173
+
174
+ /** Armed *and* built with the controls. Every gate-only site reads this. */
175
+ val diagnostics: Boolean
176
+ get() = Diagnostics.ENABLED && armed
177
+
178
+ @Volatile var provenance: Boolean = false
179
+
180
+ /**
181
+ * Gate-only: wedge the cadence program after N frames, on any shard.
182
+ *
183
+ * Set out of band rather than through the shard descriptor, because the
184
+ * production exporter builds those and must not gain a fault field to make
185
+ * a gate reachable. -1 is off.
186
+ */
187
+ @Volatile var stallAfterFrames: Int = -1
188
+
189
+ /**
190
+ * Gate-only: fail the publication at a chosen point.
191
+ *
192
+ * `"stage"` — the copy fails. `"afterStage"` — everything is written and
193
+ * verified and the rename does not happen. `"raceCreate"` — something else
194
+ * creates the target between validation and publication, which is the race
195
+ * the atomic no-replace exists for and which no amount of retrying produces
196
+ * on demand. `""` is off.
197
+ */
198
+ @Volatile var publishFault: String = ""
199
+
200
+ /** Whether the job in flight has been asked to stop. */
201
+ fun isCancelled(): Boolean = cancelled
202
+
203
+
204
+ /*
205
+ Cancels that arrive before the job registers.
206
+
207
+ `exportRunProjectAsync` is dispatched by the bridge, so a cancel issued
208
+ immediately after the call can reach native *first* — `cancelJob` then
209
+ finds no job with that id and drops the request on the floor. Measured:
210
+ such a cancel took 4.5 s to settle and the export ran to completion,
211
+ because only the JavaScript checkpoint after the native call ever saw it.
212
+
213
+ Remembering the id closes the window. Bounded, because a job either
214
+ begins and consumes its entry or was never going to.
215
+ */
216
+ private val pendingCancels = HashSet<String>()
217
+
218
+ /** Take the job. Returns false if another is already running. */
219
+ fun beginJob(jobId: String): Boolean {
220
+ synchronized(this) {
221
+ if (currentJob != null) return false
222
+ currentJob = jobId
223
+ cancelled = pendingCancels.remove(jobId)
224
+ resetHeartbeats()
225
+ // Ownership starts here and ends in `endJob`: beats from the job
226
+ // before this one carry an older epoch and are discarded.
227
+ ExportProgress.begin(ExportHeartbeat.attach { source -> heartbeat(source) })
228
+ return true
229
+ }
230
+ }
231
+
232
+ /** Give the job back, whatever happened to it. */
233
+ fun endJob() {
234
+ synchronized(this) {
235
+ currentJob = null
236
+ cancelled = false
237
+ activeTransformer = null
238
+ ExportHeartbeat.detach()
239
+ ExportProgress.end()
240
+ }
241
+ }
242
+
243
+ /**
244
+ * Stop `jobId`, if it is the one running.
245
+ *
246
+ * The id is checked so a cancel that arrives late cannot stop a different
247
+ * export that started in the meantime.
248
+ */
249
+ fun cancelJob(jobId: String): Boolean {
250
+ synchronized(this) {
251
+ if (currentJob != jobId) {
252
+ // Not running yet, or already finished. Remembered either way:
253
+ // an id that never begins simply leaves an unused entry.
254
+ pendingCancels.add(jobId)
255
+ return false
256
+ }
257
+ cancelled = true
258
+ }
259
+ val running = activeTransformer
260
+ if (running != null) {
261
+ // Transformer is main-thread only; cancelling from the export's own
262
+ // worker would be the same crash this gate wraps elsewhere.
263
+ main.post {
264
+ runCatching { running.cancel() }
265
+ // `cancel()` is silent, so the waiter is released here. Without
266
+ // this the run waits out its whole timeout after being stopped.
267
+ activeLatch?.countDown()
268
+ }
269
+ }
270
+ return true
271
+ }
272
+
273
+ /*
274
+ Three domains, never added together.
275
+
276
+ Reporting "started 29, completed 29, cancelled 10" as one balance was
277
+ incoherent: `started` counted Transformer runs, several per export, while
278
+ `cancelled` counted export jobs. The numbers were fine and the arithmetic
279
+ was nonsense, which is the worse kind of wrong — it reads like a
280
+ reconciliation.
281
+
282
+ Within the run domain a terminal outcome is exactly one of succeeded,
283
+ cancelled or failed, and **release is not one of them**: a run that failed
284
+ still releases, so releasing is lifecycle accounting kept on its own axis.
285
+ */
286
+ /*
287
+ Heartbeat: evidence that work continues, distinct from visible progress.
288
+
289
+ Progress is throttled to a few updates a second because that is all anyone
290
+ can read. A heartbeat is not — a muxed sample batch or a block of copied
291
+ bytes counts even when the percentage does not visibly move. Keeping them
292
+ separate is what stops a long mux from looking stalled and a 0.1x shard
293
+ from being mistaken for one.
294
+
295
+ The maximum gap per source is recorded because the watchdog's threshold has
296
+ to be derived from the longest *legitimate* silence, not from how long the
297
+ platform takes to notice a wedge.
298
+ */
299
+ private val beats = java.util.concurrent.ConcurrentHashMap<String, LongArray>()
300
+ @Volatile private var lastBeatNs = 0L
301
+
302
+
303
+ fun heartbeat(source: String) {
304
+ val now = System.nanoTime()
305
+ val previous = lastBeatNs
306
+ lastBeatNs = now
307
+ val gap = if (previous == 0L) 0L else (now - previous) / 1_000_000
308
+ // [count, maxGapMs, lastNs]
309
+ val entry = beats.getOrPut(source) { longArrayOf(0, 0, 0) }
310
+ synchronized(entry) {
311
+ entry[0] += 1
312
+ if (gap > entry[1]) entry[1] = gap
313
+ entry[2] = now
314
+ }
315
+ }
316
+
317
+ fun resetHeartbeats() {
318
+ beats.clear()
319
+ lastBeatNs = 0
320
+ }
321
+
322
+ /** Gate-only: the epoch a producer would capture right now. */
323
+ fun currentBeatEpoch(): Long = ExportHeartbeat.currentEpoch()
324
+
325
+ /**
326
+ * Gate-only: offer a beat on behalf of `atEpoch` and report whether it counted.
327
+ *
328
+ * The only way to prove from outside that a late beat from a released run is
329
+ * discarded rather than credited to whatever job happens to be current.
330
+ */
331
+ fun testBeat(source: String, atEpoch: Long): Boolean {
332
+ if (!diagnostics) return false
333
+ val before = beats[source]?.get(0) ?: 0
334
+ ExportHeartbeat.beat(source, atEpoch)
335
+ val after = beats[source]?.get(0) ?: 0
336
+ return after > before
337
+ }
338
+
339
+ /** Every beat from every source, summed. Liveness, not work. */
340
+ fun beatTotal(): Long {
341
+ var total = 0L
342
+ for (entry in beats.values) total += entry[0]
343
+ return total
344
+ }
345
+
346
+ /** Counted work, plus the beat total the watchdog keys on. */
347
+ fun progressSnapshotJson(): String = ExportProgress.snapshotJson(beatTotal())
348
+
349
+ fun heartbeatJson(): String {
350
+ val sources = JSONObject()
351
+ var maxGap = 0L
352
+ for ((name, entry) in beats) {
353
+ synchronized(entry) {
354
+ sources.put(
355
+ name,
356
+ JSONObject().put("count", entry[0]).put("maxGapMs", entry[1]),
357
+ )
358
+ if (entry[1] > maxGap) maxGap = entry[1]
359
+ }
360
+ }
361
+ val idleMs = if (lastBeatNs == 0L) -1L else (System.nanoTime() - lastBeatNs) / 1_000_000
362
+ return JSONObject()
363
+ .put("sources", sources)
364
+ .put("maxGapMs", maxGap)
365
+ .put("idleMs", idleMs)
366
+ .toString()
367
+ }
368
+
369
+ private val refusals = AtomicLong(0)
370
+ private val runsStarted = AtomicLong(0)
371
+ private val runsSucceeded = AtomicLong(0)
372
+ private val runsCancelled = AtomicLong(0)
373
+ private val runsFailed = AtomicLong(0)
374
+ private val runsReleased = AtomicLong(0)
375
+ private val tempFilesCreated = AtomicLong(0)
376
+ private val tempFilesReleased = AtomicLong(0)
377
+
378
+ /**
379
+ * Delete a tracked temporary, and say so.
380
+ *
381
+ * The single accounted act. Deleting a file and removing it from `created`
382
+ * without incrementing the counter is what made the ledger disagree with the
383
+ * disk, so the three steps do not exist separately.
384
+ */
385
+ private fun releaseTemp(file: File) {
386
+ runCatching { file.delete() }
387
+ synchronized(created) { if (created.remove(file)) tempFilesReleased.incrementAndGet() }
388
+ }
389
+ /** Every run's parent job, so a run is always attributable to an export. */
390
+ private val runJobs = java.util.Collections.synchronizedList(mutableListOf<String>())
391
+
392
+ /** Start a run, recorded against the job that owns it. */
393
+ private fun beginRun() {
394
+ runsStarted.incrementAndGet()
395
+ runJobs.add(currentJob ?: "")
396
+ }
397
+
398
+ /**
399
+ * End a run with exactly one terminal outcome, then release it.
400
+ *
401
+ * `wasCancelled` is checked before `ok` because a cancelled run is not a
402
+ * failure: reporting it as one would make every cancellation look like a
403
+ * defect in the export.
404
+ */
405
+ private fun finishRun(ok: Boolean, wasCancelled: Boolean) {
406
+ when {
407
+ wasCancelled -> runsCancelled.incrementAndGet()
408
+ ok -> runsSucceeded.incrementAndGet()
409
+ else -> runsFailed.incrementAndGet()
410
+ }
411
+ runsReleased.incrementAndGet()
412
+ }
413
+
414
+ fun accountingJson(): String {
415
+ val started = runsStarted.get()
416
+ val succeeded = runsSucceeded.get()
417
+ val cancelled = runsCancelled.get()
418
+ val failed = runsFailed.get()
419
+ val released = runsReleased.get()
420
+ val runs = JSONObject()
421
+ .put("started", started)
422
+ .put("succeeded", succeeded)
423
+ .put("cancelled", cancelled)
424
+ .put("failed", failed)
425
+ .put("active", started - succeeded - cancelled - failed)
426
+ .put("released", released)
427
+ .put("currentlyAllocated", started - released)
428
+ .put("attributed", runJobs.size)
429
+ .put("unattributed", runJobs.count { it.isEmpty() })
430
+ /*
431
+ `live` is counted, not derived.
432
+
433
+ It used to be `created - released`, which makes the invariant true by
434
+ construction and therefore worthless: every delete site that forgot to
435
+ increment the counter inflated `live` by one, and the ledger reported
436
+ 36 files still allocated while the cache directory held none. Reading
437
+ the set the files are actually tracked in means a missed increment
438
+ shows up as `created != released + live` instead of hiding inside it.
439
+ */
440
+ val live = synchronized(created) { created.size.toLong() }
441
+ val temp = JSONObject()
442
+ .put("created", tempFilesCreated.get())
443
+ .put("released", tempFilesReleased.get())
444
+ .put("live", live)
445
+ .put("reconciles", tempFilesCreated.get() == tempFilesReleased.get() + live)
446
+ return JSONObject()
447
+ .put("runs", runs)
448
+ .put("resources", JSONObject().put("tempFiles", temp))
449
+ .put("refusals", refusals.get())
450
+ .toString()
451
+ }
452
+
453
+
454
+ fun refusalCount(): Long = refusals.get()
455
+ fun startedCount(): Long = runsStarted.get()
456
+ fun releaseCount(): Long = runsReleased.get()
457
+ fun completedCount(): Long = runsSucceeded.get()
458
+ fun failureCount(): Long = runsFailed.get()
459
+ fun cancellationCount(): Long = runsCancelled.get()
460
+
461
+ /**
462
+ * Encode `uri` once.
463
+ *
464
+ * `withSkia` selects the effect chain: empty for the vanilla baseline, and
465
+ * the pass-through Skia effect for the gate itself. Blocking, with a
466
+ * timeout, because a gate that hangs is a gate that reports nothing.
467
+ */
468
+ /**
469
+ * Run a **validated scene** through the same Transformer path.
470
+ *
471
+ * Everything is checked before Transformer starts, so a rejected scene costs
472
+ * no encoder, no GL thread and no partial file.
473
+ */
474
+ fun runScene(
475
+ uri: String,
476
+ sceneJson: String,
477
+ timeoutSeconds: Long = 120,
478
+ failAfterFrames: Int = -1,
479
+ ): Outcome {
480
+ val (program, invalid) = SceneProgram.parse(sceneJson)
481
+ if (program == null) {
482
+ // A refusal, not a started export: nothing was allocated, so it must
483
+ // not appear in the started/released balance at all.
484
+ refusals.incrementAndGet()
485
+ return failed("scene rejected: ${invalid?.reason ?: "unknown"}", 0, Thread.currentThread().name)
486
+ }
487
+ return run(uri, true, timeoutSeconds, program, failAfterFrames)
488
+ }
489
+
490
+ /**
491
+ * Run a **compiled program** over a sequence of sources.
492
+ *
493
+ * The two-source case is the one that matters: window identity changes at
494
+ * the seam, and the program's segments must agree with what Media3 plays
495
+ * without native knowing where the seam is.
496
+ */
497
+ fun runProgram(uris: List<String>, programJson: String, timeoutSeconds: Long = 180): Outcome {
498
+ val (compiled, invalid) = SceneProgramNative.parse(programJson)
499
+ if (compiled == null) {
500
+ refusals.incrementAndGet()
501
+ return failed("program rejected: " + (invalid?.reason ?: "unknown"), 0, Thread.currentThread().name)
502
+ }
503
+ return runSequence(uris, timeoutSeconds, compiled)
504
+ }
505
+
506
+ private fun failed(error: String, elapsedMs: Long, callerThread: String) = Outcome(
507
+ ok = false,
508
+ uri = "",
509
+ error = error,
510
+ elapsedMs = elapsedMs,
511
+ fileBytes = 0,
512
+ durationMs = 0,
513
+ width = 0,
514
+ height = 0,
515
+ videoFrameCount = 0,
516
+ hasAudio = false,
517
+ videoMimeType = "",
518
+ audioMimeType = "",
519
+ colorInfo = "",
520
+ glErrors = "",
521
+ effectFrames = 0,
522
+ effectTimestampsUs = "",
523
+ boundary = "",
524
+ lifecycle = "",
525
+ programTrace = "",
526
+ overlayUploads = 0,
527
+ overlayDraws = 0,
528
+ overlayTexturesFreed = 0,
529
+ nativeHeapBytes = android.os.Debug.getNativeHeapAllocatedSize(),
530
+ javaHeapBytes = Runtime.getRuntime().let { it.totalMemory() - it.freeMemory() },
531
+ callerThread = callerThread,
532
+ )
533
+
534
+ fun run(
535
+ uri: String,
536
+ withSkia: Boolean,
537
+ timeoutSeconds: Long = 120,
538
+ program: SceneProgram? = null,
539
+ failAfterFrames: Int = -1,
540
+ ): Outcome = runSequence(listOf(uri), timeoutSeconds, null, withSkia, program, failAfterFrames)
541
+
542
+ /**
543
+ * One source in a sequence, with the clipping and speed **TypeScript chose**.
544
+ *
545
+ * Native performs none of this arithmetic: it receives a start, an end and a
546
+ * rate, and hands them to Media3. Deciding which part of a source plays, and
547
+ * how fast, is timeline interpretation and lives in the compiler.
548
+ */
549
+ class Item(
550
+ val uri: String,
551
+ val startMs: Long,
552
+ val endMs: Long,
553
+ val speed: Float,
554
+ /** Drop this item's own audio; the sequence supplies silence. */
555
+ val removeAudio: Boolean = false,
556
+ /**
557
+ * How a rate change treats audio.
558
+ *
559
+ * `"resample"` is `setSpeed`, which re-times the samples — the tape
560
+ * effect, where half speed is an octave down. `"preserve"` uses the
561
+ * coordinated Sonic processor and timestamp adjustment, which stretches
562
+ * time and leaves pitch where it was.
563
+ */
564
+ val speedMode: String = "preserve",
565
+ /** Source-time length to declare, or 0 to declare nothing. */
566
+ val durationUs: Long = 0,
567
+ /** Microsecond clip end, for spans a millisecond cannot express. */
568
+ val endUs: Long = 0,
569
+ )
570
+
571
+ /**
572
+ * One clip, exported alone, then concatenated at 1×.
573
+ *
574
+ * Per-item speed inside a sequence is broken: only the first item's rate is
575
+ * honoured, and a later item at any other rate has its video truncated or
576
+ * dropped while its duration and audio still look right. Sharding puts every
577
+ * re-timed clip in the position where `setSpeed` is proven exact — first and
578
+ * only — and leaves the concatenation nothing to re-time.
579
+ *
580
+ * The clip's scene-program segments run in the **same** encoding pass, so a
581
+ * frame is encoded once. Encoding for speed and again for visuals would put
582
+ * two generations of loss into every exported clip.
583
+ */
584
+ class Shard(
585
+ val uri: String,
586
+ val startMs: Long,
587
+ val endMs: Long,
588
+ val speed: Float,
589
+ /** Compiled in TypeScript. Native adds it to a local timestamp; nothing more. */
590
+ val timelineOffsetUs: Long,
591
+ val forceAudioTrack: Boolean,
592
+ /** Thin to this rate before the scene interpreter, or 0. */
593
+ val frameDropFps: Int = 0,
594
+ /** Encoder repeat interval, or 0. */
595
+ val encoderRepeatUs: Long = 0,
596
+ /** Copies each input frame owes, compiled by `resampleClip`. */
597
+ val emitsPerInput: IntArray? = null,
598
+ /** Output timestamps, flattened in emission order. */
599
+ val outputPtsUs: LongArray? = null,
600
+ val cadenceFailAfterEmits: Int = -1,
601
+ /**
602
+ * The shard's compiled span, used to clip it during concatenation.
603
+ *
604
+ * An intermediate's container runs longer than its video: the audio tail
605
+ * is quantised to AAC packets and the muxer's duration follows it. The
606
+ * concatenation lays items out by container duration, so without this the
607
+ * next shard starts late and the seam lands off the grid — 76 034 µs
608
+ * where 33 333 was compiled. Clipping to the span TypeScript planned
609
+ * removes the tail rather than compensating for it.
610
+ */
611
+ val spanUs: Long = 0,
612
+ /**
613
+ * Drop this clip's own audio.
614
+ *
615
+ * Not "produce no audio track": the sequence still forces one, so a muted
616
+ * clip contributes digital silence in a track structurally identical to
617
+ * its neighbours'. A shard with no audio track at all cannot be
618
+ * concatenated with one that has it by copying samples.
619
+ */
620
+ val removeAudio: Boolean = false,
621
+ /** `"preserve"` (Sonic time-stretch) or `"resample"` (tape effect). */
622
+ val speedMode: String = "preserve",
623
+ /** Fail during overlay drawing, after N draws. */
624
+ val overlayFailAfterDraws: Int = -1,
625
+ /**
626
+ * Destroy an intermediate before the concatenation reads it.
627
+ *
628
+ * The one failure mode with no other way in: shards have all succeeded,
629
+ * the encoders are gone, and the remaining work is a transmux. Proving
630
+ * that path cleans up needs it to actually fail, not to be skipped.
631
+ */
632
+ val failConcat: Boolean = false,
633
+ /**
634
+ * Gate-only: stop emitting, and stop beating, after N frames.
635
+ *
636
+ * A wedge, not a failure. Every other fault control here raises
637
+ * something; this one produces the condition the watchdog exists for and
638
+ * that nothing else can imitate — a pipeline that is neither making
639
+ * progress nor reporting an error. Media3's own stall detection finds it
640
+ * eventually, at 10.5 s, which is precisely why a shorter watchdog is
641
+ * worth having.
642
+ */
643
+ val cadenceStallAfterEmits: Int = -1,
644
+ )
645
+
646
+ /**
647
+ * A constant rate for the whole item.
648
+ *
649
+ * `C.TIME_UNSET` for the next change means "never", which is what makes this
650
+ * constant. Media3 asks; it is not told, and no duration is computed here.
651
+ */
652
+ private class ConstantSpeed(private val speed: Float) : SpeedProvider {
653
+ override fun getSpeed(timeUs: Long): Float = speed
654
+ override fun getNextSpeedChangeTimeUs(timeUs: Long): Long = C.TIME_UNSET
655
+ }
656
+
657
+ fun runItems(
658
+ items: List<Item>,
659
+ programJson: String,
660
+ timeoutSeconds: Long = 180,
661
+ forceAudioTrack: Boolean = false,
662
+ ): Outcome {
663
+ val (compiled, invalid) = SceneProgramNative.parse(programJson)
664
+ if (compiled == null) {
665
+ refusals.incrementAndGet()
666
+ return failed("program rejected: " + (invalid?.reason ?: "unknown"), 0, Thread.currentThread().name)
667
+ }
668
+ return runSequence(
669
+ items.map { it.uri }, timeoutSeconds, compiled,
670
+ items = items, forceAudioTrack = forceAudioTrack,
671
+ )
672
+ }
673
+
674
+ /**
675
+ * The first shard whose encoded format cannot be concatenated with shard 0.
676
+ *
677
+ * Compares the fields a sample copy silently reinterprets. `null` when every
678
+ * part agrees and the copy is safe.
679
+ */
680
+ private fun firstFormatMismatch(files: List<java.io.File>, shards: List<Shard>): String? {
681
+ if (files.size < 2) return null
682
+ val head = videoFormatOf(files[0]) ?: return "clip 1 produced no video track"
683
+ for (i in 1 until files.size) {
684
+ val f = videoFormatOf(files[i]) ?: return "clip " + (i + 1) + " produced no video track"
685
+ val name = "clip " + (i + 1)
686
+ fun i32(fmt: android.media.MediaFormat, key: String) =
687
+ if (fmt.containsKey(key)) fmt.getInteger(key) else -1
688
+ val keys = listOf(
689
+ android.media.MediaFormat.KEY_WIDTH to "width",
690
+ android.media.MediaFormat.KEY_HEIGHT to "height",
691
+ android.media.MediaFormat.KEY_ROTATION to "rotation",
692
+ android.media.MediaFormat.KEY_PROFILE to "profile",
693
+ android.media.MediaFormat.KEY_LEVEL to "level",
694
+ android.media.MediaFormat.KEY_COLOR_RANGE to "colour range",
695
+ android.media.MediaFormat.KEY_COLOR_STANDARD to "colour standard",
696
+ android.media.MediaFormat.KEY_COLOR_TRANSFER to "colour transfer",
697
+ )
698
+ for ((key, label) in keys) {
699
+ val a = i32(head, key)
700
+ val b = i32(f, key)
701
+ if (a != b) {
702
+ return name + " encoded a different " + label + " (" + b + ") than clip 1 (" + a + ")"
703
+ }
704
+ }
705
+ val am = head.getString(android.media.MediaFormat.KEY_MIME)
706
+ val bm = f.getString(android.media.MediaFormat.KEY_MIME)
707
+ if (am != bm) return name + " encoded " + bm + " where clip 1 encoded " + am
708
+ for (key in listOf("csd-0", "csd-1")) {
709
+ if (!sameCsd(head, f, key)) {
710
+ return name + " has codec-specific data (" + key +
711
+ ") a decoder configured for clip 1 cannot use"
712
+ }
713
+ }
714
+ if (i < shards.size) Unit
715
+ }
716
+ return null
717
+ }
718
+
719
+ /** Byte-identical codec-specific data, or neither side has it. */
720
+ private fun sameCsd(a: android.media.MediaFormat, b: android.media.MediaFormat, key: String): Boolean {
721
+ val ba = runCatching { a.getByteBuffer(key) }.getOrNull()
722
+ val bb = runCatching { b.getByteBuffer(key) }.getOrNull()
723
+ if (ba == null && bb == null) return true
724
+ if (ba == null || bb == null) return false
725
+ val xa = ByteArray(ba.remaining()); ba.duplicate().get(xa)
726
+ val xb = ByteArray(bb.remaining()); bb.duplicate().get(xb)
727
+ return xa.contentEquals(xb)
728
+ }
729
+
730
+ /** The video track's format, or `null`. */
731
+ private fun videoFormatOf(file: java.io.File): android.media.MediaFormat? {
732
+ val extractor = android.media.MediaExtractor()
733
+ return try {
734
+ extractor.setDataSource(file.absolutePath)
735
+ var found: android.media.MediaFormat? = null
736
+ for (t in 0 until extractor.trackCount) {
737
+ val f = extractor.getTrackFormat(t)
738
+ val mime = f.getString(android.media.MediaFormat.KEY_MIME) ?: continue
739
+ if (mime.startsWith("video/")) { found = f; break }
740
+ }
741
+ found
742
+ } catch (e: Exception) {
743
+ null
744
+ } finally {
745
+ runCatching { extractor.release() }
746
+ }
747
+ }
748
+
749
+ /**
750
+ * One video track's format, as the muxer would see it.
751
+ *
752
+ * Diagnostics: the fields a sample copy silently reinterprets — size,
753
+ * rotation, MIME, profile, level, colour and the codec-specific data every
754
+ * decoder configures itself from.
755
+ */
756
+ private fun describeVideoFormat(file: java.io.File): String {
757
+ val extractor = android.media.MediaExtractor()
758
+ return try {
759
+ extractor.setDataSource(file.absolutePath)
760
+ var out = "no video track"
761
+ for (t in 0 until extractor.trackCount) {
762
+ val f = extractor.getTrackFormat(t)
763
+ val mime = f.getString(android.media.MediaFormat.KEY_MIME) ?: continue
764
+ if (!mime.startsWith("video/")) continue
765
+ fun i(key: String) = if (f.containsKey(key)) f.getInteger(key).toString() else "-"
766
+ fun csd(key: String): String {
767
+ val b = runCatching { f.getByteBuffer(key) }.getOrNull() ?: return "-"
768
+ val a = ByteArray(b.remaining())
769
+ b.duplicate().get(a)
770
+ var h = 0
771
+ for (v in a) h = h * 31 + v
772
+ return a.size.toString() + ":" + Integer.toHexString(h)
773
+ }
774
+ out = "mime=" + mime +
775
+ " " + i(android.media.MediaFormat.KEY_WIDTH) + "x" +
776
+ i(android.media.MediaFormat.KEY_HEIGHT) +
777
+ " rot=" + i(android.media.MediaFormat.KEY_ROTATION) +
778
+ " fps=" + i(android.media.MediaFormat.KEY_FRAME_RATE) +
779
+ " profile=" + i(android.media.MediaFormat.KEY_PROFILE) +
780
+ " level=" + i(android.media.MediaFormat.KEY_LEVEL) +
781
+ " range=" + i(android.media.MediaFormat.KEY_COLOR_RANGE) +
782
+ " standard=" + i(android.media.MediaFormat.KEY_COLOR_STANDARD) +
783
+ " transfer=" + i(android.media.MediaFormat.KEY_COLOR_TRANSFER) +
784
+ " csd0=" + csd("csd-0") + " csd1=" + csd("csd-1")
785
+ break
786
+ }
787
+ out
788
+ } catch (e: Exception) {
789
+ "unreadable: " + e.message
790
+ } finally {
791
+ runCatching { extractor.release() }
792
+ }
793
+ }
794
+
795
+ /**
796
+ * Export each clip alone, then concatenate the intermediates by transmux.
797
+ *
798
+ * Intermediates are deleted on every path — success, failure, refusal — in a
799
+ * `finally`. They are full-size encoded video in a cache directory the user
800
+ * never asked to fill, and a gate that leaves them behind on the one run
801
+ * that crashed is the gate that fills a phone.
802
+ */
803
+ fun runShards(
804
+ shards: List<Shard>,
805
+ programJson: String,
806
+ timeoutSeconds: Long = 240,
807
+ overlaysJson: String = "[]",
808
+ ): Outcome {
809
+ val (compiled, invalid) = SceneProgramNative.parse(programJson)
810
+ if (compiled == null) {
811
+ refusals.incrementAndGet()
812
+ return failed("program rejected: " + (invalid?.reason ?: "unknown"), 0, Thread.currentThread().name)
813
+ }
814
+ /*
815
+ Overlays are loaded and checked before anything is encoded.
816
+
817
+ A resource discovered missing mid-run leaves a partial file and an
818
+ encoder to unwind, and the frames already written are silently missing
819
+ something the timeline said was on screen.
820
+ */
821
+ val (resources, badResource) = OverlayResources.parse(context, overlaysJson)
822
+ if (resources == null) {
823
+ refusals.incrementAndGet()
824
+ return failed(
825
+ "overlay resources rejected: " + (badResource?.reason ?: "unknown"),
826
+ 0, Thread.currentThread().name,
827
+ )
828
+ }
829
+ for (id in compiled.requiredResourceIds()) {
830
+ if (!resources.entries.containsKey(id)) {
831
+ refusals.incrementAndGet()
832
+ return failed(
833
+ "overlay resource '" + id + "' is referenced by the program but not supplied",
834
+ 0, Thread.currentThread().name,
835
+ )
836
+ }
837
+ }
838
+ if (shards.isEmpty()) {
839
+ refusals.incrementAndGet()
840
+ return failed("no shards", 0, Thread.currentThread().name)
841
+ }
842
+
843
+ val intermediates = mutableListOf<File>()
844
+ val perShard = org.json.JSONArray()
845
+ try {
846
+ for ((shardIndex, shard) in shards.withIndex()) {
847
+ if (cancelled) {
848
+ return failed("cancelled during betweenShards", 0, Thread.currentThread().name)
849
+ }
850
+ // Shards run one at a time, so the shader program does not have
851
+ // to know which one it is drawing for — Media3 constructs it and
852
+ // tells it nothing about the timeline.
853
+ ExportProgress.enterShard(shardIndex, ExportHeartbeat.currentEpoch())
854
+ val cadenceEffect = shard.emitsPerInput?.let { per ->
855
+ shard.outputPtsUs?.let { pts ->
856
+ CadenceEffect(
857
+ per,
858
+ pts,
859
+ // Inert without diagnostics, whatever the payload
860
+ // says. A production build must not be one malformed
861
+ // JSON field away from a failed user export.
862
+ failAfterEmits = if (diagnostics) shard.cadenceFailAfterEmits else -1,
863
+ stallAfterEmits = when {
864
+ !diagnostics -> -1
865
+ stallAfterFrames >= 0 -> stallAfterFrames
866
+ else -> shard.cadenceStallAfterEmits
867
+ },
868
+ )
869
+ }
870
+ }
871
+ val outcome = runSequence(
872
+ listOf(shard.uri),
873
+ timeoutSeconds,
874
+ compiled,
875
+ items = listOf(
876
+ Item(
877
+ shard.uri, shard.startMs, shard.endMs, shard.speed,
878
+ shard.removeAudio, shard.speedMode,
879
+ ),
880
+ ),
881
+ forceAudioTrack = shard.forceAudioTrack,
882
+ timelineOffsetUs = shard.timelineOffsetUs,
883
+ frameDropFps = shard.frameDropFps,
884
+ encoderRepeatUs = shard.encoderRepeatUs,
885
+ cadence = cadenceEffect,
886
+ overlays = resources,
887
+ overlayFailAfterDraws = if (diagnostics) shard.overlayFailAfterDraws else -1,
888
+ // Shards must be concatenable by copying, and a mono track
889
+ // beside a stereo one is not.
890
+ normaliseAudioChannels = 2,
891
+ )
892
+ val cadenceTrace = cadenceEffect?.trace() ?: ""
893
+ perShard.put(
894
+ org.json.JSONObject()
895
+ .put("ok", outcome.ok)
896
+ .put("error", outcome.error)
897
+ .put("videoFrameCount", outcome.videoFrameCount)
898
+ .put("durationMs", outcome.durationMs)
899
+ .put("hasAudio", outcome.hasAudio)
900
+ .put("videoMimeType", outcome.videoMimeType)
901
+ .put("width", outcome.width)
902
+ .put("height", outcome.height)
903
+ .put("colorInfo", outcome.colorInfo)
904
+ .put("programTrace", outcome.programTrace)
905
+ // Scene executions, to compare against output frames.
906
+ .put("effectFrames", outcome.effectFrames)
907
+ .put("cadenceTrace", cadenceTrace)
908
+ .put("overlayUploads", outcome.overlayUploads)
909
+ .put("overlayDraws", outcome.overlayDraws)
910
+ .put("overlayTexturesFreed", outcome.overlayTexturesFreed)
911
+ .put("timelineOffsetUs", shard.timelineOffsetUs)
912
+ // Gate-only, like the ones at the mux: reading every
913
+ // sample of every intermediate is conformance work, and
914
+ // a production export must not pay to verify itself.
915
+ .put(
916
+ "sampleHashes",
917
+ if (diagnostics && provenance) {
918
+ GateFrames.sampleHashes(context, outcome.uri)
919
+ } else {
920
+ "{}"
921
+ },
922
+ ),
923
+ )
924
+ if (!outcome.ok) {
925
+ return outcome.copy(
926
+ error = "shard failed: " + outcome.error,
927
+ programTrace = perShard.toString(),
928
+ )
929
+ }
930
+ intermediates.add(File(outcome.uri))
931
+ }
932
+
933
+ if (diagnostics && shards.any { it.failConcat } && intermediates.isNotEmpty()) {
934
+ // A real failure, not a synthetic one: the concatenation is left
935
+ // to discover a source it cannot read, so the same cleanup runs.
936
+ runCatching { intermediates[0].delete() }
937
+ }
938
+ /*
939
+ What each shard actually encoded, before anything copies it.
940
+
941
+ Sample-copy concatenation writes every part's samples under the
942
+ *first* part's track format, so a shard that encoded a different
943
+ size, profile or codec-specific data produces a file whose
944
+ declared format is a lie about most of its payload. That is a
945
+ container correctness problem, not a cosmetic one, and it is
946
+ invisible from the finished file — which reports one format for
947
+ everything inside it.
948
+ */
949
+ for ((i, file) in intermediates.withIndex()) {
950
+ android.util.Log.i("PictorShardFmt", "shard=" + i + " " + describeVideoFormat(file))
951
+ }
952
+ /*
953
+ Refuse rather than copy samples under another shard's format.
954
+
955
+ Concatenation moves encoded bytes and writes them under the first
956
+ part's track metadata. If a later part encoded a different size,
957
+ profile, level or codec-specific data, the result declares one
958
+ format and contains several — and no decoder can be right about
959
+ all of them. The failure is silent in the file and shows up as
960
+ corrupt or missing picture, which is how it was found.
961
+
962
+ Named by clip, because "the export failed" is not something anyone
963
+ can act on and "clip 3 encoded 960x540 where the first encoded
964
+ 1920x1080" is.
965
+ */
966
+ val mismatch = firstFormatMismatch(intermediates, shards)
967
+ if (mismatch != null) {
968
+ return failed(mismatch, 0, Thread.currentThread().name)
969
+ }
970
+ // Concatenate. No effects, no speed, and transmux: the parts are
971
+ // already normalised, so the final pass must move bytes rather than
972
+ // decode and re-encode them.
973
+ val concat = runSequence(
974
+ intermediates.map { it.absolutePath },
975
+ timeoutSeconds,
976
+ compiled = null,
977
+ withSkia = false,
978
+ transmuxVideo = true,
979
+ items = shards.mapIndexed { i, shard ->
980
+ Item(
981
+ intermediates[i].absolutePath,
982
+ 0,
983
+ 0,
984
+ 1f,
985
+ durationUs = 0,
986
+ endUs = shard.spanUs,
987
+ )
988
+ },
989
+ )
990
+ return concat.copy(programTrace = perShard.toString())
991
+ } finally {
992
+ synchronized(created) {
993
+ for (file in intermediates) releaseTemp(file)
994
+ }
995
+ }
996
+ }
997
+
998
+ /**
999
+ * Encode a still image, through the same encoder the treatment uses.
1000
+ *
1001
+ * The reference is drawn by RNSkia and the treatment by GL, but a comparison
1002
+ * between them is only about the renderers if everything after them is the
1003
+ * same. So the reference is encoded and decoded too: both sides carry
1004
+ * identical codec loss, and what is left is the difference in drawing.
1005
+ */
1006
+ fun runStill(pngBytes: ByteArray, frames: Int, timeoutSeconds: Long = 120): Outcome {
1007
+ val png = File.createTempFile(TEMP_PREFIX + "still-", ".png", context.cacheDir)
1008
+ synchronized(created) { tempFilesCreated.incrementAndGet(); created.add(png) }
1009
+ png.outputStream().use { it.write(pngBytes) }
1010
+ val durationUs = (frames.toLong() * 1_000_000L) / 30L
1011
+ return runSequence(
1012
+ listOf(png.absolutePath),
1013
+ timeoutSeconds,
1014
+ compiled = null,
1015
+ withSkia = false,
1016
+ stillDurationUs = durationUs,
1017
+ )
1018
+ }
1019
+
1020
+ /**
1021
+ * Video shards, audio timeline, then a muxer-level copy of both.
1022
+ *
1023
+ * The two pipelines never meet until the last step, and that step decodes
1024
+ * nothing. Video keeps the one-generation encode and sample-copy
1025
+ * concatenation it already proved; audio is encoded exactly once from the
1026
+ * original sources; `MediaMuxer` writes both sets of encoded samples
1027
+ * unchanged.
1028
+ */
1029
+ /**
1030
+ * The priming the finished audio track carries, for its own configuration.
1031
+ *
1032
+ * Read from the track that will actually be muxed rather than from the
1033
+ * settings we asked for, because the encoder is free to have chosen
1034
+ * otherwise, and a delay measured for the wrong configuration is a
1035
+ * confident number about a different file.
1036
+ */
1037
+ private fun primingFramesFor(context: Context, audioUri: String): Int {
1038
+ val extractor = MediaExtractor()
1039
+ try {
1040
+ if (audioUri.startsWith("/")) extractor.setDataSource(audioUri)
1041
+ else extractor.setDataSource(context, android.net.Uri.parse(audioUri), null)
1042
+ for (i in 0 until extractor.trackCount) {
1043
+ val f = extractor.getTrackFormat(i)
1044
+ if (!(f.getString(MediaFormat.KEY_MIME) ?: "").startsWith("audio/")) continue
1045
+ val rate = f.getInteger(MediaFormat.KEY_SAMPLE_RATE)
1046
+ val channels = f.getInteger(MediaFormat.KEY_CHANNEL_COUNT)
1047
+ val bitrate =
1048
+ if (f.containsKey(MediaFormat.KEY_BIT_RATE)) f.getInteger(MediaFormat.KEY_BIT_RATE)
1049
+ else 128_000
1050
+ return AacPriming.measure(rate, channels, bitrate).delayFrames
1051
+ }
1052
+ } finally {
1053
+ runCatching { extractor.release() }
1054
+ }
1055
+ return 0
1056
+ }
1057
+
1058
+ fun runProject(
1059
+ shards: List<Shard>,
1060
+ programJson: String,
1061
+ overlaysJson: String,
1062
+ mixProgramJson: String,
1063
+ mixUrisJson: String,
1064
+ timeoutSeconds: Long = 600,
1065
+ destinationJson: String = "",
1066
+ ): Outcome {
1067
+ /*
1068
+ Refused before anything is encoded, where that is knowable.
1069
+
1070
+ Not the guarantee — a directory can be unmounted and a target created
1071
+ while the export runs, and the publication below is what actually has
1072
+ to be safe — but the difference between failing now and failing after
1073
+ eight seconds of work the caller cannot use.
1074
+ */
1075
+ val target = if (destinationJson.isEmpty()) null else GateDestination.parse(destinationJson)
1076
+ if (target != null) {
1077
+ GateDestination.validate(target)?.let {
1078
+ refusals.incrementAndGet()
1079
+ return failed(it, 0, Thread.currentThread().name)
1080
+ }
1081
+ }
1082
+ var videoUri: String? = null
1083
+ var audioUri: String? = null
1084
+ /*
1085
+ Checked between stages, not only inside one.
1086
+
1087
+ Each stage is a Transformer run or a decode loop of its own, and a
1088
+ cancel arriving in the gap between two of them would otherwise wait
1089
+ for the whole of the next one before anything noticed.
1090
+ */
1091
+ fun stopped(stage: String): Outcome? {
1092
+ // No run counter here: this is the *job* stopping between runs, and
1093
+ // there is no Transformer run to give a terminal outcome to.
1094
+ return if (cancelled) {
1095
+ failed("cancelled during $stage", 0, Thread.currentThread().name)
1096
+ } else {
1097
+ null
1098
+ }
1099
+ }
1100
+ try {
1101
+ stopped("resolving")?.let { return it }
1102
+ val video = runShards(shards, programJson, timeoutSeconds, overlaysJson)
1103
+ if (!video.ok) return video
1104
+ stopped("shards")?.let { return it }
1105
+ videoUri = video.uri
1106
+ /*
1107
+ One mix, one encode.
1108
+
1109
+ The timeline's audio is decoded, normalised, placed, summed and
1110
+ saturated in `AudioMixer`, then encoded to AAC exactly once — so
1111
+ the mux has nothing left to re-encode. Driven only by the compiled
1112
+ plan: no audio kinds, no duration policy, no project.
1113
+ */
1114
+ stopped("audio")?.let { return it }
1115
+ val mixFile = File.createTempFile(TEMP_PREFIX + "mix-", ".m4a", context.cacheDir)
1116
+ synchronized(created) { tempFilesCreated.incrementAndGet(); created.add(mixFile) }
1117
+ val mixResult = JSONObject(
1118
+ run {
1119
+ // The mix is the longest uninterruptible stretch in an
1120
+ // export; without this a cancel waits out the stretch and
1121
+ // the AAC encode before any checkpoint can see it.
1122
+ //
1123
+ // Passed rather than assigned to the singleton: preview
1124
+ // mixes the same way now, and two callers sharing one
1125
+ // mutable predicate is a race where the loser becomes
1126
+ // uncancellable.
1127
+ try {
1128
+ AudioMixer.mix(
1129
+ context, mixProgramJson, mixUrisJson, mixFile,
1130
+ keepGoing = { !cancelled },
1131
+ )
1132
+ } catch (e: AudioMixer.Cancelled) {
1133
+ JSONObject().put("ok", false).put("cancelled", true).toString()
1134
+ }
1135
+ },
1136
+ )
1137
+ if (!mixResult.optBoolean("ok", false)) {
1138
+ releaseTemp(mixFile)
1139
+ // Stopped because it was asked to, not because the mix failed.
1140
+ if (mixResult.optBoolean("cancelled", false)) {
1141
+ return failed("cancelled during audio", 0, Thread.currentThread().name)
1142
+ }
1143
+ return failed("audio mix: " + mixResult.optString("error"), 0, Thread.currentThread().name)
1144
+ }
1145
+ val sound = Outcome(
1146
+ ok = true, uri = mixFile.absolutePath, error = "", elapsedMs = 0,
1147
+ fileBytes = mixFile.length(), durationMs = 0, width = 0, height = 0,
1148
+ videoFrameCount = 0, hasAudio = true, videoMimeType = "", audioMimeType = "audio/mp4a-latm",
1149
+ colorInfo = "", glErrors = "", effectFrames = 0, effectTimestampsUs = "",
1150
+ boundary = "", lifecycle = "", programTrace = "", muxTrace = "",
1151
+ overlayUploads = 0, overlayDraws = 0, overlayTexturesFreed = 0,
1152
+ nativeHeapBytes = 0, javaHeapBytes = 0, callerThread = Thread.currentThread().name,
1153
+ )
1154
+ audioUri = sound.uri
1155
+
1156
+ val output = File.createTempFile(TEMP_PREFIX + "final-", ".mp4", context.cacheDir)
1157
+ synchronized(created) { tempFilesCreated.incrementAndGet(); created.add(output) }
1158
+ // The compiled end, so the AAC tail cannot make the file longer
1159
+ // than the timeline it was built from.
1160
+ val compiledEndUs = shards.sumOf { it.spanUs }
1161
+ /*
1162
+ Provenance hashing is conformance work, not export work.
1163
+
1164
+ It reads every sample of both intermediates and of the result to
1165
+ prove the mux copied rather than re-encoded — worth doing in a
1166
+ gate, and pure cost in a production export. It is off by default:
1167
+ a watchdog threshold measured with it present would be calibrated
1168
+ against work the user never pays for, and would then be wrong for
1169
+ the export they actually run.
1170
+ */
1171
+ stopped("concatenating")?.let { return it }
1172
+ val videoBefore =
1173
+ if (diagnostics && provenance) JSONObject(GateFrames.sampleHashes(context, video.uri))
1174
+ else JSONObject()
1175
+ val audioBefore =
1176
+ if (diagnostics && provenance) JSONObject(GateFrames.sampleHashes(context, sound.uri))
1177
+ else JSONObject()
1178
+ val muxed = JSONObject(
1179
+ GateMux.mux(
1180
+ context, video.uri, sound.uri, output, compiledEndUs,
1181
+ primingFramesFor(context, sound.uri),
1182
+ keepGoing = { !cancelled },
1183
+ ),
1184
+ )
1185
+ if (!muxed.optBoolean("ok", false)) {
1186
+ releaseTemp(output)
1187
+ // A cancelled mux is a cancellation, not a mux failure: the copy
1188
+ // stopped because it was asked to.
1189
+ if (muxed.optBoolean("cancelled", false)) {
1190
+ return failed("cancelled during concatenating", 0, Thread.currentThread().name)
1191
+ }
1192
+ return failed("mux: " + muxed.optString("error"), 0, Thread.currentThread().name)
1193
+ }
1194
+ // Muxed, not published. The bar must not reach 1 here: the file is
1195
+ // written but nothing has yet confirmed it opens.
1196
+ ExportProgress.muxed(ExportHeartbeat.currentEpoch())
1197
+ ExportHeartbeat.beat("publishBytes", ExportHeartbeat.currentEpoch())
1198
+ val after =
1199
+ if (diagnostics && provenance) JSONObject(GateFrames.sampleHashes(context, output.absolutePath))
1200
+ else JSONObject()
1201
+ ExportHeartbeat.beat("publishBytes", ExportHeartbeat.currentEpoch())
1202
+ val trace = JSONObject()
1203
+ .put("videoBefore", videoBefore.optJSONArray("sampleHashes")?.length() ?: -1)
1204
+ .put("videoAfter", after.optJSONArray("sampleHashes")?.length() ?: -1)
1205
+ .put(
1206
+ "videoIdentical",
1207
+ videoBefore.optJSONArray("sampleHashes")?.toString() ==
1208
+ after.optJSONArray("sampleHashes")?.toString(),
1209
+ )
1210
+ .put("audioBefore", audioBefore.optJSONArray("audioHashes")?.length() ?: -1)
1211
+ .put("audioAfter", after.optJSONArray("audioHashes")?.length() ?: -1)
1212
+ .put("audioPrimingUnitsDropped", muxed.optInt("primingUnitsDropped"))
1213
+ .put("audioPrimingResidualFrames", muxed.optInt("primingResidualFrames"))
1214
+ .put(
1215
+ "audioPrefixIdentical",
1216
+ prefixMatches(audioBefore, after, muxed.optInt("primingUnitsDropped")),
1217
+ )
1218
+ .put("muxVideoSamples", muxed.optInt("videoSamples"))
1219
+ .put("muxAudioSamples", muxed.optInt("audioSamples"))
1220
+ /*
1221
+ Temporary: the caller keeps it.
1222
+
1223
+ Nothing sweeps it out from under them, because the adapter cannot
1224
+ know when a host has finished previewing. It stays registered, so
1225
+ an explicit release still finds it.
1226
+ */
1227
+ if (target == null) {
1228
+ return video.copy(
1229
+ uri = output.absolutePath,
1230
+ fileBytes = output.length(),
1231
+ hasAudio = true,
1232
+ programTrace = video.programTrace,
1233
+ muxTrace = trace.toString(),
1234
+ )
1235
+ }
1236
+
1237
+ stopped("publishing")?.let { releaseTemp(output); return it }
1238
+ /*
1239
+ Staged beside the target, so the last step is a rename.
1240
+
1241
+ Writing the destination in place would make it visible while it is
1242
+ half-written — a player opens a truncated container, a scanner
1243
+ indexes a file that will change under it, and a failure at 90%
1244
+ leaves something that looks like a video and is not one.
1245
+ */
1246
+ // Anything a previous run left in the destination's directory. It
1247
+ // is not in the cache, so the temporary sweep never sees it.
1248
+ val sweptStaging = GateDestination.sweepStaging(target)
1249
+ val staging = GateDestination.stagingFor(target, System.nanoTime())
1250
+ synchronized(created) { tempFilesCreated.incrementAndGet(); created.add(staging) }
1251
+ val stageError = GateDestination.stage(
1252
+ output, staging,
1253
+ keepGoing = { !cancelled },
1254
+ beatEpoch = ExportHeartbeat.currentEpoch(),
1255
+ hold = diagnostics && publishFault == "holdCopy",
1256
+ )
1257
+ val injected = if (diagnostics) publishFault else ""
1258
+ if (stageError == null && injected == "stage") {
1259
+ releaseTemp(staging)
1260
+ releaseTemp(output)
1261
+ return failed(
1262
+ "staging '" + GateDestination.describe(target.path) + "' failed: injected",
1263
+ 0, Thread.currentThread().name,
1264
+ )
1265
+ }
1266
+ if (stageError != null) {
1267
+ releaseTemp(staging)
1268
+ releaseTemp(output)
1269
+ // The existing target has not been touched: nothing has been
1270
+ // renamed, and the staging file is what took the damage.
1271
+ return failed(
1272
+ if (stageError == "cancelled") "cancelled during publishing" else stageError,
1273
+ 0, Thread.currentThread().name,
1274
+ )
1275
+ }
1276
+ // The staged container, before it becomes the destination. A file
1277
+ // that will not open must never be the thing that replaces one that
1278
+ // would have.
1279
+ val stagedProbe = GateDestination.probe(staging.absolutePath)
1280
+ if (!stagedProbe.optBoolean("readable", false)) {
1281
+ releaseTemp(staging)
1282
+ releaseTemp(output)
1283
+ return failed(
1284
+ "the staged file could not be read back: " + stagedProbe.optString("error", "no video track"),
1285
+ 0, Thread.currentThread().name,
1286
+ )
1287
+ }
1288
+ if (injected == "afterStage") {
1289
+ // Written, flushed, verified — and not published. The target
1290
+ // must be exactly as it was.
1291
+ releaseTemp(staging)
1292
+ releaseTemp(output)
1293
+ return failed(
1294
+ "publishing '" + GateDestination.describe(target.path) + "' failed: injected",
1295
+ 0, Thread.currentThread().name,
1296
+ )
1297
+ }
1298
+ if (injected == "raceCreate") {
1299
+ // Between the preflight check and the publication, which is
1300
+ // exactly where a preflight-only guard loses.
1301
+ runCatching { File(target.path).writeText("raced") }
1302
+ }
1303
+ GateDestination.publish(staging, target)?.let {
1304
+ releaseTemp(staging)
1305
+ releaseTemp(output)
1306
+ return failed(it, 0, Thread.currentThread().name)
1307
+ }
1308
+ // Published: the staging entry is gone from the filesystem either
1309
+ // way, so the ledger is told rather than left to guess.
1310
+ synchronized(created) {
1311
+ if (created.remove(staging)) tempFilesReleased.incrementAndGet()
1312
+ }
1313
+ releaseTemp(output)
1314
+ return video.copy(
1315
+ uri = target.path,
1316
+ fileBytes = File(target.path).length(),
1317
+ hasAudio = true,
1318
+ programTrace = video.programTrace,
1319
+ muxTrace = trace.toString(),
1320
+ boundary = if (sweptStaging > 0) "sweptStaging=" + sweptStaging else video.boundary,
1321
+ )
1322
+ } finally {
1323
+ /*
1324
+ The intermediates go on every path.
1325
+
1326
+ They are full-size encoded media, and the run that failed is
1327
+ exactly the one nobody comes back for.
1328
+ */
1329
+ for (path in listOfNotNull(videoUri, audioUri)) {
1330
+ val file = File(path)
1331
+ releaseTemp(file)
1332
+ }
1333
+ }
1334
+ }
1335
+
1336
+ /**
1337
+ * One audio-only pass over the whole timeline.
1338
+ *
1339
+ * The video pipeline is separate and already proven: video shards carry
1340
+ * video only, concatenate by sample copy, and never touch audio. Audio is
1341
+ * built once, here, straight from the original sources — so it is encoded
1342
+ * exactly once and the final step has nothing left to re-encode.
1343
+ *
1344
+ * A muted or audio-less span becomes a **gap of exactly its compiled
1345
+ * duration**. Not a silent media item: a gap cannot drift by an encoder's
1346
+ * priming, and every clip after it keeps the timeline position the compiler
1347
+ * gave it.
1348
+ */
1349
+ fun runAudioTimeline(items: List<AudioSpan>, timeoutSeconds: Long = 240): Outcome {
1350
+ beginRun()
1351
+ val callerThread = Thread.currentThread().name
1352
+ val output = File.createTempFile(TEMP_PREFIX + "audio-", ".mp4", context.cacheDir)
1353
+ synchronized(created) { tempFilesCreated.incrementAndGet(); created.add(output) }
1354
+
1355
+ val startedNs = System.nanoTime()
1356
+ val latch = CountDownLatch(1)
1357
+ var result: ExportResult? = null
1358
+ var failure: ExportException? = null
1359
+ var buildFailure: Throwable? = null
1360
+ // A run stopped before its Transformer started is not a success, and
1361
+ // not a failure either: `finishRun` reads this and the cancel flag.
1362
+ var cancelledEarly = false
1363
+
1364
+ main.post {
1365
+ try {
1366
+ val builder = EditedMediaItemSequence.Builder()
1367
+ builder.experimentalSetForceAudioTrack(true)
1368
+ for (span in items) {
1369
+ if (span.silent) {
1370
+ builder.addGap(span.durationUs)
1371
+ continue
1372
+ }
1373
+ val media = MediaItem.Builder()
1374
+ .setUri(span.uri)
1375
+ .setClippingConfiguration(
1376
+ MediaItem.ClippingConfiguration.Builder()
1377
+ .setStartPositionMs(span.startMs)
1378
+ .setEndPositionMs(span.endMs)
1379
+ .build(),
1380
+ )
1381
+ .build()
1382
+ val edited = EditedMediaItem.Builder(media).setRemoveVideo(true)
1383
+ val processors = ArrayList<androidx.media3.common.audio.AudioProcessor>()
1384
+ // One channel layout across the whole timeline, so nothing later
1385
+ // has to reconcile a mono span with a stereo one.
1386
+ val mixer = androidx.media3.common.audio.ChannelMixingAudioProcessor()
1387
+ for (inChannels in 1..2) {
1388
+ mixer.putChannelMixingMatrix(
1389
+ androidx.media3.common.audio.ChannelMixingMatrix.createForConstantPower(inChannels, 2),
1390
+ )
1391
+ }
1392
+ processors.add(mixer)
1393
+ if (span.speed != 1f) {
1394
+ edited.setSpeed(ConstantSpeed(span.speed))
1395
+ if (span.speedMode != "resample") {
1396
+ val sonic = androidx.media3.common.audio.SonicAudioProcessor()
1397
+ sonic.setPitch(1f / span.speed)
1398
+ processors.add(sonic)
1399
+ }
1400
+ }
1401
+ edited.setEffects(Effects(ImmutableList.copyOf(processors), ImmutableList.of()))
1402
+ builder.addItem(edited.build())
1403
+ }
1404
+ /*
1405
+ The sequence forces the audio track, not the composition.
1406
+
1407
+ Setting it at composition level on an audio-only composition throws
1408
+ `trackTypes must only contain TRACK_TYPE_AUDIO and/or
1409
+ TRACK_TYPE_VIDEO` from inside `Composition.Builder.build`, which
1410
+ took the process with it before this was wrapped.
1411
+ */
1412
+ val composition = Composition.Builder(builder.build()).build()
1413
+ val transformer = Transformer.Builder(context)
1414
+ .addListener(object : Transformer.Listener {
1415
+ override fun onCompleted(c: Composition, r: ExportResult) {
1416
+ activeTransformer = null
1417
+ activeLatch = null
1418
+ result = r; latch.countDown()
1419
+ }
1420
+ override fun onError(c: Composition, r: ExportResult, e: ExportException) {
1421
+ activeTransformer = null
1422
+ activeLatch = null
1423
+ result = r; failure = e; latch.countDown()
1424
+ }
1425
+ })
1426
+ .build()
1427
+ /*
1428
+ The race a flag alone does not close.
1429
+
1430
+ A Transformer is built inside a main-thread post, so a cancel can
1431
+ arrive after the job has started and before the object exists --
1432
+ `cancelJob` then finds nothing to cancel, sets the flag, and the
1433
+ Transformer starts anyway and runs to completion. Measured: a
1434
+ cancel issued before any native work still took 4.7 s to settle,
1435
+ which is the whole of the first shard.
1436
+ */
1437
+ if (cancelled) {
1438
+ // Not counted here. `finishRun` is the only place a run reaches
1439
+ // a terminal outcome, so it cannot be counted twice.
1440
+ cancelledEarly = true
1441
+ latch.countDown()
1442
+ return@post
1443
+ }
1444
+ activeTransformer = transformer
1445
+ activeLatch = latch
1446
+ transformer.start(composition, output.absolutePath)
1447
+ } catch (e: Throwable) {
1448
+ /*
1449
+ A malformed composition must fail the export, not the process.
1450
+
1451
+ Media3 validates on `build()` and throws on the main thread; an
1452
+ unwrapped throw here killed the app outright, which loses the
1453
+ error, the cleanup and every other run in the session.
1454
+ */
1455
+ buildFailure = e
1456
+ latch.countDown()
1457
+ }
1458
+ }
1459
+
1460
+ val finished = latch.await(timeoutSeconds, TimeUnit.SECONDS)
1461
+ val elapsedMs = (System.nanoTime() - startedNs) / 1_000_000
1462
+ val ok = finished && failure == null && buildFailure == null && !cancelledEarly
1463
+ if (!ok) {
1464
+ releaseTemp(output)
1465
+ }
1466
+ ExportHeartbeat.beat("videoFrame", ExportHeartbeat.currentEpoch())
1467
+ finishRun(ok, cancelled)
1468
+ return Outcome(
1469
+ ok = ok,
1470
+ uri = if (ok) output.absolutePath else "",
1471
+ error = when {
1472
+ buildFailure != null -> "audio composition rejected: " +
1473
+ buildFailure!!.javaClass.simpleName + ": " + buildFailure!!.message
1474
+ !finished -> "audio timeline timed out after ${'$'}timeoutSeconds s"
1475
+ failure != null -> "${'$'}{failure!!.errorCode}: ${'$'}{failure!!.message}"
1476
+ else -> ""
1477
+ },
1478
+ elapsedMs = elapsedMs,
1479
+ fileBytes = if (ok) output.length() else 0,
1480
+ durationMs = result?.durationMs ?: 0,
1481
+ width = 0, height = 0, videoFrameCount = 0,
1482
+ hasAudio = true,
1483
+ videoMimeType = "", audioMimeType = result?.audioEncoderName ?: "",
1484
+ colorInfo = "", glErrors = "", effectFrames = 0, effectTimestampsUs = "",
1485
+ boundary = "", lifecycle = "", programTrace = "",
1486
+ overlayUploads = 0, overlayDraws = 0, overlayTexturesFreed = 0,
1487
+ nativeHeapBytes = android.os.Debug.getNativeHeapAllocatedSize(),
1488
+ javaHeapBytes = Runtime.getRuntime().let { it.totalMemory() - it.freeMemory() },
1489
+ callerThread = callerThread,
1490
+ )
1491
+ }
1492
+
1493
+ /** One span of the audio timeline: a clipped source, or silence. */
1494
+ class AudioSpan(
1495
+ val uri: String,
1496
+ val startMs: Long,
1497
+ val endMs: Long,
1498
+ val speed: Float,
1499
+ val speedMode: String,
1500
+ /** Silence of exactly [durationUs]; `uri` and the trim are ignored. */
1501
+ val silent: Boolean,
1502
+ val durationUs: Long,
1503
+ )
1504
+
1505
+ /**
1506
+ * Whether the final's audio is the timeline's audio, minus a trimmed tail.
1507
+ *
1508
+ * Exact equality is the wrong test: the tail is deliberately dropped so the
1509
+ * container cannot outlast the compiled timeline. What must hold is that
1510
+ * every sample kept is byte-identical and in order — copied, never
1511
+ * re-encoded.
1512
+ */
1513
+ /**
1514
+ * Whether the muxed audio is the encoder's own bytes, allowing for the
1515
+ * priming units the mux removed.
1516
+ *
1517
+ * The comparison starts at `skip` rather than at zero, because removing
1518
+ * priming is exactly a change to the beginning of the stream. Comparing from
1519
+ * zero would report every correctly trimmed file as a payload mismatch --
1520
+ * an alarm that fires on the fix -- while ignoring the offset entirely would
1521
+ * stop checking the one thing this exists to check.
1522
+ */
1523
+ private fun prefixMatches(before: JSONObject, after: JSONObject, skip: Int): Boolean {
1524
+ val a = before.optJSONArray("audioHashes") ?: return false
1525
+ val b = after.optJSONArray("audioHashes") ?: return false
1526
+ if (skip < 0 || b.length() + skip > a.length()) return false
1527
+ for (i in 0 until b.length()) {
1528
+ if (a.optLong(i + skip) != b.optLong(i)) return false
1529
+ }
1530
+ return b.length() > 0
1531
+ }
1532
+
1533
+ private fun runSequence(
1534
+ uris: List<String>,
1535
+ timeoutSeconds: Long,
1536
+ compiled: SceneProgramNative?,
1537
+ withSkia: Boolean = true,
1538
+ program: SceneProgram? = null,
1539
+ failAfterFrames: Int = -1,
1540
+ items: List<Item>? = null,
1541
+ forceAudioTrack: Boolean = false,
1542
+ timelineOffsetUs: Long = 0,
1543
+ transmuxVideo: Boolean = false,
1544
+ /** Thin the stream to this rate **before** the scene interpreter, or 0. */
1545
+ frameDropFps: Int = 0,
1546
+ /** Encoder-level repeat interval, or 0 for the default encoder. */
1547
+ encoderRepeatUs: Long = 0,
1548
+ /** Grid normalisation, upstream of the scene interpreter. */
1549
+ cadence: CadenceEffect? = null,
1550
+ /** Non-zero when the sources are still images rather than video. */
1551
+ stillDurationUs: Long = 0,
1552
+ /** Channel count every shard's audio is mixed to, or 0 to leave it alone. */
1553
+ normaliseAudioChannels: Int = 0,
1554
+ /** Preloaded overlay rasters, or null when the scene names none. */
1555
+ overlays: OverlayResources? = null,
1556
+ overlayFailAfterDraws: Int = -1,
1557
+ ): Outcome {
1558
+ beginRun()
1559
+ val callerThread = Thread.currentThread().name
1560
+ val output = File.createTempFile(TEMP_PREFIX, ".mp4", context.cacheDir)
1561
+ synchronized(created) { tempFilesCreated.incrementAndGet(); created.add(output) }
1562
+
1563
+ val startedNs = System.nanoTime()
1564
+ val latch = CountDownLatch(1)
1565
+ var result: ExportResult? = null
1566
+ var failure: ExportException? = null
1567
+ // A shard stopped before its Transformer started is neither a success
1568
+ // nor a failure; `finishRun` reads this alongside the cancel flag.
1569
+ var cancelledEarly = false
1570
+ val probe =
1571
+ if (withSkia) {
1572
+ SkiaPassThroughEffect(
1573
+ program, failAfterFrames, compiled, timelineOffsetUs, overlays, overlayFailAfterDraws,
1574
+ )
1575
+ } else {
1576
+ null
1577
+ }
1578
+
1579
+ main.post {
1580
+ val effects: ImmutableList<Effect> =
1581
+ if (probe == null) ImmutableList.of() else ImmutableList.of(probe as Effect)
1582
+ val builder = EditedMediaItemSequence.Builder()
1583
+ /*
1584
+ Media3's speed path is driven through audio. A source with no audio
1585
+ track fails outright at any rate other than 1, with the muxer
1586
+ aborting on "no output sample written" — so a silent clip is not a
1587
+ smaller case of an audible one. Generating silence gives the path
1588
+ the track it is waiting for.
1589
+ */
1590
+ if (forceAudioTrack) builder.experimentalSetForceAudioTrack(true)
1591
+ for ((index, source) in uris.withIndex()) {
1592
+ val item = items?.getOrNull(index)
1593
+ val media = if (stillDurationUs > 0) {
1594
+ /*
1595
+ An image needs three things the default path does not
1596
+ supply: a MIME so the factory routes it to a bitmap loader
1597
+ rather than the ExoPlayer one, and a duration **on the
1598
+ MediaItem** — `EditedMediaItem.setDurationUs` is not the
1599
+ same field, and Media3 says so in the failure it raises.
1600
+ */
1601
+ MediaItem.Builder()
1602
+ .setUri(android.net.Uri.fromFile(File(source)))
1603
+ .setMimeType(androidx.media3.common.MimeTypes.IMAGE_PNG)
1604
+ .setImageDurationMs(stillDurationUs / 1000)
1605
+ .build()
1606
+ } else if (item != null && item.endUs > 0) {
1607
+ MediaItem.Builder()
1608
+ .setUri(source)
1609
+ .setClippingConfiguration(
1610
+ MediaItem.ClippingConfiguration.Builder()
1611
+ .setStartPositionUs(0)
1612
+ .setEndPositionUs(item.endUs)
1613
+ .build(),
1614
+ )
1615
+ .build()
1616
+ } else if (item != null && item.endMs > item.startMs) {
1617
+ MediaItem.Builder()
1618
+ .setUri(source)
1619
+ .setClippingConfiguration(
1620
+ MediaItem.ClippingConfiguration.Builder()
1621
+ .setStartPositionMs(item.startMs)
1622
+ .setEndPositionMs(item.endMs)
1623
+ .build(),
1624
+ )
1625
+ .build()
1626
+ } else {
1627
+ MediaItem.fromUri(source)
1628
+ }
1629
+ /*
1630
+ Drop *upstream* of the scene interpreter.
1631
+
1632
+ Order decides what the interpreter is asked to draw. Dropping
1633
+ first means it runs once per output frame, which is the
1634
+ contract. Dropping after would have it draw frames that are
1635
+ then thrown away — the same picture, at more cost, with scene
1636
+ executions no longer equal to output frames.
1637
+ */
1638
+ val chain: ImmutableList<Effect> =
1639
+ if (frameDropFps > 0 || cadence != null) {
1640
+ val all = ArrayList<Effect>()
1641
+ if (frameDropFps > 0) {
1642
+ all.add(FrameDropEffect.createDefaultFrameDropEffect(frameDropFps.toFloat()))
1643
+ }
1644
+ if (cadence != null) all.add(cadence)
1645
+ all.addAll(effects)
1646
+ ImmutableList.copyOf(all)
1647
+ } else {
1648
+ effects
1649
+ }
1650
+ val edited = EditedMediaItem.Builder(media)
1651
+ .also { b ->
1652
+ if (stillDurationUs > 0) {
1653
+ // An image has no duration or rate of its own; both
1654
+ // are required before it can become video.
1655
+ b.setDurationUs(stillDurationUs)
1656
+ b.setFrameRate(30)
1657
+ }
1658
+ }
1659
+ /*
1660
+ Muting is silence, not absence, and it is set alongside the
1661
+ rate change deliberately: `setRemoveAudio` with a speed change
1662
+ asks Media3 to re-time a stream it has been told to discard,
1663
+ which is the deadlock this slice was warned about. Every
1664
+ combination is exercised, and the run has a timeout so a hang
1665
+ would be reported rather than waited on.
1666
+ */
1667
+ if (item != null && item.removeAudio) {
1668
+ edited.setRemoveAudio(true)
1669
+ }
1670
+ /*
1671
+ Pitch is preserved by default, and that is a measurement rather
1672
+ than a preference: `setSpeed` alone put a 440 Hz tone at 215 Hz
1673
+ on a 0.5x clip and 883 Hz on a 2x one -- half and double, the
1674
+ tape effect. `createExperimentalSpeedChangingEffect` returns a
1675
+ Sonic audio processor paired with the matching timestamp
1676
+ adjustment, so the two stay coordinated instead of being two
1677
+ independent rate changes that could disagree.
1678
+ */
1679
+ var speedAudio: androidx.media3.common.audio.AudioProcessor? = null
1680
+ val speedVideo: Effect? = null
1681
+ if (item != null && item.speed != 1f) {
1682
+ edited.setSpeed(ConstantSpeed(item.speed))
1683
+ if (item.speedMode != "resample") {
1684
+ /*
1685
+ Media3 re-times audio by **resampling** it, on both
1686
+ public paths: `setSpeed` and
1687
+ `createExperimentalSpeedChangingEffect` both put a
1688
+ 440 Hz tone at 215 Hz on a 0.5x clip and 883 Hz on a
1689
+ 2x one, measured, identically. That is the tape
1690
+ effect, and it is not what a speed control on a clip
1691
+ should do to a voice.
1692
+
1693
+ Sonic's pitch shift is the correction: the resampling
1694
+ already moved tempo and pitch together by the rate, so
1695
+ shifting pitch back by its reciprocal leaves the tempo
1696
+ changed and the pitch where it started. Composed
1697
+ rather than replaced, because the resampling is also
1698
+ what keeps audio and video the same length.
1699
+ */
1700
+ val sonic = androidx.media3.common.audio.SonicAudioProcessor()
1701
+ sonic.setPitch(1f / item.speed)
1702
+ speedAudio = sonic
1703
+ }
1704
+ }
1705
+ run {
1706
+ val audio = ArrayList<androidx.media3.common.audio.AudioProcessor>()
1707
+ /*
1708
+ Every shard's audio ends up the same shape.
1709
+
1710
+ A source's own track is whatever it is -- the fixtures are
1711
+ mono -- while the silence Media3 generates for a muted or
1712
+ audio-less clip is stereo. A concatenation cannot copy
1713
+ samples between tracks with different channel counts, so it
1714
+ silently re-encodes instead: measured, 176 shard samples
1715
+ became 174 of which only 120 were byte-identical. Forcing
1716
+ one layout on every shard is what makes the final transmux
1717
+ a copy.
1718
+ */
1719
+ if (normaliseAudioChannels > 0) {
1720
+ val mixer = androidx.media3.common.audio.ChannelMixingAudioProcessor()
1721
+ for (inChannels in 1..2) {
1722
+ mixer.putChannelMixingMatrix(
1723
+ androidx.media3.common.audio.ChannelMixingMatrix.createForConstantPower(
1724
+ inChannels, normaliseAudioChannels,
1725
+ ),
1726
+ )
1727
+ }
1728
+ audio.add(mixer)
1729
+ }
1730
+ speedAudio?.let { audio.add(it) }
1731
+ val video = ArrayList<Effect>()
1732
+ // The timestamp adjustment first: everything downstream must
1733
+ // see the re-timed stream, not the original one.
1734
+ speedVideo?.let { video.add(it) }
1735
+ video.addAll(chain)
1736
+ edited.setEffects(Effects(ImmutableList.copyOf(audio), ImmutableList.copyOf(video)))
1737
+ }
1738
+ if (item != null && item.durationUs > 0) {
1739
+ // Source-time length, which is what the clip actually is.
1740
+ // Declared, not invented: Media3 cannot always determine an
1741
+ // item's duration itself, and a sequence that guesses wrong
1742
+ // lays out the following items at the wrong offset.
1743
+ edited.setDurationUs(item.durationUs)
1744
+ }
1745
+ builder.addItem(edited.build())
1746
+ }
1747
+ val composition = Composition.Builder(builder.build())
1748
+ // Force the decode/encode round trip even with no effect, so the
1749
+ // vanilla run measures the codec rather than a byte copy.
1750
+ .setTransmuxVideo(transmuxVideo)
1751
+ /*
1752
+ Audio too, or the concatenation re-encodes it.
1753
+
1754
+ `setTransmuxVideo` covers only video: the first measured
1755
+ concatenation copied all 120 video samples byte-for-byte while
1756
+ turning 86 audio samples into 175 re-encoded ones. A second
1757
+ generation of lossy audio on every export, invisible to any
1758
+ video-only check.
1759
+ */
1760
+ .setTransmuxAudio(transmuxVideo)
1761
+ .build()
1762
+ val transformer = Transformer.Builder(context)
1763
+ .also { builderRef ->
1764
+ if (stillDurationUs > 0) {
1765
+ // The default factory routes everything to the ExoPlayer
1766
+ // asset loader, which looks for tracks a PNG does not
1767
+ // have. An image needs a loader that can read a bitmap.
1768
+ builderRef.setAssetLoaderFactory(
1769
+ androidx.media3.transformer.DefaultAssetLoaderFactory(
1770
+ context,
1771
+ androidx.media3.datasource.DataSourceBitmapLoader(context),
1772
+ ),
1773
+ )
1774
+ }
1775
+ }
1776
+ .also { builderRef ->
1777
+ if (encoderRepeatUs > 0 && Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
1778
+ builderRef.setEncoderFactory(
1779
+ RepeatingEncoderFactory(context, encoderRepeatUs, 30),
1780
+ )
1781
+ }
1782
+ }
1783
+ .addListener(object : Transformer.Listener {
1784
+ override fun onCompleted(composition: Composition, exportResult: ExportResult) {
1785
+ activeTransformer = null
1786
+ activeLatch = null
1787
+ result = exportResult
1788
+ latch.countDown()
1789
+ }
1790
+
1791
+ override fun onError(
1792
+ composition: Composition,
1793
+ exportResult: ExportResult,
1794
+ exportException: ExportException,
1795
+ ) {
1796
+ activeTransformer = null
1797
+ activeLatch = null
1798
+ failure = exportException
1799
+ latch.countDown()
1800
+ }
1801
+ })
1802
+ .build()
1803
+ /*
1804
+ Published so a cancel can reach *this* Transformer.
1805
+
1806
+ This is the one that encodes the video shards, and it was the only
1807
+ construction site that never registered itself. `cancelJob` found
1808
+ nothing to cancel and returned in 30 microseconds while the shard
1809
+ carried on emitting — traced at 0.1x, where the cadence loop went
1810
+ from 30 to 220 copies after the cancel had already "returned".
1811
+ Every cancellation then waited for the shard to finish on its own
1812
+ and was caught by the next stage check, which is where the seconds
1813
+ came from.
1814
+ */
1815
+ if (cancelled) {
1816
+ cancelledEarly = true
1817
+ latch.countDown()
1818
+ return@post
1819
+ }
1820
+ activeTransformer = transformer
1821
+ activeLatch = latch
1822
+ runCatching { transformer.start(composition, output.absolutePath) }
1823
+ .onFailure {
1824
+ failure = ExportException.createForUnexpected(RuntimeException(it))
1825
+ latch.countDown()
1826
+ }
1827
+ }
1828
+
1829
+ val finished = latch.await(timeoutSeconds, TimeUnit.SECONDS)
1830
+ val elapsedMs = (System.nanoTime() - startedNs) / 1_000_000
1831
+ val ok = finished && failure == null && result != null && !cancelledEarly
1832
+
1833
+ if (!ok) {
1834
+ // A failed gate leaves no partial file. Transformer writes as it
1835
+ // goes, so there usually *is* one by now.
1836
+ releaseTemp(output)
1837
+ probe?.release()
1838
+ finishRun(ok = false, wasCancelled = cancelled)
1839
+ return Outcome(
1840
+ ok = false,
1841
+ uri = "",
1842
+ error = when {
1843
+ !finished -> "timeout after ${timeoutSeconds}s"
1844
+ failure != null -> buildString {
1845
+ append(failure!!.errorCode).append(": ").append(failure!!.message)
1846
+ // The message alone says "Video frame processing error"
1847
+ // and nothing about which effect refused. The cause
1848
+ // chain is where the actual reason lives.
1849
+ var cause: Throwable? = failure!!.cause
1850
+ var depth = 0
1851
+ while (cause != null && depth < 4) {
1852
+ append(" <- ").append(cause.javaClass.simpleName)
1853
+ append(": ").append(cause.message)
1854
+ cause = cause.cause
1855
+ depth += 1
1856
+ }
1857
+ }
1858
+ else -> "no result"
1859
+ },
1860
+ elapsedMs = elapsedMs,
1861
+ fileBytes = 0,
1862
+ durationMs = 0,
1863
+ width = 0,
1864
+ height = 0,
1865
+ videoFrameCount = 0,
1866
+ hasAudio = false,
1867
+ videoMimeType = "",
1868
+ audioMimeType = "",
1869
+ colorInfo = "",
1870
+ glErrors = probe?.glErrors() ?: "",
1871
+ effectFrames = probe?.frameCount() ?: 0,
1872
+ effectTimestampsUs = probe?.timestamps() ?: "",
1873
+ boundary = probe?.boundary() ?: "",
1874
+ lifecycle = probe?.lifecycle() ?: "",
1875
+ programTrace = probe?.programTrace() ?: "",
1876
+ overlayUploads = probe?.overlayUploads?.get() ?: 0,
1877
+ overlayDraws = probe?.overlayDraws?.get() ?: 0,
1878
+ overlayTexturesFreed = probe?.overlayTexturesFreed?.get() ?: 0,
1879
+ nativeHeapBytes = android.os.Debug.getNativeHeapAllocatedSize(),
1880
+ javaHeapBytes = Runtime.getRuntime().let { it.totalMemory() - it.freeMemory() },
1881
+ callerThread = callerThread,
1882
+ )
1883
+ }
1884
+
1885
+ val r = result!!
1886
+ val outcome = Outcome(
1887
+ ok = true,
1888
+ // Scheme and nothing else at this boundary; the caller gets a path
1889
+ // it already owns.
1890
+ uri = output.absolutePath,
1891
+ error = "",
1892
+ elapsedMs = elapsedMs,
1893
+ fileBytes = output.length(),
1894
+ durationMs = r.durationMs,
1895
+ width = r.width,
1896
+ height = r.height,
1897
+ videoFrameCount = r.videoFrameCount,
1898
+ hasAudio = r.audioMimeType != null,
1899
+ videoMimeType = r.videoMimeType ?: "",
1900
+ audioMimeType = r.audioMimeType ?: "",
1901
+ colorInfo = r.colorInfo?.toString() ?: "",
1902
+ glErrors = probe?.glErrors() ?: "",
1903
+ effectFrames = probe?.frameCount() ?: 0,
1904
+ effectTimestampsUs = probe?.timestamps() ?: "",
1905
+ boundary = probe?.boundary() ?: "",
1906
+ lifecycle = probe?.lifecycle() ?: "",
1907
+ programTrace = probe?.programTrace() ?: "",
1908
+ overlayUploads = probe?.overlayUploads?.get() ?: 0,
1909
+ overlayDraws = probe?.overlayDraws?.get() ?: 0,
1910
+ overlayTexturesFreed = probe?.overlayTexturesFreed?.get() ?: 0,
1911
+ nativeHeapBytes = android.os.Debug.getNativeHeapAllocatedSize(),
1912
+ javaHeapBytes = Runtime.getRuntime().let { it.totalMemory() - it.freeMemory() },
1913
+ callerThread = callerThread,
1914
+ )
1915
+ // The effect's own resources go exactly once, whether or not it ran.
1916
+ probe?.release()
1917
+ finishRun(ok = outcome.ok, wasCancelled = cancelled)
1918
+ return outcome
1919
+ }
1920
+
1921
+ /** Delete every temporary this gate still owns. Idempotent. */
1922
+ /**
1923
+ * Release one temporary output, by path, and only if this made it.
1924
+ *
1925
+ * The production counterpart to `releaseTemporary`, which sweeps. A sweep is
1926
+ * exactly wrong here: a caller holding the temporary from an *earlier*
1927
+ * export still owns that file, and a later export's cleanup taking it is a
1928
+ * result disappearing under whoever was previewing it.
1929
+ *
1930
+ * Idempotent by construction. A path that is not in `created` was not made
1931
+ * here, so nothing is deleted and nothing is an error — which also means a
1932
+ * host cannot hand this an arbitrary path and have it removed.
1933
+ */
1934
+ fun releaseTemporaryOutput(path: String): Boolean {
1935
+ val file = synchronized(created) { created.firstOrNull { it.absolutePath == path } }
1936
+ ?: return false
1937
+ releaseTemp(file)
1938
+ return true
1939
+ }
1940
+
1941
+ /** Gate-only: which temporaries are still registered, by name. */
1942
+ fun liveTemporaryNames(): List<String> =
1943
+ synchronized(created) { created.map { it.name } }
1944
+
1945
+ fun releaseTemporary(): Int {
1946
+ var gone: Int
1947
+ synchronized(created) {
1948
+ // Everything registered is accounted for, whether or not the delete
1949
+ // found a file: a run that already removed its own output must not
1950
+ // leave the created/released ledger permanently short.
1951
+ tempFilesReleased.addAndGet(created.size.toLong())
1952
+ gone = created.count { runCatching { it.delete() }.getOrDefault(false) }
1953
+ created.clear()
1954
+ }
1955
+ /*
1956
+ Also sweep by name.
1957
+
1958
+ `created` only knows about this instance. A run whose process was
1959
+ killed, or an export deliberately kept for inspection, leaves a file no
1960
+ later instance can see — and the gate writes megabytes per run into a
1961
+ cache directory the user never asked to fill. The prefix is specific
1962
+ enough that nothing else can match it.
1963
+ */
1964
+ gone += runCatching {
1965
+ context.cacheDir.listFiles { f -> f.name.startsWith(TEMP_PREFIX) }
1966
+ ?.count { runCatching { it.delete() }.getOrDefault(false) } ?: 0
1967
+ }.getOrDefault(0)
1968
+ return gone
1969
+ }
1970
+
1971
+ private companion object {
1972
+ const val TEMP_PREFIX = "pictor-gate-"
1973
+ }
1974
+ }