@pictor-sdk/video-android 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +40 -0
- package/android/CMakeLists.txt +30 -0
- package/android/build.gradle +119 -0
- package/android/src/main/AndroidManifest.xml +1 -0
- package/android/src/main/cpp/pictor_overlaystage.cpp +372 -0
- package/android/src/main/cpp/pictor_publish.cpp +87 -0
- package/android/src/main/cpp/pictor_videosurface.cpp +735 -0
- package/android/src/main/java/dev/pictor/video/AacPriming.kt +261 -0
- package/android/src/main/java/dev/pictor/video/AudioMixer.kt +1112 -0
- package/android/src/main/java/dev/pictor/video/AudioProbe.kt +162 -0
- package/android/src/main/java/dev/pictor/video/CadenceShaderProgram.kt +337 -0
- package/android/src/main/java/dev/pictor/video/CodecLedger.kt +126 -0
- package/android/src/main/java/dev/pictor/video/CompositionSpike.kt +978 -0
- package/android/src/main/java/dev/pictor/video/Diagnostics.kt +54 -0
- package/android/src/main/java/dev/pictor/video/EncoderGate.kt +476 -0
- package/android/src/main/java/dev/pictor/video/ExportGate.kt +1974 -0
- package/android/src/main/java/dev/pictor/video/ExportHeartbeat.kt +72 -0
- package/android/src/main/java/dev/pictor/video/ExportProgress.kt +103 -0
- package/android/src/main/java/dev/pictor/video/FrameClock.kt +591 -0
- package/android/src/main/java/dev/pictor/video/FramePending.kt +181 -0
- package/android/src/main/java/dev/pictor/video/GateAudio.kt +456 -0
- package/android/src/main/java/dev/pictor/video/GateDestination.kt +445 -0
- package/android/src/main/java/dev/pictor/video/GateFrames.kt +662 -0
- package/android/src/main/java/dev/pictor/video/GateMux.kt +185 -0
- package/android/src/main/java/dev/pictor/video/ManagedSources.kt +333 -0
- package/android/src/main/java/dev/pictor/video/Media3Player.kt +934 -0
- package/android/src/main/java/dev/pictor/video/MediaStoreSave.kt +419 -0
- package/android/src/main/java/dev/pictor/video/OverlayResources.kt +169 -0
- package/android/src/main/java/dev/pictor/video/PictorVideoModule.kt +1612 -0
- package/android/src/main/java/dev/pictor/video/PictorVideoPackage.kt +26 -0
- package/android/src/main/java/dev/pictor/video/PlaybackAnchor.kt +161 -0
- package/android/src/main/java/dev/pictor/video/PreviewAudioPlayer.kt +291 -0
- package/android/src/main/java/dev/pictor/video/PreviewConfig.kt +96 -0
- package/android/src/main/java/dev/pictor/video/PreviewMix.kt +396 -0
- package/android/src/main/java/dev/pictor/video/PreviewSync.kt +374 -0
- package/android/src/main/java/dev/pictor/video/PreviewTrace.kt +478 -0
- package/android/src/main/java/dev/pictor/video/PrimingTrim.kt +337 -0
- package/android/src/main/java/dev/pictor/video/RepeatingEncoderFactory.kt +97 -0
- package/android/src/main/java/dev/pictor/video/SceneProgram.kt +169 -0
- package/android/src/main/java/dev/pictor/video/SceneProgramNative.kt +299 -0
- package/android/src/main/java/dev/pictor/video/SkiaPassThroughEffect.kt +695 -0
- package/android/src/main/java/dev/pictor/video/SurfaceAttachment.kt +110 -0
- package/android/src/main/java/dev/pictor/video/ThumbnailDecoder.kt +171 -0
- package/android/src/main/java/dev/pictor/video/VideoSurface.kt +394 -0
- package/app.plugin.js +43 -0
- package/dist/index.d.mts +3622 -0
- package/dist/index.d.ts +3622 -0
- package/dist/index.js +3737 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +3625 -0
- package/dist/index.mjs.map +1 -0
- package/package.json +82 -0
- package/plugin/gradleProperties.d.ts +19 -0
- package/plugin/gradleProperties.js +64 -0
- package/react-native.config.js +13 -0
- package/src/NativePictorVideo.ts +574 -0
- package/src/audioProbe.ts +72 -0
- package/src/compositionSpike.ts +305 -0
- package/src/convergence.ts +112 -0
- package/src/encoderGate.ts +66 -0
- package/src/exportDestination.ts +149 -0
- package/src/exportJob.ts +232 -0
- package/src/exportProgress.ts +391 -0
- package/src/exportProgressPump.ts +190 -0
- package/src/exporter.ts +655 -0
- package/src/index.ts +276 -0
- package/src/managedSources.ts +252 -0
- package/src/media3.ts +412 -0
- package/src/mediaStore.ts +177 -0
- package/src/memoryGuard.ts +118 -0
- package/src/mixSources.ts +82 -0
- package/src/overlayStaging.ts +216 -0
- package/src/playbackLifecycle.ts +137 -0
- package/src/player.ts +39 -0
- package/src/playlistSession.ts +125 -0
- package/src/previewAudioDiagnostics.ts +67 -0
- package/src/previewAudioSession.ts +168 -0
- package/src/previewCadence.ts +950 -0
- package/src/previewMix.ts +212 -0
- package/src/previewTrace.ts +348 -0
- package/src/spikeExcursions.ts +405 -0
- package/src/submissionScoring.ts +680 -0
- package/src/surfaceOwner.ts +264 -0
- package/src/surfaceRecovery.ts +121 -0
- package/src/syncTrace.ts +189 -0
- package/src/thumbnailProvider.ts +163 -0
- package/src/tracePhases.ts +217 -0
- package/src/useCompositionPlayback.ts +327 -0
- package/src/useMedia3Playback.ts +492 -0
- package/src/usePreviewAudio.ts +305 -0
- package/src/useVideoSurface.ts +282 -0
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,3622 @@
|
|
|
1
|
+
import { PlaylistResult, PlayIntent, PlaylistItem, UnsupportedSource, VideoProject, ThumbnailProvider, MediaSource, ExportSourceContext, VideoExporter, VideoExportProgress, MediaSaver, AudioProbe, AudioMixProgram } from '@pictor-sdk/video-core';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* A debug safety cutoff on native memory.
|
|
5
|
+
*
|
|
6
|
+
* **Remove before production.** This exists because an earlier revision drove
|
|
7
|
+
* the native heap to 1.5 GB in sixty seconds and the process was killed —
|
|
8
|
+
* taking other applications on the device with it, since Android reclaims from
|
|
9
|
+
* everything before it gives up on the foreground app. A diagnostic run must
|
|
10
|
+
* not be able to do that again.
|
|
11
|
+
*
|
|
12
|
+
* Two independent triggers, because a leak can be fast or merely relentless:
|
|
13
|
+
*
|
|
14
|
+
* - **Ceiling.** An absolute limit, well under anything that would put the
|
|
15
|
+
* device under pressure.
|
|
16
|
+
* - **Sustained rise.** Memory that climbs across every sample of a window with
|
|
17
|
+
* no plateau, which is what a slow leak looks like and what a ceiling alone
|
|
18
|
+
* would take minutes to catch.
|
|
19
|
+
*
|
|
20
|
+
* Pure, so the rule is tested rather than trusted: a cutoff that misfires stops
|
|
21
|
+
* a good run, and one that never fires is not a cutoff.
|
|
22
|
+
*
|
|
23
|
+
* @experimental
|
|
24
|
+
*/
|
|
25
|
+
interface MemoryGuard {
|
|
26
|
+
readonly ceilingBytes: number;
|
|
27
|
+
readonly windowSamples: number;
|
|
28
|
+
/** Minimum total climb across the window to count as a rise, in bytes. */
|
|
29
|
+
readonly riseBytes: number;
|
|
30
|
+
samples: number[];
|
|
31
|
+
tripped: null | 'ceiling' | 'sustained-rise';
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* 250 MB. Healthy single-clip playback plateaus near 65 MB.
|
|
35
|
+
*
|
|
36
|
+
* Was 400 MB, which a 1 MB/s leak takes six minutes to reach — long enough for
|
|
37
|
+
* a five-minute run to finish, look successful, and leave the process fat
|
|
38
|
+
* enough that the next run pushed the device into reclaiming from other
|
|
39
|
+
* applications.
|
|
40
|
+
*/
|
|
41
|
+
declare const DEFAULT_CEILING_BYTES: number;
|
|
42
|
+
/** Forty samples at 4 Hz — ten seconds of uninterrupted climbing. */
|
|
43
|
+
declare const DEFAULT_WINDOW_SAMPLES = 40;
|
|
44
|
+
/**
|
|
45
|
+
* 8 MB across the window: ~0.8 MB/s.
|
|
46
|
+
*
|
|
47
|
+
* Was 24 MB over three seconds — 8 MB/s, calibrated against a leak that hit
|
|
48
|
+
* 1.5 GB in a minute. It was blind to the 1.07 MB/s leak that actually got the
|
|
49
|
+
* process killed, so the cutoff never fired. A cutoff tuned only to the last
|
|
50
|
+
* failure is not a cutoff.
|
|
51
|
+
*
|
|
52
|
+
* A threshold this small is safe only because the rise must also be *strictly*
|
|
53
|
+
* increasing at every one of the forty samples. Real memory jitters; a leak
|
|
54
|
+
* does not.
|
|
55
|
+
*/
|
|
56
|
+
declare const DEFAULT_RISE_BYTES: number;
|
|
57
|
+
declare function createMemoryGuard(ceilingBytes?: number, windowSamples?: number, riseBytes?: number): MemoryGuard;
|
|
58
|
+
/**
|
|
59
|
+
* Record a reading and decide whether to stop.
|
|
60
|
+
*
|
|
61
|
+
* Once tripped it stays tripped: the caller has already torn the session down,
|
|
62
|
+
* and a guard that untrips would invite it to start again into the same leak.
|
|
63
|
+
*/
|
|
64
|
+
declare function observeMemory(guard: MemoryGuard, bytes: number): MemoryGuard;
|
|
65
|
+
/** A sentence a human can act on, or null while nothing is wrong. */
|
|
66
|
+
declare function describeTrip(guard: MemoryGuard): string | null;
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Per-tick synchronisation forensics.
|
|
70
|
+
*
|
|
71
|
+
* A single offset number cannot say *why* video is late, and optimising against
|
|
72
|
+
* it invites exactly the wrong fixes — presenting future frames, nudging the
|
|
73
|
+
* clock, delaying audio. Each of those improves the number and worsens the
|
|
74
|
+
* playback.
|
|
75
|
+
*
|
|
76
|
+
* So every tick is decomposed into causes that can be acted on separately:
|
|
77
|
+
*
|
|
78
|
+
* - **Frame quantisation** — video only changes every frame period, so the
|
|
79
|
+
* picture is always somewhat behind the audio instant. Bounded by one frame
|
|
80
|
+
* by definition, and not a defect.
|
|
81
|
+
* - **Decoder lateness** — the frame that *should* be showing had not been
|
|
82
|
+
* decoded yet. Fix by adjusting the bounded lead or queue depth.
|
|
83
|
+
* - **Scheduler lateness** — the right frame *was* available and was not
|
|
84
|
+
* selected. Fix the missed tick or the contention causing it.
|
|
85
|
+
* - **Clock correction** — how far each raw `AudioTrack` report moves the
|
|
86
|
+
* extrapolated position. Large jumps mean stale timestamps or a bad baseline.
|
|
87
|
+
*
|
|
88
|
+
* Pure, so the decomposition is tested rather than trusted.
|
|
89
|
+
*
|
|
90
|
+
* @experimental
|
|
91
|
+
*/
|
|
92
|
+
interface TickSample {
|
|
93
|
+
/** Authoritative extrapolated audio time. */
|
|
94
|
+
audioUs: number;
|
|
95
|
+
/** Age of the newest raw `getTimestamp()` report at this tick. */
|
|
96
|
+
rawAgeUs: number;
|
|
97
|
+
/** How far the newest raw report moved the extrapolated position. */
|
|
98
|
+
correctionUs: number;
|
|
99
|
+
/** Newest decoded frame at or before `audioUs`, or null if none is eligible. */
|
|
100
|
+
newestEligiblePtsUs: number | null;
|
|
101
|
+
/** What was actually presented, or null. */
|
|
102
|
+
presentedPtsUs: number | null;
|
|
103
|
+
/** Interval since the previous tick. */
|
|
104
|
+
tickIntervalUs: number;
|
|
105
|
+
queueDepth: number;
|
|
106
|
+
decoderLeadUs: number;
|
|
107
|
+
}
|
|
108
|
+
interface SyncTrace {
|
|
109
|
+
readonly frameUs: number;
|
|
110
|
+
quantisation: number[];
|
|
111
|
+
decoderLate: number[];
|
|
112
|
+
schedulerLate: number[];
|
|
113
|
+
correction: number[];
|
|
114
|
+
presentedOffset: number[];
|
|
115
|
+
tickInterval: number[];
|
|
116
|
+
ticks: number;
|
|
117
|
+
decoderLateTicks: number;
|
|
118
|
+
schedulerLateTicks: number;
|
|
119
|
+
}
|
|
120
|
+
declare function createSyncTrace(fps: number): SyncTrace;
|
|
121
|
+
/**
|
|
122
|
+
* The frame that ought to be on screen at `audioUs`.
|
|
123
|
+
*
|
|
124
|
+
* Assumes a regular frame grid anchored at zero, which the controlled fixture
|
|
125
|
+
* has. On a variable-rate source this is an approximation and the quantisation
|
|
126
|
+
* figure should be read as indicative — which is why it is stated here rather
|
|
127
|
+
* than left for a reader to infer from a clean-looking number.
|
|
128
|
+
*/
|
|
129
|
+
declare function expectedPtsAt(audioUs: number, frameUs: number): number;
|
|
130
|
+
declare function record(trace: SyncTrace, sample: TickSample): SyncTrace;
|
|
131
|
+
interface Distribution {
|
|
132
|
+
n: number;
|
|
133
|
+
p50: number;
|
|
134
|
+
p95: number;
|
|
135
|
+
p99: number;
|
|
136
|
+
max: number;
|
|
137
|
+
}
|
|
138
|
+
declare function distribution(values: readonly number[]): Distribution;
|
|
139
|
+
interface SyncReport {
|
|
140
|
+
quantisation: Distribution;
|
|
141
|
+
decoderLate: Distribution;
|
|
142
|
+
schedulerLate: Distribution;
|
|
143
|
+
correction: Distribution;
|
|
144
|
+
presentedOffset: Distribution;
|
|
145
|
+
tickInterval: Distribution;
|
|
146
|
+
ticks: number;
|
|
147
|
+
decoderLateTicks: number;
|
|
148
|
+
schedulerLateTicks: number;
|
|
149
|
+
frameUs: number;
|
|
150
|
+
/** Which cause dominates, so the fix is chosen by evidence. */
|
|
151
|
+
dominant: 'none' | 'decoder' | 'scheduler' | 'quantisation';
|
|
152
|
+
}
|
|
153
|
+
declare function syncReport(trace: SyncTrace): SyncReport;
|
|
154
|
+
|
|
155
|
+
/** @experimental */
|
|
156
|
+
declare const AndroidVideoPlayer: {
|
|
157
|
+
/**
|
|
158
|
+
* Publish the presentation surface host object on the JS runtime.
|
|
159
|
+
*
|
|
160
|
+
* It is captured into a worklet closure from there; the UI runtime has its
|
|
161
|
+
* own global and does not see this one.
|
|
162
|
+
*/
|
|
163
|
+
installVideoSurface(): boolean;
|
|
164
|
+
};
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* Playback backed by Media3's ExoPlayer.
|
|
168
|
+
*
|
|
169
|
+
* Media3 owns decoding, buffering, audio focus, synchronisation, seeking, speed
|
|
170
|
+
* and lifecycle. Pictor owns the document, the timeline and the Skia
|
|
171
|
+
* composition. This is the whole of the seam between them.
|
|
172
|
+
*
|
|
173
|
+
* **No Media3 or Android type appears here.** Positions are microseconds
|
|
174
|
+
* because that is Pictor's unit; the adapter converts.
|
|
175
|
+
*
|
|
176
|
+
* Video is rendered into the persistent external texture, so the renderer holds
|
|
177
|
+
* one `SkImage` for a whole session and nothing is imported per frame.
|
|
178
|
+
*
|
|
179
|
+
* @experimental
|
|
180
|
+
*/
|
|
181
|
+
interface Media3State {
|
|
182
|
+
positionUs: number;
|
|
183
|
+
durationUs: number;
|
|
184
|
+
playing: boolean;
|
|
185
|
+
buffering: boolean;
|
|
186
|
+
ready: boolean;
|
|
187
|
+
ended: boolean;
|
|
188
|
+
rate: number;
|
|
189
|
+
width: number;
|
|
190
|
+
height: number;
|
|
191
|
+
released: boolean;
|
|
192
|
+
error: string;
|
|
193
|
+
/** Bumped on seek, source change and release; pairs frames with a clip. */
|
|
194
|
+
generation: number;
|
|
195
|
+
/** Frame metadata awaiting a renderer. Bounded. */
|
|
196
|
+
queueDepth: number;
|
|
197
|
+
/** Metadata dropped because the queue was full. Should stay zero. */
|
|
198
|
+
overflowed: number;
|
|
199
|
+
/** Frames consumed with no sample to pair. Expected briefly after a seek. */
|
|
200
|
+
unpaired: number;
|
|
201
|
+
/** Samples discarded because their generation was superseded. */
|
|
202
|
+
stale: number;
|
|
203
|
+
queueHighWater: number;
|
|
204
|
+
/** Live threads in the process. Diagnostics for leak attribution. */
|
|
205
|
+
threads: number;
|
|
206
|
+
/** Allocated native heap, for the safety cutoff. */
|
|
207
|
+
nativeHeapBytes: number;
|
|
208
|
+
/** The anchor's own position, before extrapolation. Diagnostics. */
|
|
209
|
+
anchorPositionUs: number;
|
|
210
|
+
/** Frames the buffer queue dropped because the renderer was behind. */
|
|
211
|
+
coalesced: number;
|
|
212
|
+
/** How coarse `Player.currentPosition` is, measured rather than assumed. */
|
|
213
|
+
anchorPublications: number;
|
|
214
|
+
anchorChanges: number;
|
|
215
|
+
anchorMaxStepUs: number;
|
|
216
|
+
anchorMeanStepUs: number;
|
|
217
|
+
anchorMaxStaleNs: number;
|
|
218
|
+
/** Which playlist window is playing, and how many there are. */
|
|
219
|
+
windowIndex: number;
|
|
220
|
+
windowCount: number;
|
|
221
|
+
/** Offset inside the current window, in microseconds. */
|
|
222
|
+
windowPositionUs: number;
|
|
223
|
+
/** Window transitions so far — seams crossed. Diagnostics. */
|
|
224
|
+
transitions: number;
|
|
225
|
+
/**
|
|
226
|
+
* Frames whose latch failed and was retried on the next tick.
|
|
227
|
+
*
|
|
228
|
+
* `updateTexImage` returns quietly when Skia's GL context is not current on
|
|
229
|
+
* the calling thread, so this counts the retries rather than letting the
|
|
230
|
+
* frames disappear. A handful around startup and playlist changes is normal;
|
|
231
|
+
* a rate that tracks the frame rate is not.
|
|
232
|
+
*/
|
|
233
|
+
missedLatches: number;
|
|
234
|
+
/**
|
|
235
|
+
* What the single latch owner did, per frame.
|
|
236
|
+
*
|
|
237
|
+
* `skipped` counts ticks refused because the surface was retired or replaced
|
|
238
|
+
* — the invalid call, prevented rather than caught. `failed` must stay
|
|
239
|
+
* visible: a latch that quietly does nothing looks exactly like a paused
|
|
240
|
+
* video, which is how an earlier defect hid for a whole session.
|
|
241
|
+
*/
|
|
242
|
+
updatesAttempted: number;
|
|
243
|
+
updatesSucceeded: number;
|
|
244
|
+
updatesSkipped: number;
|
|
245
|
+
updatesFailed: number;
|
|
246
|
+
/**
|
|
247
|
+
* Times a requested rate exceeded the range and had to be clamped.
|
|
248
|
+
*
|
|
249
|
+
* Should stay zero: the UI only offers rates that compose within range. A
|
|
250
|
+
* non-zero value means the picture and the timeline are counting differently.
|
|
251
|
+
*/
|
|
252
|
+
rateConstrained: number;
|
|
253
|
+
/** The rate actually in force: clip speed times preview rate, clamped. */
|
|
254
|
+
effectiveRate: number;
|
|
255
|
+
/**
|
|
256
|
+
* The rate asked for, before clamping.
|
|
257
|
+
*
|
|
258
|
+
* Beside `effectiveRate` this says whether a ceiling was *selected* or merely
|
|
259
|
+
* reached: 10 requested and 10 effective is a legitimate extreme, while 25
|
|
260
|
+
* requested and 10 effective is a refusal.
|
|
261
|
+
*/
|
|
262
|
+
requestedRate: number;
|
|
263
|
+
/**
|
|
264
|
+
* Window indices that had no speed. Should stay zero.
|
|
265
|
+
*
|
|
266
|
+
* Non-zero means the playlist and the window index were read at different
|
|
267
|
+
* moments — the shape that left a cleared playlist running at the rate of the
|
|
268
|
+
* playlist before it.
|
|
269
|
+
*/
|
|
270
|
+
staleWindowSpeeds: number;
|
|
271
|
+
/** How many per-window speeds the player holds. Zero when idle. */
|
|
272
|
+
windowSpeedCount: number;
|
|
273
|
+
/**
|
|
274
|
+
* The two halves of frame delivery, counted separately.
|
|
275
|
+
*
|
|
276
|
+
* `framesOffered` is Media3 telling us a frame is about to be released;
|
|
277
|
+
* `framesAvailable` is the SurfaceTexture saying one arrived. If the first
|
|
278
|
+
* climbs while the second does not, the decoder is running but its output is
|
|
279
|
+
* not reaching our surface — which is a completely different fault from the
|
|
280
|
+
* renderer failing to pull.
|
|
281
|
+
*/
|
|
282
|
+
framesOffered: number;
|
|
283
|
+
framesAvailable: number;
|
|
284
|
+
/**
|
|
285
|
+
* Frames waiting to be latched, right now.
|
|
286
|
+
*
|
|
287
|
+
* A level rather than an edge, so it survives the consumer being disarmed.
|
|
288
|
+
* Usually 0 or 1; larger after a background, and it must fall to 0 within a
|
|
289
|
+
* tick of the consumer re-arming.
|
|
290
|
+
*/
|
|
291
|
+
pendingDepth: number;
|
|
292
|
+
/** Availability announced for a generation already retired. Should stay 0. */
|
|
293
|
+
framesRejected: number;
|
|
294
|
+
/**
|
|
295
|
+
* The generation the pending counters are keeping for.
|
|
296
|
+
*
|
|
297
|
+
* Must equal the consumer's generation. A frame pending for a different one
|
|
298
|
+
* is a frame nobody will ever take.
|
|
299
|
+
*/
|
|
300
|
+
pendingGeneration: number;
|
|
301
|
+
/** Producer identity, for proving which object a consumer is holding. */
|
|
302
|
+
controllerId: number;
|
|
303
|
+
pendingId: number;
|
|
304
|
+
orphaned: number;
|
|
305
|
+
/** `mime WxH rotN @fps` for the active item. Empty when there is no video. */
|
|
306
|
+
videoFormat: string;
|
|
307
|
+
/** The decoder Media3 selected, e.g. `c2.exynos.h264.decoder`. */
|
|
308
|
+
decoderName: string;
|
|
309
|
+
/**
|
|
310
|
+
* What the decoder was configured for, and what it produced.
|
|
311
|
+
*
|
|
312
|
+
* Read from the player's own events rather than inferred from the device: a
|
|
313
|
+
* mixed-resolution seam is the one moment input and output can disagree, and
|
|
314
|
+
* naming the decoder from the model instead of asking is how an attribution
|
|
315
|
+
* survives without ever being checked.
|
|
316
|
+
*/
|
|
317
|
+
inputFormats: number;
|
|
318
|
+
inputWidth: number;
|
|
319
|
+
inputHeight: number;
|
|
320
|
+
inputRotation: number;
|
|
321
|
+
/** Media3's decoder-reuse verdict across the last format change; -1 if none. */
|
|
322
|
+
decoderReuse: number;
|
|
323
|
+
outputSizes: number;
|
|
324
|
+
outputWidth: number;
|
|
325
|
+
outputHeight: number;
|
|
326
|
+
/** Increments on each decoder init — makes re-attachment visible. */
|
|
327
|
+
surfaceGeneration: number;
|
|
328
|
+
/**
|
|
329
|
+
* Seek bookkeeping for the cross-window freeze soak.
|
|
330
|
+
*
|
|
331
|
+
* `seeksRequested` counts calls; `seeksAccepted` counts those the player's
|
|
332
|
+
* own thread acted on. A gap between them, or a `pendingSeek` that stays
|
|
333
|
+
* true, is a seek that never reached the player — which is what the observed
|
|
334
|
+
* freeze would look like from this side.
|
|
335
|
+
*/
|
|
336
|
+
/**
|
|
337
|
+
* Surface attachment bookkeeping, for diagnostics only.
|
|
338
|
+
*
|
|
339
|
+
* The adapter attaches automatically the instant a surface becomes live, so a
|
|
340
|
+
* host never calls anything. These exist so the invariant can be *checked*:
|
|
341
|
+
* `attachedSurfaceGeneration` trailing `liveSurfaceGeneration` means the
|
|
342
|
+
* player is rendering into a surface nobody is watching.
|
|
343
|
+
*/
|
|
344
|
+
surfaceAttachments: number;
|
|
345
|
+
attachedSurfaceGeneration: number;
|
|
346
|
+
liveSurfaceGeneration: number;
|
|
347
|
+
seeksRequested: number;
|
|
348
|
+
seeksAccepted: number;
|
|
349
|
+
pendingSeek: boolean;
|
|
350
|
+
/** Media3's own playback state constant. 1 idle, 2 buffering, 3 ready, 4 ended. */
|
|
351
|
+
playbackState: number;
|
|
352
|
+
/** PTS of the last frame the renderer actually paired, or -1. */
|
|
353
|
+
lastPairedPtsUs: number;
|
|
354
|
+
/**
|
|
355
|
+
* Source failures reported by the player, not inferred from the state.
|
|
356
|
+
*
|
|
357
|
+
* A failed source otherwise looks exactly like an idle one — nothing playing,
|
|
358
|
+
* nothing buffering, no duration — so the count is what distinguishes "never
|
|
359
|
+
* started" from "started and broke".
|
|
360
|
+
*/
|
|
361
|
+
errorCount: number;
|
|
362
|
+
/** Media3's error code, and its name plus message. Empty when healthy. */
|
|
363
|
+
errorCode: number;
|
|
364
|
+
errorMessage: string;
|
|
365
|
+
/** Whether Media3 classifies the failure as worth retrying. */
|
|
366
|
+
errorRecoverable: boolean;
|
|
367
|
+
}
|
|
368
|
+
/** @experimental */
|
|
369
|
+
declare const Media3: {
|
|
370
|
+
/**
|
|
371
|
+
* Open a local clip and render it into the presentation surface.
|
|
372
|
+
*
|
|
373
|
+
* The surface must already exist: it is created on the UI runtime, where
|
|
374
|
+
* Skia's GL context lives, and preparing without one is an error rather than
|
|
375
|
+
* a silently black picture.
|
|
376
|
+
*/
|
|
377
|
+
prepare(uri: string): boolean;
|
|
378
|
+
/** Diagnostics: a player with no video output, for heap attribution. */
|
|
379
|
+
prepareNoSurface(uri: string): boolean;
|
|
380
|
+
/** Diagnostics: reuse the existing player for another source. */
|
|
381
|
+
rebind(uri: string): boolean;
|
|
382
|
+
/** Diagnostics: end the source, keep the player. */
|
|
383
|
+
stopKeepingPlayer(): void;
|
|
384
|
+
/**
|
|
385
|
+
* Replace the playlist from a project's playable windows.
|
|
386
|
+
*
|
|
387
|
+
* Takes the `PlaylistResult` straight from `@pictor-sdk/video-core` so the
|
|
388
|
+
* project → window mapping has exactly one definition, tested there as a
|
|
389
|
+
* round trip.
|
|
390
|
+
* The sources must already be URI strings; this adapter does not interpret a
|
|
391
|
+
* host's media descriptors, and a non-URI source is a host normalisation bug
|
|
392
|
+
* rather than something to guess at.
|
|
393
|
+
*
|
|
394
|
+
* The OES texture and the `SkImage` survive this. Changing the playlist is
|
|
395
|
+
* not a reason to rebuild what the renderer holds.
|
|
396
|
+
*/
|
|
397
|
+
setPlaylist(playlist: PlaylistResult, start?: {
|
|
398
|
+
windowIndex: number;
|
|
399
|
+
positionUs: number;
|
|
400
|
+
}, options?: {
|
|
401
|
+
allowDivergence?: boolean;
|
|
402
|
+
}): boolean;
|
|
403
|
+
/** Seek in the player's coordinates. Pair with `projectTimeToWindow`. */
|
|
404
|
+
seekToWindow(windowIndex: number, positionUs: number): void;
|
|
405
|
+
play(): void;
|
|
406
|
+
pause(): void;
|
|
407
|
+
/** Microseconds, like every other time in Pictor. */
|
|
408
|
+
seek(positionUs: number): void;
|
|
409
|
+
setRate(rate: number): void;
|
|
410
|
+
/**
|
|
411
|
+
* Try the current source again after a failure.
|
|
412
|
+
*
|
|
413
|
+
* This is Media3's own recovery path: it re-resolves the source and resumes
|
|
414
|
+
* from the current position. Worth calling when `errorRecoverable` is true —
|
|
415
|
+
* a network source that went away and came back — and pointless otherwise.
|
|
416
|
+
*/
|
|
417
|
+
retry(): void;
|
|
418
|
+
release(): void;
|
|
419
|
+
/** A snapshot for throttled reads. Never call this per frame. */
|
|
420
|
+
state(): Media3State;
|
|
421
|
+
};
|
|
422
|
+
|
|
423
|
+
/**
|
|
424
|
+
* Which playlist Media3 is holding, and against which surface.
|
|
425
|
+
*
|
|
426
|
+
* The bug this exists to prevent, in order:
|
|
427
|
+
*
|
|
428
|
+
* 1. The editor opens and builds surface A.
|
|
429
|
+
* 2. The system picker opens. The app backgrounds; A is no longer usable.
|
|
430
|
+
* 3. Two clips are added to the document while the picker is closing.
|
|
431
|
+
* 4. The editor returns and builds surface B.
|
|
432
|
+
* 5. Nothing prepares, because the only thing that ever triggered preparation
|
|
433
|
+
* was the *playlist changing*, and it changed in step 3 — when there was no
|
|
434
|
+
* surface to prepare against. `setPlaylist` was refused, the signature was
|
|
435
|
+
* recorded as applied anyway, and the editor sat there with two clips on the
|
|
436
|
+
* timeline and a player holding nothing.
|
|
437
|
+
*
|
|
438
|
+
* Two corrections follow from that. Readiness is **level-triggered**: the
|
|
439
|
+
* question is "does the playlist Media3 holds match the document, against the
|
|
440
|
+
* surface that is live *now*", asked whenever either side moves — not "did the
|
|
441
|
+
* playlist just change". And a refusal is **not** recorded as applied, so it is
|
|
442
|
+
* retried rather than remembered as done.
|
|
443
|
+
*
|
|
444
|
+
* A generation is what makes "the surface that is live now" expressible. Without
|
|
445
|
+
* it a caller can only know that *a* surface exists, which cannot distinguish
|
|
446
|
+
* the one it prepared against from the one that replaced it.
|
|
447
|
+
*
|
|
448
|
+
* Pure, so the sequence above is a test rather than a device run.
|
|
449
|
+
*
|
|
450
|
+
* @experimental
|
|
451
|
+
*/
|
|
452
|
+
/** No surface. Distinct from every real generation, which start at 1. */
|
|
453
|
+
declare const NO_SURFACE = 0;
|
|
454
|
+
/** @experimental */
|
|
455
|
+
interface PlaylistSession {
|
|
456
|
+
/** The playlist Media3 is holding, or null if it holds none. */
|
|
457
|
+
appliedSignature: string | null;
|
|
458
|
+
/** The surface generation it was prepared against. */
|
|
459
|
+
appliedGeneration: number;
|
|
460
|
+
/**
|
|
461
|
+
* A preparation was attempted and refused.
|
|
462
|
+
*
|
|
463
|
+
* Kept so the retry is driven by this rather than by hoping the document
|
|
464
|
+
* changes again. A user who has added their clips will not add them twice.
|
|
465
|
+
*/
|
|
466
|
+
pending: boolean;
|
|
467
|
+
}
|
|
468
|
+
/** @experimental */
|
|
469
|
+
declare function idlePlaylistSession(): PlaylistSession;
|
|
470
|
+
/** What the document and the device currently want. @experimental */
|
|
471
|
+
interface PlaylistIntent {
|
|
472
|
+
/** A stable description of the playlist the document implies. */
|
|
473
|
+
signature: string;
|
|
474
|
+
/** The live surface generation, or {@link NO_SURFACE}. */
|
|
475
|
+
generation: number;
|
|
476
|
+
}
|
|
477
|
+
/**
|
|
478
|
+
* Whether to call `setPlaylist` now.
|
|
479
|
+
*
|
|
480
|
+
* @experimental
|
|
481
|
+
*/
|
|
482
|
+
declare function shouldPrepare(session: PlaylistSession, intent: PlaylistIntent): boolean;
|
|
483
|
+
/**
|
|
484
|
+
* Record the outcome.
|
|
485
|
+
*
|
|
486
|
+
* A refusal leaves the session **unchanged except for `pending`**. Recording the
|
|
487
|
+
* signature on failure is precisely the mistake that made the editor wait for an
|
|
488
|
+
* edit that was never going to come.
|
|
489
|
+
*
|
|
490
|
+
* @experimental
|
|
491
|
+
*/
|
|
492
|
+
declare function afterPrepare(session: PlaylistSession, intent: PlaylistIntent, ok: boolean): PlaylistSession;
|
|
493
|
+
/**
|
|
494
|
+
* The surface went away — backgrounded, replaced, released.
|
|
495
|
+
*
|
|
496
|
+
* The signature is kept: it still describes what Media3 was asked to hold, and
|
|
497
|
+
* the comparison that matters next is against the *generation*, which is now
|
|
498
|
+
* guaranteed to differ from any live one.
|
|
499
|
+
*
|
|
500
|
+
* @experimental
|
|
501
|
+
*/
|
|
502
|
+
declare function retireSurface(session: PlaylistSession): PlaylistSession;
|
|
503
|
+
/**
|
|
504
|
+
* Whether a callback carrying `generation` is still worth acting on.
|
|
505
|
+
*
|
|
506
|
+
* A screen's teardown routinely runs after the next screen's setup, and an
|
|
507
|
+
* asynchronous surface creation can land after the surface it created has been
|
|
508
|
+
* replaced. Both arrive looking exactly like a valid result.
|
|
509
|
+
*
|
|
510
|
+
* @experimental
|
|
511
|
+
*/
|
|
512
|
+
declare function isCurrentGeneration(generation: number, live: number): boolean;
|
|
513
|
+
|
|
514
|
+
/**
|
|
515
|
+
* Who is allowed to create a presentation surface, and when.
|
|
516
|
+
*
|
|
517
|
+
* There is one external texture in the process, so there is one of these. Not
|
|
518
|
+
* one per hook instance: a hook is remounted by a navigation, a fast refresh or
|
|
519
|
+
* a rotation, and each remount would bring its own lock and its own idea of
|
|
520
|
+
* what is in flight. Two creates then overlap, each registers a generation, and
|
|
521
|
+
* the second overwrites the native handles the first is still holding — the GL
|
|
522
|
+
* texture and `SurfaceTexture` from the first are then unreachable and never
|
|
523
|
+
* released. The symptom is a hook reporting one generation while the adapter
|
|
524
|
+
* has moved on to another, with an abandoned surface in between.
|
|
525
|
+
*
|
|
526
|
+
* ## The generation is native's, and only native's
|
|
527
|
+
*
|
|
528
|
+
* `create()` returns the generation `VideoSurface.register` assigned. This
|
|
529
|
+
* module publishes that number unchanged and keeps no counter of its own — a
|
|
530
|
+
* second counter is a second answer to "which surface is live", and the whole
|
|
531
|
+
* class of bug here is two answers to one question.
|
|
532
|
+
*
|
|
533
|
+
* ## A superseded attempt is retired, not ignored
|
|
534
|
+
*
|
|
535
|
+
* An attempt whose owner went away while it was on the UI runtime still
|
|
536
|
+
* *succeeds*: the texture exists, the `Surface` is registered, Media3 could
|
|
537
|
+
* attach to it. Ignoring its report leaves exactly that — a live, registered
|
|
538
|
+
* surface nobody will ever draw or release. So it is released, on the runtime
|
|
539
|
+
* that owns the GL context, before anything else is started.
|
|
540
|
+
*
|
|
541
|
+
* ## Serialised, and retirement does not unlock
|
|
542
|
+
*
|
|
543
|
+
* A request that arrives while an attempt is unresolved is queued, and
|
|
544
|
+
* backgrounding or unmounting does **not** release the lock: the earlier
|
|
545
|
+
* attempt is still out there on the UI runtime and will still come back. The
|
|
546
|
+
* queue is one deep, because every waiting request wants the same thing.
|
|
547
|
+
*
|
|
548
|
+
* Effects are injected so the interleavings are deterministic in a test rather
|
|
549
|
+
* than a matter of timing on a device.
|
|
550
|
+
*
|
|
551
|
+
* @experimental
|
|
552
|
+
*/
|
|
553
|
+
/** What native `create()` reports. @experimental */
|
|
554
|
+
interface SurfaceCreateResult {
|
|
555
|
+
ok: boolean;
|
|
556
|
+
/** The generation `register` assigned, or 0 on failure. */
|
|
557
|
+
generation: number;
|
|
558
|
+
textureId: number;
|
|
559
|
+
error: string;
|
|
560
|
+
}
|
|
561
|
+
/** @experimental */
|
|
562
|
+
interface SurfaceOwnerEffects {
|
|
563
|
+
/**
|
|
564
|
+
* Schedule a create on the UI runtime.
|
|
565
|
+
*
|
|
566
|
+
* Must eventually call {@link SurfaceOwner.complete}, whatever happens.
|
|
567
|
+
*/
|
|
568
|
+
create(): void;
|
|
569
|
+
/**
|
|
570
|
+
* Release a specific generation, on the runtime that owns the GL context.
|
|
571
|
+
*
|
|
572
|
+
* Takes a generation rather than "the current surface" so a late attempt
|
|
573
|
+
* cannot release the surface that replaced it.
|
|
574
|
+
*/
|
|
575
|
+
retire(generation: number): void;
|
|
576
|
+
}
|
|
577
|
+
/** @experimental */
|
|
578
|
+
interface SurfaceReadiness {
|
|
579
|
+
/** The live generation, exactly as native reported it. 0 when there is none. */
|
|
580
|
+
generation: number;
|
|
581
|
+
/** The last failure, or empty. */
|
|
582
|
+
error: string;
|
|
583
|
+
}
|
|
584
|
+
/** @experimental */
|
|
585
|
+
interface SurfaceOwner {
|
|
586
|
+
readonly readiness: SurfaceReadiness;
|
|
587
|
+
/** Whether an attempt is on the UI runtime right now. */
|
|
588
|
+
readonly busy: boolean;
|
|
589
|
+
/** Generations created by a superseded attempt and released. Diagnostics. */
|
|
590
|
+
readonly retired: number;
|
|
591
|
+
/**
|
|
592
|
+
* Ask for a surface on behalf of `token`.
|
|
593
|
+
*
|
|
594
|
+
* Returns false when the request was queued or refused rather than started.
|
|
595
|
+
*/
|
|
596
|
+
request(token: number): boolean;
|
|
597
|
+
/** That owner is gone. Does not unlock: an unresolved attempt is still out there. */
|
|
598
|
+
retireOwner(token: number): void;
|
|
599
|
+
/** An attempt finished. */
|
|
600
|
+
complete(result: SurfaceCreateResult): void;
|
|
601
|
+
/** Adopt a surface that already exists, without creating one. */
|
|
602
|
+
adopt(generation: number, token: number): void;
|
|
603
|
+
subscribe(listener: () => void): () => void;
|
|
604
|
+
/** Point the owner at the newest caller's effects. */
|
|
605
|
+
useEffects(next: SurfaceOwnerEffects): void;
|
|
606
|
+
}
|
|
607
|
+
/** @experimental */
|
|
608
|
+
declare function createSurfaceOwner(initial: SurfaceOwnerEffects): SurfaceOwner;
|
|
609
|
+
/** @experimental */
|
|
610
|
+
declare function surfaceOwner(effects: SurfaceOwnerEffects): SurfaceOwner;
|
|
611
|
+
/** @experimental */
|
|
612
|
+
declare function resetSurfaceOwnerForTests(): void;
|
|
613
|
+
/** @experimental */
|
|
614
|
+
declare function nextOwnerToken(): number;
|
|
615
|
+
|
|
616
|
+
/**
|
|
617
|
+
* The one stable slot, installed on first ask.
|
|
618
|
+
*
|
|
619
|
+
* Synchronous, and callable during a render, because the consumer has to be able
|
|
620
|
+
* to capture it *before* any surface exists. An effect is too late: the frame
|
|
621
|
+
* callback is built on the first render, and anything it captures then is what
|
|
622
|
+
* it keeps.
|
|
623
|
+
*
|
|
624
|
+
* The object never changes. `create` fills its target in place and retirement
|
|
625
|
+
* clears it, so every call through it resolves to whatever the slot points at
|
|
626
|
+
* now. There is deliberately nothing replaceable to hold.
|
|
627
|
+
*
|
|
628
|
+
* @experimental
|
|
629
|
+
*/
|
|
630
|
+
declare function videoSurfaceSlot(): VideoSurfaceHandle | undefined;
|
|
631
|
+
/** @experimental */
|
|
632
|
+
interface VideoSurfaceOptions {
|
|
633
|
+
/** Backing size of the texture. Frames are scaled into it. */
|
|
634
|
+
width: number;
|
|
635
|
+
height: number;
|
|
636
|
+
enabled?: boolean;
|
|
637
|
+
}
|
|
638
|
+
/**
|
|
639
|
+
* The surface as the UI runtime sees it.
|
|
640
|
+
*
|
|
641
|
+
* Structural rather than an imported Android type: nothing platform-specific
|
|
642
|
+
* reaches Pictor's public API.
|
|
643
|
+
*
|
|
644
|
+
* @experimental
|
|
645
|
+
*/
|
|
646
|
+
interface VideoSurfaceHandle {
|
|
647
|
+
create(width: number, height: number): {
|
|
648
|
+
textureId: number;
|
|
649
|
+
ok: boolean;
|
|
650
|
+
generation: number;
|
|
651
|
+
error?: string;
|
|
652
|
+
};
|
|
653
|
+
generation(): number;
|
|
654
|
+
alive(): boolean;
|
|
655
|
+
/** Release a named generation. Does nothing if it is not the one held. */
|
|
656
|
+
releaseGeneration(generation: number): boolean;
|
|
657
|
+
}
|
|
658
|
+
/** @experimental */
|
|
659
|
+
interface VideoSurface {
|
|
660
|
+
/**
|
|
661
|
+
* The stable slot.
|
|
662
|
+
*
|
|
663
|
+
* Available from the first render and never replaced, so a consumer captures
|
|
664
|
+
* it once and reads through it. It answers `generation() === 0` while empty,
|
|
665
|
+
* which is how a renderer knows there is nothing to draw — a null would have
|
|
666
|
+
* to be captured, and a captured null is what it stays.
|
|
667
|
+
*/
|
|
668
|
+
surface: unknown | undefined;
|
|
669
|
+
/**
|
|
670
|
+
* The live generation, exactly as native reported it. 0 when there is none.
|
|
671
|
+
*
|
|
672
|
+
* Feed to `useMedia3Playback`. Every change is a reason to reconcile the
|
|
673
|
+
* playlist, including the change back to 0.
|
|
674
|
+
*/
|
|
675
|
+
generation: number;
|
|
676
|
+
/** Failure to install or create, for a host's error slot. Empty when healthy. */
|
|
677
|
+
error: string;
|
|
678
|
+
}
|
|
679
|
+
/** @experimental */
|
|
680
|
+
declare function useVideoSurface(options: VideoSurfaceOptions): VideoSurface;
|
|
681
|
+
|
|
682
|
+
/**
|
|
683
|
+
* Who owns playback, and until when.
|
|
684
|
+
*
|
|
685
|
+
* Media3's player is a **process-wide singleton**. Nothing about it ends when
|
|
686
|
+
* the screen that started it goes away, so on a Pixel 7 leaving the editor while
|
|
687
|
+
* playing left the audio running, and backgrounding the app did the same: the
|
|
688
|
+
* picture stopped because its surface was retired, and the sound carried on
|
|
689
|
+
* because nobody had told the player anything.
|
|
690
|
+
*
|
|
691
|
+
* A singleton with no owner is the whole problem, so this gives it one. A screen
|
|
692
|
+
* takes ownership when it mounts and gives it back exactly once — and a screen
|
|
693
|
+
* that mounts afterwards starts from nothing rather than inheriting whatever the
|
|
694
|
+
* last one was doing.
|
|
695
|
+
*
|
|
696
|
+
* ## Two kinds of stopping, deliberately different
|
|
697
|
+
*
|
|
698
|
+
* **Suspend** is temporary: the app went to the background, or the picker came
|
|
699
|
+
* up over the editor. Playback pauses and the player is kept, because the screen
|
|
700
|
+
* is coming back and rebuilding a player it never lost would be a stall the user
|
|
701
|
+
* did nothing to earn.
|
|
702
|
+
*
|
|
703
|
+
* **Teardown** is final: the screen unmounted, or navigation left it. The player
|
|
704
|
+
* is paused *and then* released, in that order, so nothing is audible in the
|
|
705
|
+
* gap. After a teardown this object does nothing at all — a late suspend from a
|
|
706
|
+
* listener that has not been removed yet must not resurrect anything.
|
|
707
|
+
*
|
|
708
|
+
* Neither is a reason to touch the document. Playback stopping says nothing
|
|
709
|
+
* about what the project is or where its playhead should be, and clearing either
|
|
710
|
+
* would turn leaving a screen into an edit.
|
|
711
|
+
*
|
|
712
|
+
* ## Epochs
|
|
713
|
+
*
|
|
714
|
+
* Every stop bumps an epoch, and callbacks carry the epoch they were scheduled
|
|
715
|
+
* under. A poll that fires after the screen backgrounded still runs — clearing
|
|
716
|
+
* an interval is not instantaneous — and `accepts` is how it finds out that the
|
|
717
|
+
* answer it is about to deliver is no longer wanted.
|
|
718
|
+
*
|
|
719
|
+
* @experimental
|
|
720
|
+
*/
|
|
721
|
+
/** Why playback stopped. Recorded so a log can say which path ran. @experimental */
|
|
722
|
+
type PlaybackStopReason = 'unmount' | 'background' | 'blur';
|
|
723
|
+
/** @experimental */
|
|
724
|
+
interface PlaybackLifecycleEffects {
|
|
725
|
+
/** Stop the sound. Must be safe to call when already paused. */
|
|
726
|
+
pause(): void;
|
|
727
|
+
/** Give up the native player, and with it audio focus. */
|
|
728
|
+
release(): void;
|
|
729
|
+
}
|
|
730
|
+
/** @experimental */
|
|
731
|
+
interface PlaybackLifecycle {
|
|
732
|
+
/** Whether this screen currently owns running playback. */
|
|
733
|
+
readonly active: boolean;
|
|
734
|
+
/** Whether the player has been given back. Final. */
|
|
735
|
+
readonly released: boolean;
|
|
736
|
+
/** Why it last stopped, or `null`. */
|
|
737
|
+
readonly reason: PlaybackStopReason | null;
|
|
738
|
+
/** Bumped by every stop; callbacks carry the value they were scheduled under. */
|
|
739
|
+
readonly epoch: number;
|
|
740
|
+
/** Whether work scheduled under `epoch` may still act. */
|
|
741
|
+
accepts(epoch: number): boolean;
|
|
742
|
+
/** Pause and keep the player. Idempotent within one suspension. */
|
|
743
|
+
suspend(reason: PlaybackStopReason): void;
|
|
744
|
+
/** Become active again. Idempotent; ignored after teardown. */
|
|
745
|
+
resume(): void;
|
|
746
|
+
/** Pause, then release, exactly once, forever. */
|
|
747
|
+
teardown(reason: PlaybackStopReason): void;
|
|
748
|
+
}
|
|
749
|
+
/** @experimental */
|
|
750
|
+
declare function createPlaybackLifecycle(effects: PlaybackLifecycleEffects): PlaybackLifecycle;
|
|
751
|
+
|
|
752
|
+
/**
|
|
753
|
+
* Deciding that a restored surface is not actually showing anything.
|
|
754
|
+
*
|
|
755
|
+
* After a background, a picker, or any other trip away from the editor, the
|
|
756
|
+
* player and the surface have to find each other again. When that handshake
|
|
757
|
+
* half-succeeds the result is the worst kind of failure: audio plays, the clock
|
|
758
|
+
* advances, no error is reported, and the picture is black. Nothing in the
|
|
759
|
+
* player's own state says anything is wrong, because from its point of view
|
|
760
|
+
* nothing is.
|
|
761
|
+
*
|
|
762
|
+
* So it is decided from the counters instead, and the counters are already
|
|
763
|
+
* separated into the four things that can go wrong — see `Media3State`, where
|
|
764
|
+
* `framesOffered` is the decoder saying a frame is coming and `framesAvailable`
|
|
765
|
+
* is the surface saying one arrived.
|
|
766
|
+
*
|
|
767
|
+
* The point of naming the branch is that the repairs differ. Re-running the
|
|
768
|
+
* handshake fixes a stale generation and does nothing at all for a consumer
|
|
769
|
+
* that has stopped pulling, and "retry everything" would have made both look
|
|
770
|
+
* like the same intermittent bug.
|
|
771
|
+
*
|
|
772
|
+
* @experimental
|
|
773
|
+
*/
|
|
774
|
+
/** How long a stall must persist before it counts. @experimental */
|
|
775
|
+
declare const RECOVERY_GRACE_MS = 1500;
|
|
776
|
+
/** What a sample of the pipeline looked like at one instant. @experimental */
|
|
777
|
+
interface RecoverySample {
|
|
778
|
+
atMs: number;
|
|
779
|
+
/** The player's own clock. Advancing means audio is being produced. */
|
|
780
|
+
positionUs: number;
|
|
781
|
+
/** Media3 announcing frames. */
|
|
782
|
+
framesOffered: number;
|
|
783
|
+
/** The SurfaceTexture receiving them. */
|
|
784
|
+
framesAvailable: number;
|
|
785
|
+
/** Frames the consumer actually took. */
|
|
786
|
+
framesLatched: number;
|
|
787
|
+
/** Announcements for a generation already retired. */
|
|
788
|
+
framesRejected: number;
|
|
789
|
+
/** The generation the producer is filling. */
|
|
790
|
+
pendingGeneration: number;
|
|
791
|
+
/** The generation the consumer is watching. */
|
|
792
|
+
consumerGeneration: number;
|
|
793
|
+
}
|
|
794
|
+
/**
|
|
795
|
+
* Which half of the pipeline is broken, or `null` when it is healthy.
|
|
796
|
+
*
|
|
797
|
+
* @experimental
|
|
798
|
+
*/
|
|
799
|
+
type StallKind =
|
|
800
|
+
/** Frames arrive and nobody takes them: the consumer or its frame loop. */
|
|
801
|
+
'consumer'
|
|
802
|
+
/** Producer and consumer disagree about which surface: a stale handshake. */
|
|
803
|
+
| 'handshake'
|
|
804
|
+
/** The player is running but nothing reaches the surface: attachment. */
|
|
805
|
+
| 'attachment'
|
|
806
|
+
/** Frames are latched and the picture is still wrong: scene or image. */
|
|
807
|
+
| 'resource';
|
|
808
|
+
/**
|
|
809
|
+
* Name the branch from two samples.
|
|
810
|
+
*
|
|
811
|
+
* `null` means nothing is wrong *that this can see* — including the ordinary
|
|
812
|
+
* case of a player that is simply not advancing, which is what a paused editor
|
|
813
|
+
* looks like and is not a fault.
|
|
814
|
+
*
|
|
815
|
+
* @experimental
|
|
816
|
+
*/
|
|
817
|
+
declare function classifyStall(since: RecoverySample, now: RecoverySample): StallKind | null;
|
|
818
|
+
/**
|
|
819
|
+
* Whether the stall has lasted long enough to act on.
|
|
820
|
+
*
|
|
821
|
+
* A grace period because none of this is instantaneous: a surface is recreated,
|
|
822
|
+
* a playlist is reconciled and the first frame decoded, and a check that fired
|
|
823
|
+
* inside that window would recover a pipeline that was about to work.
|
|
824
|
+
*
|
|
825
|
+
* @experimental
|
|
826
|
+
*/
|
|
827
|
+
declare function shouldRecover(since: RecoverySample, now: RecoverySample, graceMs?: number): boolean;
|
|
828
|
+
|
|
829
|
+
/**
|
|
830
|
+
* Keep a Media3 playlist in step with a project, and the playhead with playback.
|
|
831
|
+
*
|
|
832
|
+
* Two directions, deliberately separated:
|
|
833
|
+
*
|
|
834
|
+
* - **Project → player.** When the clips, their order or their trims change,
|
|
835
|
+
* the playlist is replaced. The current *project time* is preserved across
|
|
836
|
+
* that, not the window index — deleting the second of four clips must not
|
|
837
|
+
* leave the playhead in whatever the second window now happens to be.
|
|
838
|
+
* - **Player → playhead.** While playing, the player's position is polled at a
|
|
839
|
+
* readable rate and mapped back to project time. This is a **throttled
|
|
840
|
+
* semantic update**, not a per-frame one: the picture is composed on the UI
|
|
841
|
+
* runtime from the external texture and never waits for React.
|
|
842
|
+
*
|
|
843
|
+
* The playlist is only replaced when it genuinely differs. A trim drag
|
|
844
|
+
* dispatches a command per finger sample, and rebuilding the playlist on each
|
|
845
|
+
* would restart decoding many times a second.
|
|
846
|
+
*
|
|
847
|
+
* @experimental
|
|
848
|
+
*/
|
|
849
|
+
/** @experimental */
|
|
850
|
+
interface UseMedia3PlaybackOptions {
|
|
851
|
+
project: VideoProject;
|
|
852
|
+
/** Timeline time the host wants shown. */
|
|
853
|
+
playheadUs: number;
|
|
854
|
+
/** Called with the project time as playback advances. Throttled. */
|
|
855
|
+
onPositionUs?: (timeUs: number) => void;
|
|
856
|
+
/** How often to poll the player, in milliseconds. */
|
|
857
|
+
pollMs?: number;
|
|
858
|
+
/**
|
|
859
|
+
* The live presentation surface generation, from `useVideoSurface`.
|
|
860
|
+
*
|
|
861
|
+
* Not a boolean. A playlist prepared against a retired surface renders into
|
|
862
|
+
* something nobody is watching — the player decodes, releases frames and
|
|
863
|
+
* reports no error — so the question is *which* surface, not whether one
|
|
864
|
+
* exists. Every change is a reason to reconcile, including the change to 0.
|
|
865
|
+
*/
|
|
866
|
+
surfaceGeneration?: number;
|
|
867
|
+
enabled?: boolean;
|
|
868
|
+
}
|
|
869
|
+
/** @experimental */
|
|
870
|
+
interface Media3Playback {
|
|
871
|
+
playing: boolean;
|
|
872
|
+
/**
|
|
873
|
+
* Start or resume playback, replaying from zero when there is nothing left.
|
|
874
|
+
*
|
|
875
|
+
* Returns what it decided, so a host can show it or count it. See
|
|
876
|
+
* `playIntent` in `@pictor-sdk/video-core` for why the end is a special case.
|
|
877
|
+
*/
|
|
878
|
+
play(): PlayIntent;
|
|
879
|
+
pause(): void;
|
|
880
|
+
setRate(rate: number): void;
|
|
881
|
+
/** Seek in project time. Mapped to the player's own coordinates. */
|
|
882
|
+
seek(timeUs: number): void;
|
|
883
|
+
state: Media3State | null;
|
|
884
|
+
/** The windows currently loaded, for diagnostics and gates. */
|
|
885
|
+
items: readonly PlaylistItem[];
|
|
886
|
+
/**
|
|
887
|
+
* Clips the player cannot open. Non-empty means playback is **not** loaded.
|
|
888
|
+
*
|
|
889
|
+
* Surfaced rather than swallowed: the alternative is an editor that plays a
|
|
890
|
+
* shorter timeline than it draws, which reads as a synchronisation bug.
|
|
891
|
+
*/
|
|
892
|
+
unsupported: readonly UnsupportedSource[];
|
|
893
|
+
/**
|
|
894
|
+
* Which half of the pipeline stalled after a surface restoration, or `null`.
|
|
895
|
+
*
|
|
896
|
+
* Non-null means a black picture was detected and the handshake re-run — a
|
|
897
|
+
* host should surface it rather than show a silent audio-only result.
|
|
898
|
+
*/
|
|
899
|
+
stall: StallKind | null;
|
|
900
|
+
/**
|
|
901
|
+
* This screen's ownership of the shared player, for diagnostics and gates.
|
|
902
|
+
*
|
|
903
|
+
* Exposed so a host can log which path stopped playback and assert that a
|
|
904
|
+
* screen released exactly once.
|
|
905
|
+
*/
|
|
906
|
+
lifecycle: PlaybackLifecycle;
|
|
907
|
+
/**
|
|
908
|
+
* How far the timeline and the player disagree, in microseconds. Zero when
|
|
909
|
+
* everything is playable, and asserted by the device gates.
|
|
910
|
+
*/
|
|
911
|
+
divergenceUs: number;
|
|
912
|
+
/**
|
|
913
|
+
* The player's position in project time, read now, without rendering.
|
|
914
|
+
*
|
|
915
|
+
* For a host driving a UI-thread playhead: it samples this a few times a
|
|
916
|
+
* second and advances between samples itself. Returns `null` when the player
|
|
917
|
+
* has no position to give — before a playlist is loaded, or after release.
|
|
918
|
+
*/
|
|
919
|
+
positionUs(): number | null;
|
|
920
|
+
/**
|
|
921
|
+
* The rate actually in force: clip speed and preview rate, composed and
|
|
922
|
+
* clamped by the adapter. A host showing a rate should show this one.
|
|
923
|
+
*/
|
|
924
|
+
effectiveRate: number;
|
|
925
|
+
/**
|
|
926
|
+
* The last seek this hook actually issued.
|
|
927
|
+
*
|
|
928
|
+
* Global time in, and the `{windowIndex, localUs}` it was converted to. A
|
|
929
|
+
* player reporting an unexpected position is either being sent the wrong
|
|
930
|
+
* thing or mishandling the right thing, and those need different fixes — this
|
|
931
|
+
* is what tells them apart instead of leaving it to arithmetic on a
|
|
932
|
+
* screenshot.
|
|
933
|
+
*/
|
|
934
|
+
lastSeek: {
|
|
935
|
+
globalUs: number;
|
|
936
|
+
windowIndex: number;
|
|
937
|
+
localUs: number;
|
|
938
|
+
} | null;
|
|
939
|
+
}
|
|
940
|
+
/** @experimental */
|
|
941
|
+
declare function useMedia3Playback(options: UseMedia3PlaybackOptions): Media3Playback;
|
|
942
|
+
|
|
943
|
+
/**
|
|
944
|
+
* The preview, on one clock.
|
|
945
|
+
*
|
|
946
|
+
* ## What this replaces
|
|
947
|
+
*
|
|
948
|
+
* Two players kept in step: a video player and a separate player for the
|
|
949
|
+
* prepared audio mix, with a drift check correcting between them. That
|
|
950
|
+
* architecture was measured to its limit and could not hold synchronisation
|
|
951
|
+
* across a clip seam — the correction is a seek, a seek costs a decode, and a
|
|
952
|
+
* decode at the moment of a seam is what the correction was trying to cover.
|
|
953
|
+
*
|
|
954
|
+
* A `CompositionPlayer` has one clock, one playback thread and one audio graph.
|
|
955
|
+
* The prepared mix becomes an audio-only sequence beside the video sequence, so
|
|
956
|
+
* the two cannot drift: there is nothing to drift against.
|
|
957
|
+
*
|
|
958
|
+
* ## What it does not fix
|
|
959
|
+
*
|
|
960
|
+
* The clip seam still costs a picture. Media3 tears down the video decoder at
|
|
961
|
+
* an item boundary and builds another, and for 119 to 168 ms nothing reaches
|
|
962
|
+
* the renderer while the clock runs on — the audio is continuous and the
|
|
963
|
+
* picture holds. That behaviour was watched on a device and accepted for v1;
|
|
964
|
+
* see `docs/research/preview-acceptance.md` for what was accepted and what was
|
|
965
|
+
* explicitly not.
|
|
966
|
+
*
|
|
967
|
+
* ## Why the same contract
|
|
968
|
+
*
|
|
969
|
+
* It returns {@link Media3Playback}, unchanged. The editor, the transport and
|
|
970
|
+
* the gates were all written against it, and swapping the engine underneath is
|
|
971
|
+
* a smaller and more reviewable change than rewriting every caller to a shape
|
|
972
|
+
* that would end up equivalent.
|
|
973
|
+
*
|
|
974
|
+
* @experimental
|
|
975
|
+
*/
|
|
976
|
+
interface UseCompositionPlaybackOptions {
|
|
977
|
+
project: VideoProject;
|
|
978
|
+
/** Timeline time the host wants shown. */
|
|
979
|
+
playheadUs: number;
|
|
980
|
+
/**
|
|
981
|
+
* The prepared audio program, from `usePreviewAudio`.
|
|
982
|
+
*
|
|
983
|
+
* Null while it is being prepared, after it has failed, and — legitimately —
|
|
984
|
+
* when the document has no audio at all.
|
|
985
|
+
*
|
|
986
|
+
* Those are different situations and the hook has to tell them apart. A
|
|
987
|
+
* document *with* audio waits: building the composition without its sound
|
|
988
|
+
* would play a silent preview and gain the music a second later, which is two
|
|
989
|
+
* timelines to anyone watching. A document *without* audio has nothing to
|
|
990
|
+
* wait for, and refusing to build would leave the picture dead.
|
|
991
|
+
*/
|
|
992
|
+
mix: {
|
|
993
|
+
uri: string;
|
|
994
|
+
durationUs: number;
|
|
995
|
+
} | null;
|
|
996
|
+
/** Turns a document source into something the player can open. */
|
|
997
|
+
resolveSource: (source: {
|
|
998
|
+
kind: string;
|
|
999
|
+
uri?: string;
|
|
1000
|
+
}) => string | null;
|
|
1001
|
+
/** @see UseMedia3PlaybackOptions.surfaceGeneration */
|
|
1002
|
+
surfaceGeneration?: number;
|
|
1003
|
+
enabled?: boolean;
|
|
1004
|
+
}
|
|
1005
|
+
/**
|
|
1006
|
+
* What a composition knows about itself.
|
|
1007
|
+
*
|
|
1008
|
+
* Deliberately not `Media3State`. That type is the shape the video player's
|
|
1009
|
+
* native state comes back in — forty fields about queue depth, anchors, native
|
|
1010
|
+
* heap and window indices — and a composition has none of them. Synthesising
|
|
1011
|
+
* one would be inventing forty numbers to satisfy a type, and a reader would
|
|
1012
|
+
* have no way to tell the invented ones from the measured ones.
|
|
1013
|
+
*
|
|
1014
|
+
* @experimental
|
|
1015
|
+
*/
|
|
1016
|
+
interface CompositionPlaybackState {
|
|
1017
|
+
playing: boolean;
|
|
1018
|
+
ready: boolean;
|
|
1019
|
+
buffering: boolean;
|
|
1020
|
+
ended: boolean;
|
|
1021
|
+
released: boolean;
|
|
1022
|
+
/** Project time, which for a single composition is the only time there is. */
|
|
1023
|
+
positionUs: number;
|
|
1024
|
+
durationUs: number;
|
|
1025
|
+
/** The presentation size, or 0 while nothing has been measured. */
|
|
1026
|
+
width: number;
|
|
1027
|
+
height: number;
|
|
1028
|
+
error: string | null;
|
|
1029
|
+
}
|
|
1030
|
+
/** @experimental */
|
|
1031
|
+
type CompositionPlayback = Omit<Media3Playback, 'state'> & {
|
|
1032
|
+
state: CompositionPlaybackState | null;
|
|
1033
|
+
};
|
|
1034
|
+
/** @experimental */
|
|
1035
|
+
declare function useCompositionPlayback(options: UseCompositionPlaybackOptions): CompositionPlayback;
|
|
1036
|
+
|
|
1037
|
+
/**
|
|
1038
|
+
* `ThumbnailProvider` over Media3's retriever, with the core policy on top.
|
|
1039
|
+
*
|
|
1040
|
+
* The division is the point: `@pictor-sdk/video-core` owns deduplication, the
|
|
1041
|
+
* bounded cache and visible-range ordering, all of which are pure; this file
|
|
1042
|
+
* owns the part that needs a codec. A host that writes its own provider — for a
|
|
1043
|
+
* remote thumbnail service, say — reuses the policy and replaces only this.
|
|
1044
|
+
*
|
|
1045
|
+
* Results arrive as base64 JPEG rather than as a native image handle. A strip
|
|
1046
|
+
* thumbnail is a few kilobytes and it is drawn by React Native's own image
|
|
1047
|
+
* pipeline, so there is nothing here to leak and no ownership to transfer. The
|
|
1048
|
+
* `AHardwareBuffer` lesson applies to full-resolution frames at 30 per second,
|
|
1049
|
+
* not to a 64-pixel square that changes when the user scrolls.
|
|
1050
|
+
*
|
|
1051
|
+
* @experimental
|
|
1052
|
+
*/
|
|
1053
|
+
/** @experimental */
|
|
1054
|
+
interface AndroidThumbnailOptions {
|
|
1055
|
+
/**
|
|
1056
|
+
* How many decoded thumbnails to keep.
|
|
1057
|
+
*
|
|
1058
|
+
* 240 covers several screens of a dense strip at 64 px — enough that
|
|
1059
|
+
* scrolling back does not re-decode, bounded enough that a long project
|
|
1060
|
+
* cannot grow without limit.
|
|
1061
|
+
*/
|
|
1062
|
+
capacity?: number;
|
|
1063
|
+
/** How often to check whether a decode has finished, in milliseconds. */
|
|
1064
|
+
pollMs?: number;
|
|
1065
|
+
}
|
|
1066
|
+
/** @experimental */
|
|
1067
|
+
declare function createAndroidThumbnailProvider(options?: AndroidThumbnailOptions): ThumbnailProvider;
|
|
1068
|
+
/** Decode counters, for gates and diagnostics. @experimental */
|
|
1069
|
+
declare function thumbnailCounters(): {
|
|
1070
|
+
decoded: number;
|
|
1071
|
+
cancelled: number;
|
|
1072
|
+
failed: number;
|
|
1073
|
+
running: number;
|
|
1074
|
+
retrievers: number;
|
|
1075
|
+
};
|
|
1076
|
+
|
|
1077
|
+
/**
|
|
1078
|
+
* Deciding when playback has settled, instead of assuming it after a timeout.
|
|
1079
|
+
*
|
|
1080
|
+
* The first attempt excluded a fixed 1.5 s after every transition. That is a
|
|
1081
|
+
* guess in both directions: too short for a seek, whose tail leaked into the
|
|
1082
|
+
* steady figures, and too long for a rate change that settles immediately. It
|
|
1083
|
+
* also hides the thing worth knowing — how long recovery actually took.
|
|
1084
|
+
*
|
|
1085
|
+
* So a transition opens a phase, and the phase closes when the offsets say so:
|
|
1086
|
+
* [CONVERGED_RUN] consecutive samples inside the expected bound. The recovery
|
|
1087
|
+
* duration and the worst offset seen on the way are kept, because a transient
|
|
1088
|
+
* that is excluded but not reported is a failure nobody can see.
|
|
1089
|
+
*
|
|
1090
|
+
* [MAX_RECOVERY_SAMPLES] is a safety net, not a policy. A phase that never
|
|
1091
|
+
* converges is closed anyway and marked, so a permanent failure surfaces as a
|
|
1092
|
+
* failure instead of an ever-growing exclusion window.
|
|
1093
|
+
*
|
|
1094
|
+
* Pure, so the rule is tested rather than trusted.
|
|
1095
|
+
*
|
|
1096
|
+
* @experimental
|
|
1097
|
+
*/
|
|
1098
|
+
/** What kind of transition opened a phase. Steady is the absence of one. */
|
|
1099
|
+
type Phase = 'steady' | 'startup' | 'seek' | 'source' | 'pauseResume' | 'rateChange';
|
|
1100
|
+
/**
|
|
1101
|
+
* Consecutive in-bound samples that end a transient.
|
|
1102
|
+
*
|
|
1103
|
+
* Eight at 30 fps is about a quarter second of agreement — long enough that a
|
|
1104
|
+
* single lucky frame cannot close a phase, short enough to measure recovery
|
|
1105
|
+
* rather than round it up.
|
|
1106
|
+
*/
|
|
1107
|
+
declare const CONVERGED_RUN = 8;
|
|
1108
|
+
/** A phase is closed after this many samples whatever they say. */
|
|
1109
|
+
declare const MAX_RECOVERY_SAMPLES = 600;
|
|
1110
|
+
interface PhaseState {
|
|
1111
|
+
phase: Phase;
|
|
1112
|
+
/** Offsets collected while this phase was open. */
|
|
1113
|
+
samples: number[];
|
|
1114
|
+
/** Consecutive in-bound samples so far. */
|
|
1115
|
+
run: number;
|
|
1116
|
+
/** Worst absolute offset seen during the transient, in microseconds. */
|
|
1117
|
+
worstUs: number;
|
|
1118
|
+
/** Samples taken before it converged. */
|
|
1119
|
+
recoverySamples: number;
|
|
1120
|
+
/** True once converged; further samples belong to steady state. */
|
|
1121
|
+
converged: boolean;
|
|
1122
|
+
/** True when the safety net closed it instead of convergence. */
|
|
1123
|
+
timedOut: boolean;
|
|
1124
|
+
}
|
|
1125
|
+
declare function beginPhase(phase: Phase): PhaseState;
|
|
1126
|
+
/**
|
|
1127
|
+
* Feed one offset. Returns the updated phase and where the sample belongs.
|
|
1128
|
+
*
|
|
1129
|
+
* A sample taken while a transient is open belongs to that transient, never to
|
|
1130
|
+
* steady state — merging them would report the transitions as if they were
|
|
1131
|
+
* playback.
|
|
1132
|
+
*/
|
|
1133
|
+
declare function observe(state: PhaseState, offsetUs: number, boundUs: number): {
|
|
1134
|
+
state: PhaseState;
|
|
1135
|
+
bucket: Phase;
|
|
1136
|
+
};
|
|
1137
|
+
/** How long recovery took, in frames, or null while still recovering. */
|
|
1138
|
+
declare function recoveryFrames(state: PhaseState): number | null;
|
|
1139
|
+
|
|
1140
|
+
/**
|
|
1141
|
+
* The Media3 exporter.
|
|
1142
|
+
*
|
|
1143
|
+
* Honours `VideoExporter` in full: asynchronous, cancellable while it runs,
|
|
1144
|
+
* monotonic progress, temporary and file destinations, one export at a time,
|
|
1145
|
+
* typed refusal, cancellation, stall and failure, and deterministic cleanup on
|
|
1146
|
+
* every one of those paths.
|
|
1147
|
+
*
|
|
1148
|
+
* Nothing constructs it for you. It is not on `VideoRuntime`, not behind the
|
|
1149
|
+
* editor's Next button, and not reachable from `@pictor-sdk/video-ui` — a host
|
|
1150
|
+
* builds it explicitly, which is what keeps the surface reviewable. Wiring it
|
|
1151
|
+
* to a button is Slice 5's job, not this file's.
|
|
1152
|
+
*
|
|
1153
|
+
* ## The gate boundary
|
|
1154
|
+
*
|
|
1155
|
+
* Nothing here reaches a `exportGate*` method. Provenance hashing, fault
|
|
1156
|
+
* injection, artificial holds and test beats live behind
|
|
1157
|
+
* `exportGateEnableDiagnostics`, which this never calls, and which leaves them
|
|
1158
|
+
* inert even when a payload asks for them.
|
|
1159
|
+
*
|
|
1160
|
+
* ## The whole project is interpreted here, in TypeScript
|
|
1161
|
+
*
|
|
1162
|
+
* `planClipShards` and `compileVideoSceneProgram` run once, sources are resolved
|
|
1163
|
+
* once, overlays are rasterised once, and what crosses to native is a list of
|
|
1164
|
+
* shards, a compiled program and a resource manifest — all JSON and file paths.
|
|
1165
|
+
* Native never sees a `VideoProject`, a `VideoClip`, a trim, a speed or a
|
|
1166
|
+
* transform field. It looks up segments by timestamp and draws what they say.
|
|
1167
|
+
*
|
|
1168
|
+
* ## It refuses rather than approximating
|
|
1169
|
+
*
|
|
1170
|
+
* Every clip kind, spec field and overlay state this slice does not implement is
|
|
1171
|
+
* rejected by name before anything is encoded. A photo clip silently skipped, or
|
|
1172
|
+
* a bitrate silently ignored, produces a file that is plausible and wrong — and
|
|
1173
|
+
* a plausible wrong file is worse than no file, because nothing downstream can
|
|
1174
|
+
* tell it apart from a correct one.
|
|
1175
|
+
*
|
|
1176
|
+
* @experimental
|
|
1177
|
+
*/
|
|
1178
|
+
/**
|
|
1179
|
+
* An overlay raster, described structurally.
|
|
1180
|
+
*
|
|
1181
|
+
* Deliberately not imported from `@pictor-sdk/video-skia`. The adapter has no
|
|
1182
|
+
* opinion about how an overlay is rendered — that is the whole point of the
|
|
1183
|
+
* boundary — and importing the renderer would drag Skia into a package whose
|
|
1184
|
+
* job is to talk to Media3. What it needs is the shape of a descriptor, which
|
|
1185
|
+
* is JSON.
|
|
1186
|
+
*
|
|
1187
|
+
* @experimental
|
|
1188
|
+
*/
|
|
1189
|
+
interface OverlayResourceDescriptor {
|
|
1190
|
+
overlayId: string;
|
|
1191
|
+
resourceId: string;
|
|
1192
|
+
size: {
|
|
1193
|
+
width: number;
|
|
1194
|
+
height: number;
|
|
1195
|
+
};
|
|
1196
|
+
rasterBounds: {
|
|
1197
|
+
x: number;
|
|
1198
|
+
y: number;
|
|
1199
|
+
width: number;
|
|
1200
|
+
height: number;
|
|
1201
|
+
};
|
|
1202
|
+
rasterScale: number;
|
|
1203
|
+
rasterWidth: number;
|
|
1204
|
+
rasterHeight: number;
|
|
1205
|
+
format: string;
|
|
1206
|
+
premultiplied: boolean;
|
|
1207
|
+
colorSpace: string;
|
|
1208
|
+
file: string;
|
|
1209
|
+
byteLength: number;
|
|
1210
|
+
checksum: number;
|
|
1211
|
+
}
|
|
1212
|
+
/** What preparing a program's overlays produced. @experimental */
|
|
1213
|
+
interface OverlayPreparationResult {
|
|
1214
|
+
resources: readonly OverlayResourceDescriptor[];
|
|
1215
|
+
/** Still resolving. An export must not proceed and omit them. */
|
|
1216
|
+
pending: readonly string[];
|
|
1217
|
+
/** Cannot be exported at all. */
|
|
1218
|
+
invalid: readonly {
|
|
1219
|
+
overlayId: string;
|
|
1220
|
+
kind: string;
|
|
1221
|
+
reason: string;
|
|
1222
|
+
}[];
|
|
1223
|
+
}
|
|
1224
|
+
/** Everything the host must supply. @experimental */
|
|
1225
|
+
interface Media3ExporterOptions {
|
|
1226
|
+
/**
|
|
1227
|
+
* Turn an opaque `MediaSource` into something Android can open.
|
|
1228
|
+
*
|
|
1229
|
+
* The exporter never inspects a descriptor: `MediaSource` is `JsonValue` and
|
|
1230
|
+
* only the host knows whether it is an asset id, a content URI or a row in a
|
|
1231
|
+
* database. Returning `null` fails the export naming the clip or the span.
|
|
1232
|
+
*
|
|
1233
|
+
* `context` says which part of the project is being resolved. It used to be
|
|
1234
|
+
* the clip, which left the audio path with nothing honest to pass: every
|
|
1235
|
+
* span, project music included, was resolved with the *first video clip* as
|
|
1236
|
+
* context. A resolver that keys on the clip returned that clip's media for
|
|
1237
|
+
* the whole timeline's audio — a plausible file with the wrong sound in it.
|
|
1238
|
+
*/
|
|
1239
|
+
resolveSource(source: MediaSource, context: ExportSourceContext): string | null;
|
|
1240
|
+
/**
|
|
1241
|
+
* Rasterise every overlay the compiled program names.
|
|
1242
|
+
*
|
|
1243
|
+
* The host wires its registry, its prepared resources and its raster sink
|
|
1244
|
+
* behind this — typically `prepareProgramOverlayExports` from
|
|
1245
|
+
* `@pictor-sdk/video-skia`. Injecting the whole step rather than its three inputs
|
|
1246
|
+
* is what keeps this package free of a renderer dependency, and it is the same
|
|
1247
|
+
* reason `resolveSource` is injected rather than a media library being.
|
|
1248
|
+
*
|
|
1249
|
+
* Omitted means "no overlays are exportable": a project that has any then
|
|
1250
|
+
* fails naming each one, rather than exporting without them.
|
|
1251
|
+
*/
|
|
1252
|
+
prepareOverlays?(program: {
|
|
1253
|
+
segments: readonly {
|
|
1254
|
+
commands: readonly unknown[];
|
|
1255
|
+
}[];
|
|
1256
|
+
}, projectSize: {
|
|
1257
|
+
width: number;
|
|
1258
|
+
height: number;
|
|
1259
|
+
}, rasterScale: number): OverlayPreparationResult;
|
|
1260
|
+
/**
|
|
1261
|
+
* Called after an export finishes, successfully or not.
|
|
1262
|
+
*
|
|
1263
|
+
* The sidecars belong to the host's raster sink, so the host removes them.
|
|
1264
|
+
* Passing the list rather than the sink keeps this from having to know how
|
|
1265
|
+
* they were stored.
|
|
1266
|
+
*/
|
|
1267
|
+
releaseResources?(files: readonly string[]): void;
|
|
1268
|
+
/**
|
|
1269
|
+
* How long without any heartbeat counts as wedged.
|
|
1270
|
+
*
|
|
1271
|
+
* Defaults to {@link STALL_GRACE_MS}, which is **Pixel 7 evidence**: the worst
|
|
1272
|
+
* legitimate silence measured across seven production-path cases on that
|
|
1273
|
+
* device, with margin. It is not a claim about Android. A slower encoder, a
|
|
1274
|
+
* thermally throttled phone or a codec that batches differently can produce a
|
|
1275
|
+
* longer honest gap, and the number to change is this one — raising it costs
|
|
1276
|
+
* only how late a genuine wedge is noticed, while leaving it too low kills
|
|
1277
|
+
* healthy exports.
|
|
1278
|
+
*/
|
|
1279
|
+
stallGraceMs?: number;
|
|
1280
|
+
/**
|
|
1281
|
+
* How often native counters are read, and progress delivered.
|
|
1282
|
+
*
|
|
1283
|
+
* Defaults to 125 ms — 8 Hz, inside the 5–10 Hz a person can actually read.
|
|
1284
|
+
* Native counting is unthrottled regardless; this is only the bridge rate.
|
|
1285
|
+
*/
|
|
1286
|
+
progressPollMs?: number;
|
|
1287
|
+
}
|
|
1288
|
+
/** Why an export will not start. @experimental */
|
|
1289
|
+
declare class VideoExportRejected extends Error {
|
|
1290
|
+
readonly reasons: readonly string[];
|
|
1291
|
+
constructor(reasons: readonly string[]);
|
|
1292
|
+
}
|
|
1293
|
+
/**
|
|
1294
|
+
* Build an exporter.
|
|
1295
|
+
*
|
|
1296
|
+
* @experimental
|
|
1297
|
+
*/
|
|
1298
|
+
declare function createMedia3VideoExporter(options: Media3ExporterOptions): VideoExporter;
|
|
1299
|
+
|
|
1300
|
+
/**
|
|
1301
|
+
* One export at a time, settled exactly once, cancellable anywhere.
|
|
1302
|
+
*
|
|
1303
|
+
* The experimental exporter blocked the bridge for a whole encode and threw
|
|
1304
|
+
* from `cancel()`. That was honest — a silent no-op behind a cancel button is
|
|
1305
|
+
* worse than a refusal — but it is not something a host can ship, so this is the
|
|
1306
|
+
* orchestration that makes cancellation real.
|
|
1307
|
+
*
|
|
1308
|
+
* ## Why cancellation cannot just reject the promise
|
|
1309
|
+
*
|
|
1310
|
+
* A Transformer nobody is listening to still encodes, still writes and still
|
|
1311
|
+
* holds its codecs. Rejecting in JavaScript would hand the host a cancelled
|
|
1312
|
+
* state over an export that is still running, still filling the cache and still
|
|
1313
|
+
* about to produce a file nobody is waiting for. So `cancel` reaches native
|
|
1314
|
+
* first and the promise settles from the job's own checkpoints.
|
|
1315
|
+
*
|
|
1316
|
+
* ## Checkpoints
|
|
1317
|
+
*
|
|
1318
|
+
* Cancellation is cooperative. The work is a sequence of stages, each of which
|
|
1319
|
+
* calls {@link ExportJob.checkpoint} on the way past, and the first checkpoint
|
|
1320
|
+
* after a cancel throws. That is why the stages are named: a cancellation
|
|
1321
|
+
* carries where it stopped, which is the difference between a log that says
|
|
1322
|
+
* "cancelled" and one that says which resources were live at the time.
|
|
1323
|
+
*
|
|
1324
|
+
* ## Settling once
|
|
1325
|
+
*
|
|
1326
|
+
* Every path — success, failure, cancellation — clears the active job, and
|
|
1327
|
+
* nothing may be delivered afterwards. A progress callback that arrives after a
|
|
1328
|
+
* host has shown a result is a UI that goes backwards from "done".
|
|
1329
|
+
*
|
|
1330
|
+
* @experimental
|
|
1331
|
+
*/
|
|
1332
|
+
/** The lifecycle points an export can be stopped at. @experimental */
|
|
1333
|
+
declare const EXPORT_STAGES: readonly ["resolving", "overlays", "shards", "cadence", "audio", "concatenating", "finalising", "publishing"];
|
|
1334
|
+
/** @experimental */
|
|
1335
|
+
type ExportStage = (typeof EXPORT_STAGES)[number];
|
|
1336
|
+
/** @experimental */
|
|
1337
|
+
interface ExportJob {
|
|
1338
|
+
readonly id: string;
|
|
1339
|
+
/**
|
|
1340
|
+
* Give cancellation a chance to take effect.
|
|
1341
|
+
*
|
|
1342
|
+
* Throws {@link VideoExportCancelled} if the job has been cancelled. Called
|
|
1343
|
+
* between stages and inside long loops, because a cancel that can only be
|
|
1344
|
+
* observed when the work finishes is not a cancel.
|
|
1345
|
+
*/
|
|
1346
|
+
checkpoint(stage: ExportStage): void;
|
|
1347
|
+
/** Deliver progress, unless the job has settled. */
|
|
1348
|
+
progress(value: number, deliver: (value: number) => void): void;
|
|
1349
|
+
}
|
|
1350
|
+
/** @experimental */
|
|
1351
|
+
interface ExportJobsOptions {
|
|
1352
|
+
/**
|
|
1353
|
+
* Stop the native side of a job.
|
|
1354
|
+
*
|
|
1355
|
+
* Given the job id so a late cancel cannot stop a *different* export that
|
|
1356
|
+
* started in the meantime.
|
|
1357
|
+
*/
|
|
1358
|
+
cancelNative(jobId: string): void;
|
|
1359
|
+
/** Injected in tests. */
|
|
1360
|
+
now?: () => number;
|
|
1361
|
+
}
|
|
1362
|
+
/**
|
|
1363
|
+
* The job domain's ledger.
|
|
1364
|
+
*
|
|
1365
|
+
* Kept apart from the native run counters on purpose. One export job drives
|
|
1366
|
+
* several Transformer runs, so adding a job total to a run total produces a
|
|
1367
|
+
* number that reconciles with nothing — and reads like a reconciliation, which
|
|
1368
|
+
* is worse than reporting neither.
|
|
1369
|
+
*
|
|
1370
|
+
* @experimental
|
|
1371
|
+
*/
|
|
1372
|
+
interface ExportJobCounts {
|
|
1373
|
+
/** Jobs that began. Refused starts are not accepted jobs. */
|
|
1374
|
+
accepted: number;
|
|
1375
|
+
succeeded: number;
|
|
1376
|
+
cancelled: number;
|
|
1377
|
+
failed: number;
|
|
1378
|
+
/** Exactly 0 or 1: one job at a time. */
|
|
1379
|
+
active: number;
|
|
1380
|
+
}
|
|
1381
|
+
/** @experimental */
|
|
1382
|
+
interface ExportJobs {
|
|
1383
|
+
/** The running job's id, or `null`. */
|
|
1384
|
+
readonly active: string | null;
|
|
1385
|
+
/**
|
|
1386
|
+
* `accepted === succeeded + cancelled + failed + active`, always.
|
|
1387
|
+
*
|
|
1388
|
+
* Terminal outcomes are mutually exclusive: a cancelled job is not also a
|
|
1389
|
+
* failed one, however it unwound.
|
|
1390
|
+
*/
|
|
1391
|
+
readonly counts: ExportJobCounts;
|
|
1392
|
+
/** Run `body` as the one active job. */
|
|
1393
|
+
run<T>(body: (job: ExportJob) => Promise<T>): Promise<T>;
|
|
1394
|
+
/** Ask the running job to stop. Idempotent; a no-op when none is running. */
|
|
1395
|
+
cancel(): void;
|
|
1396
|
+
}
|
|
1397
|
+
/** @experimental */
|
|
1398
|
+
declare function createExportJobs(options: ExportJobsOptions): ExportJobs;
|
|
1399
|
+
|
|
1400
|
+
/**
|
|
1401
|
+
* How far an export has got, and how to tell slow from stuck.
|
|
1402
|
+
*
|
|
1403
|
+
* ## Weighting is derived, not decided
|
|
1404
|
+
*
|
|
1405
|
+
* Equal percentages per stage would be a lie the bar tells: a 0.1× shard writes
|
|
1406
|
+
* three hundred frames while its neighbour writes sixty, and splitting them
|
|
1407
|
+
* evenly makes the first appear stuck and the second appear instant. The weights
|
|
1408
|
+
* come from the work the compiler already planned — output frames per shard,
|
|
1409
|
+
* audio samples, the mux — so the bar moves at the rate the encoder does.
|
|
1410
|
+
*
|
|
1411
|
+
* Output frames, specifically. Cadence changes how many frames a shard *writes*
|
|
1412
|
+
* for a given source, and writing is what the progress is measuring the passage
|
|
1413
|
+
* of; counting source frames would make slow motion look fast and fast motion
|
|
1414
|
+
* look stuck.
|
|
1415
|
+
*
|
|
1416
|
+
* ## What 1 means
|
|
1417
|
+
*
|
|
1418
|
+
* One is not "the encoder finished". It is "the destination is complete and
|
|
1419
|
+
* readable", which is a later moment: there is still a mux, and after that a
|
|
1420
|
+
* publication. Reporting 1 at the end of encoding produces a "done" that cannot
|
|
1421
|
+
* yet be opened, which is worse than a bar that sits at 97 for a moment.
|
|
1422
|
+
*
|
|
1423
|
+
* ## Progress and heartbeat are different signals
|
|
1424
|
+
*
|
|
1425
|
+
* **Progress** is completed work shown to a person, throttled to a handful of
|
|
1426
|
+
* updates a second because nobody can read more than that and every update
|
|
1427
|
+
* costs a bridge crossing.
|
|
1428
|
+
*
|
|
1429
|
+
* **Heartbeat** is evidence that work is still happening. It is not throttled,
|
|
1430
|
+
* it does not have to move the visible percentage, and it is counted natively:
|
|
1431
|
+
* a single muxed sample batch or a block of copied destination bytes is a
|
|
1432
|
+
* heartbeat even when the bar does not visibly change for a second.
|
|
1433
|
+
*
|
|
1434
|
+
* Conflating them is what makes a watchdog wrong in both directions. Keyed on
|
|
1435
|
+
* visible progress, a long mux or a large destination copy looks stalled
|
|
1436
|
+
* because its percentage barely moves; keyed on elapsed time per stage, a 0.1×
|
|
1437
|
+
* shard that legitimately runs for minutes looks stalled too. What actually
|
|
1438
|
+
* marks a wedge is **silence** — no heartbeat from any source at all.
|
|
1439
|
+
*
|
|
1440
|
+
* @experimental
|
|
1441
|
+
*/
|
|
1442
|
+
/** What a shard will write. @experimental */
|
|
1443
|
+
interface ShardPlan {
|
|
1444
|
+
/** Output frames, after cadence. */
|
|
1445
|
+
frames: number;
|
|
1446
|
+
}
|
|
1447
|
+
/** @experimental */
|
|
1448
|
+
interface ProgressPlanInput {
|
|
1449
|
+
shards: readonly ShardPlan[];
|
|
1450
|
+
/** Mixed audio samples the timeline will produce. Zero when silent. */
|
|
1451
|
+
audioSamples: number;
|
|
1452
|
+
durationUs: number;
|
|
1453
|
+
}
|
|
1454
|
+
/** Fractions of the whole export, summing to 1. @experimental */
|
|
1455
|
+
interface ProgressPlan {
|
|
1456
|
+
shardWeights: number[];
|
|
1457
|
+
audioWeight: number;
|
|
1458
|
+
/** Concatenation and mux: real work, and the last thing before publication. */
|
|
1459
|
+
muxWeight: number;
|
|
1460
|
+
/**
|
|
1461
|
+
* Publication: writing the destination and making it readable.
|
|
1462
|
+
*
|
|
1463
|
+
* Held back deliberately so the encoded-and-muxed state cannot reach 1. A bar
|
|
1464
|
+
* that fills before the file exists produces a "done" that cannot be opened,
|
|
1465
|
+
* which is a worse lie than sitting at 97 for a moment.
|
|
1466
|
+
*/
|
|
1467
|
+
publishWeight: number;
|
|
1468
|
+
durationUs: number;
|
|
1469
|
+
/** The totals the fractions are measured against. */
|
|
1470
|
+
shardFrames: number[];
|
|
1471
|
+
audioSamples: number;
|
|
1472
|
+
}
|
|
1473
|
+
/**
|
|
1474
|
+
* How long without a heartbeat counts as wedged.
|
|
1475
|
+
*
|
|
1476
|
+
* Derived from the longest silence real work produces, not from how long the
|
|
1477
|
+
* platform takes to notice a wedge. The first version had that backwards — it
|
|
1478
|
+
* justified twelve seconds by sitting *past* Media3's observed 10.5 s stall
|
|
1479
|
+
* detection, which makes a watchdog decorative: it can only ever be the second
|
|
1480
|
+
* thing to notice.
|
|
1481
|
+
*
|
|
1482
|
+
* The maximum legitimate heartbeat gap, measured across **three passes of seven
|
|
1483
|
+
* cases** on a Pixel 7 — short and long 1×, 0.1× and 10× cadence, mixed-speed
|
|
1484
|
+
* multi-clip, muted, and project audio — is **2046 ms**, in the 0.1× case. Seven
|
|
1485
|
+
* seconds is 3.4× that, and 3.5 s ahead of the platform's own 10.5 s stall
|
|
1486
|
+
* detection. Both margins are the point: too close to the first and it fires on
|
|
1487
|
+
* healthy work, too close to the second and it is not worth having.
|
|
1488
|
+
*
|
|
1489
|
+
* Three passes rather than one because a single pass is not a maximum. The same
|
|
1490
|
+
* 0.1× case measured 1668, 1426 and 2046 ms in three different sessions, and a
|
|
1491
|
+
* threshold frozen from the lowest of those would have been derived from a run
|
|
1492
|
+
* that happened to go well.
|
|
1493
|
+
*
|
|
1494
|
+
* ## Two corrections this number carries
|
|
1495
|
+
*
|
|
1496
|
+
* The first series measured 1668 ms with gate provenance hashing running — every
|
|
1497
|
+
* sample of both intermediates and the result read and digested, beating as it
|
|
1498
|
+
* went. A threshold calibrated against work the user never pays for is
|
|
1499
|
+
* calibrated against the wrong export, so the hashing moved behind a conformance
|
|
1500
|
+
* flag. The second series missed the *per-shard* hashing, which was still
|
|
1501
|
+
* running. With all of it gated the worst source moved from `muxBatch` to
|
|
1502
|
+
* `audioBlock`: the mux beats on its own sample batches, and once the hashing
|
|
1503
|
+
* that was reading them stopped, the audio mix became the quietest stage again.
|
|
1504
|
+
*
|
|
1505
|
+
* ## The headroom here is small, and that is the finding
|
|
1506
|
+
*
|
|
1507
|
+
* 3.4× the worst gap and 3.5 s before the platform leaves roughly one doubling
|
|
1508
|
+
* of the worst legitimate silence before the two constraints meet. On a slower
|
|
1509
|
+
* encoder there may be no gap between them at all, which is why
|
|
1510
|
+
* `Media3ExporterOptions.stallGraceMs` exists.
|
|
1511
|
+
*
|
|
1512
|
+
* Any of this was only measurable after the silence was instrumented rather
|
|
1513
|
+
* than tolerated. Before, the maximum gap was 5.4 s and no useful threshold
|
|
1514
|
+
* existed between it and the platform's timeout; the gap turned out to be the
|
|
1515
|
+
* Sonic time-stretch and AAC encode inside the audio mix, which at 0.1× turn one
|
|
1516
|
+
* second of source into ten with nothing beating.
|
|
1517
|
+
*
|
|
1518
|
+
* @experimental
|
|
1519
|
+
*/
|
|
1520
|
+
declare const STALL_GRACE_MS = 7000;
|
|
1521
|
+
/** The measured maximum, kept beside the threshold it justifies. @experimental */
|
|
1522
|
+
declare const MAX_OBSERVED_HEARTBEAT_GAP_MS = 2046;
|
|
1523
|
+
/** @experimental */
|
|
1524
|
+
declare function planProgressWeights(input: ProgressPlanInput): ProgressPlan;
|
|
1525
|
+
/** @experimental */
|
|
1526
|
+
interface ProgressReporterOptions {
|
|
1527
|
+
plan: ProgressPlan;
|
|
1528
|
+
durationUs: number;
|
|
1529
|
+
onProgress?: (progress: VideoExportProgress) => void;
|
|
1530
|
+
/** Bridge crossings are bounded to roughly this spacing. */
|
|
1531
|
+
throttleMs?: number;
|
|
1532
|
+
now?: () => number;
|
|
1533
|
+
}
|
|
1534
|
+
/** @experimental */
|
|
1535
|
+
interface ProgressReporter {
|
|
1536
|
+
/** Emit 0, once, when work begins. */
|
|
1537
|
+
begin(): void;
|
|
1538
|
+
/** Report a shard's completed output frames. */
|
|
1539
|
+
shard(index: number, framesDone: number): void;
|
|
1540
|
+
/** Report mixed audio samples. */
|
|
1541
|
+
audio(samplesDone: number): void;
|
|
1542
|
+
/** Concatenation and mux finished. Not 1. */
|
|
1543
|
+
muxed(): void;
|
|
1544
|
+
/** The destination is complete and readable. Emits 1, once. */
|
|
1545
|
+
complete(): void;
|
|
1546
|
+
/**
|
|
1547
|
+
* Apply several updates and emit at most once.
|
|
1548
|
+
*
|
|
1549
|
+
* A poll reads every shard, the audio and the mux together, and without this
|
|
1550
|
+
* the first of those spends the throttle slot and the rest are dropped until
|
|
1551
|
+
* the next tick — so the last shard's frames and the whole audio mix could
|
|
1552
|
+
* arrive a tick late or, at the end of an export, never. One read of native
|
|
1553
|
+
* state is one update to the bar.
|
|
1554
|
+
*/
|
|
1555
|
+
batch(body: () => void): void;
|
|
1556
|
+
/** Deliver nothing further. */
|
|
1557
|
+
stop(): void;
|
|
1558
|
+
}
|
|
1559
|
+
/** @experimental */
|
|
1560
|
+
declare function createProgressReporter(options: ProgressReporterOptions): ProgressReporter;
|
|
1561
|
+
/** What a wedged export looked like when it was noticed. @experimental */
|
|
1562
|
+
interface StallVerdict {
|
|
1563
|
+
stage: ExportStage | string;
|
|
1564
|
+
units: number;
|
|
1565
|
+
idleMs: number;
|
|
1566
|
+
}
|
|
1567
|
+
/** @experimental */
|
|
1568
|
+
interface StallWatchdogOptions {
|
|
1569
|
+
thresholdMs?: number;
|
|
1570
|
+
}
|
|
1571
|
+
/** @experimental */
|
|
1572
|
+
interface StallWatchdog {
|
|
1573
|
+
/**
|
|
1574
|
+
* Record a heartbeat. Any beat counts, whether or not it moved the bar.
|
|
1575
|
+
*
|
|
1576
|
+
* `units` is carried for the report only — a repeated count is still a
|
|
1577
|
+
* heartbeat, because the pipeline spoke.
|
|
1578
|
+
*/
|
|
1579
|
+
advanced(stage: ExportStage | string, units: number, atMs: number): void;
|
|
1580
|
+
/** The verdict if nothing has reported for the threshold, else `null`. */
|
|
1581
|
+
stalledFor(atMs: number): StallVerdict | null;
|
|
1582
|
+
}
|
|
1583
|
+
/**
|
|
1584
|
+
* Slow is not stuck.
|
|
1585
|
+
*
|
|
1586
|
+
* Keyed on *any* report rather than on elapsed time per stage: a 0.1× export
|
|
1587
|
+
* spends minutes inside one shard and is perfectly healthy while doing it, and
|
|
1588
|
+
* a stage timeout would either fire on that or be too long to catch anything.
|
|
1589
|
+
* What distinguishes a wedge is silence.
|
|
1590
|
+
*
|
|
1591
|
+
* Fires once. A wedged export is one event, and the cleanup it triggers is the
|
|
1592
|
+
* same path cancellation already proved.
|
|
1593
|
+
*
|
|
1594
|
+
* @experimental
|
|
1595
|
+
*/
|
|
1596
|
+
declare function createStallWatchdog(options?: StallWatchdogOptions): StallWatchdog;
|
|
1597
|
+
|
|
1598
|
+
/**
|
|
1599
|
+
* Saving a finished export into Android's video collection.
|
|
1600
|
+
*
|
|
1601
|
+
* A thin boundary. The insert-pending / stream / verify / publish sequence is
|
|
1602
|
+
* native, because it is where the file descriptors and the cancellation live;
|
|
1603
|
+
* this turns one JSON result into either a {@link SavedMedia} or a typed error,
|
|
1604
|
+
* and gives every save an id so a cancel reaches the right one.
|
|
1605
|
+
*
|
|
1606
|
+
* ## What a failure guarantees
|
|
1607
|
+
*
|
|
1608
|
+
* Nothing partial. Every unsuccessful native path — a short write, a full disk,
|
|
1609
|
+
* a cancellation, an unreadable result — deletes the pending row before
|
|
1610
|
+
* returning, and a pending row was never visible to the gallery in the first
|
|
1611
|
+
* place. So a failed save leaves the collection exactly as it was.
|
|
1612
|
+
*
|
|
1613
|
+
* ## What it never does
|
|
1614
|
+
*
|
|
1615
|
+
* Delete a published item. Releasing the export's temporary is a separate act
|
|
1616
|
+
* on a separate file, and a saved video belongs to the user from the moment it
|
|
1617
|
+
* is published.
|
|
1618
|
+
*
|
|
1619
|
+
* @experimental
|
|
1620
|
+
*/
|
|
1621
|
+
/** Why a save did not happen. @experimental */
|
|
1622
|
+
declare class MediaSaveFailed extends Error {
|
|
1623
|
+
/** Safe to display: never a path, a stack, or a platform message. */
|
|
1624
|
+
readonly reason: string;
|
|
1625
|
+
constructor(reason: string);
|
|
1626
|
+
}
|
|
1627
|
+
/** A save the user asked to stop. @experimental */
|
|
1628
|
+
declare class MediaSaveCancelled extends Error {
|
|
1629
|
+
constructor();
|
|
1630
|
+
}
|
|
1631
|
+
/**
|
|
1632
|
+
* Offer a saved video to the system share sheet.
|
|
1633
|
+
*
|
|
1634
|
+
* Only after a successful save, and only ever the durable `content:` uri. The
|
|
1635
|
+
* cache path the export produced is app-private: a receiver could not open it,
|
|
1636
|
+
* and handing one out is the mistake `FileUriExposedException` exists to catch.
|
|
1637
|
+
*
|
|
1638
|
+
* Sharing changes nothing. The same row is offered every time — no copy, no
|
|
1639
|
+
* rename, no re-publish, and nothing released.
|
|
1640
|
+
*
|
|
1641
|
+
* Returns `false` when the device has nothing that accepts a video, which is an
|
|
1642
|
+
* outcome rather than a failure: the saved video is untouched either way.
|
|
1643
|
+
*
|
|
1644
|
+
* @experimental
|
|
1645
|
+
*/
|
|
1646
|
+
declare function shareSavedMedia(uri: string): {
|
|
1647
|
+
ok: boolean;
|
|
1648
|
+
noTargets: boolean;
|
|
1649
|
+
reason: string;
|
|
1650
|
+
};
|
|
1651
|
+
/** @experimental */
|
|
1652
|
+
declare function createMediaStoreSaver(): MediaSaver;
|
|
1653
|
+
/**
|
|
1654
|
+
* Delete pending rows a killed process left behind.
|
|
1655
|
+
*
|
|
1656
|
+
* Worth calling once at startup. A pending row is invisible to the gallery and
|
|
1657
|
+
* still occupies storage, so nothing will ever prompt a user to clean one up.
|
|
1658
|
+
*
|
|
1659
|
+
* Scoped to this app's own still-pending rows under its own folder: a published
|
|
1660
|
+
* item is never a candidate, and neither is anything another app wrote.
|
|
1661
|
+
*
|
|
1662
|
+
* @experimental
|
|
1663
|
+
*/
|
|
1664
|
+
declare function sweepPendingMediaSaves(): {
|
|
1665
|
+
swept: number;
|
|
1666
|
+
inspected: number;
|
|
1667
|
+
/** Pending rows seen and deliberately left alone. */
|
|
1668
|
+
skipped: number;
|
|
1669
|
+
};
|
|
1670
|
+
|
|
1671
|
+
/**
|
|
1672
|
+
* Install the staging primitive, once.
|
|
1673
|
+
*
|
|
1674
|
+
* Idempotent, and safe to call before an export rather than at module load: a
|
|
1675
|
+
* host that never exports an overlay never pays for it.
|
|
1676
|
+
*/
|
|
1677
|
+
declare function installOverlayStaging(): boolean;
|
|
1678
|
+
/** What a sink hands back, plus the token that owns it. @experimental */
|
|
1679
|
+
interface StagedRaster {
|
|
1680
|
+
file: string;
|
|
1681
|
+
byteLength: number;
|
|
1682
|
+
checksum: number;
|
|
1683
|
+
}
|
|
1684
|
+
/**
|
|
1685
|
+
* A sink, and the tokens it is holding.
|
|
1686
|
+
*
|
|
1687
|
+
* The token list is the whole point of the wrapper: the structural
|
|
1688
|
+
* `OverlayRasterSink` contract identifies a staged raster by its file, and a
|
|
1689
|
+
* caller that released by path would be naming something it was never given
|
|
1690
|
+
* authority over. Every release here goes through the token native issued.
|
|
1691
|
+
*
|
|
1692
|
+
* @experimental
|
|
1693
|
+
*/
|
|
1694
|
+
interface OverlayStagingSink {
|
|
1695
|
+
stage(bytes: Uint8Array, meta: {
|
|
1696
|
+
overlayId: string;
|
|
1697
|
+
resourceId: string;
|
|
1698
|
+
rasterWidth: number;
|
|
1699
|
+
rasterHeight: number;
|
|
1700
|
+
}): StagedRaster;
|
|
1701
|
+
/** Discard one staged raster, by the file the descriptor carried. */
|
|
1702
|
+
remove(file: string): void;
|
|
1703
|
+
/** Discard everything this sink still holds. Idempotent. */
|
|
1704
|
+
releaseAll(): number;
|
|
1705
|
+
/** For assertions: created, released and still live. */
|
|
1706
|
+
ledger(): {
|
|
1707
|
+
created: number;
|
|
1708
|
+
released: number;
|
|
1709
|
+
live: number;
|
|
1710
|
+
};
|
|
1711
|
+
}
|
|
1712
|
+
/**
|
|
1713
|
+
* Remove sidecars a previous process left behind.
|
|
1714
|
+
*
|
|
1715
|
+
* Scoped by native to this module's own directory and file suffix, so it cannot
|
|
1716
|
+
* take a cache file belonging to anything else. Returns how many it removed.
|
|
1717
|
+
*
|
|
1718
|
+
* @experimental
|
|
1719
|
+
*/
|
|
1720
|
+
declare function sweepOverlayStaging(): number;
|
|
1721
|
+
/** @experimental */
|
|
1722
|
+
declare function createOverlayStagingSink(): OverlayStagingSink;
|
|
1723
|
+
/**
|
|
1724
|
+
* How long a synchronous stage blocks the JS thread, for a raster this size.
|
|
1725
|
+
*
|
|
1726
|
+
* Exists because the synchronous contract is only defensible if the number is
|
|
1727
|
+
* small. If a maximum-size raster stalls long enough to drop frames in the
|
|
1728
|
+
* export sheet, the answer is to make `OverlayRasterSink.stage` asynchronous —
|
|
1729
|
+
* not to keep a blocking write behind a synchronous signature because it is
|
|
1730
|
+
* convenient.
|
|
1731
|
+
*
|
|
1732
|
+
* @experimental
|
|
1733
|
+
*/
|
|
1734
|
+
declare function measureStagingStall(sink: OverlayStagingSink, width: number, height: number): {
|
|
1735
|
+
ms: number;
|
|
1736
|
+
bytes: number;
|
|
1737
|
+
};
|
|
1738
|
+
|
|
1739
|
+
/**
|
|
1740
|
+
* Reading what a picked audio file actually contains.
|
|
1741
|
+
*
|
|
1742
|
+
* The structural half of {@link AudioProbe} lives in `@pictor-sdk/video-core`,
|
|
1743
|
+
* which has no decoder and never will; this is the adapter that can answer it.
|
|
1744
|
+
* A host calls this once, before adding, and passes the result to
|
|
1745
|
+
* `acceptAudioSource`.
|
|
1746
|
+
*
|
|
1747
|
+
* Nothing here identifies the file. The display name is the host's to carry —
|
|
1748
|
+
* a picker URI embeds an id and sometimes a user's own folder names.
|
|
1749
|
+
*
|
|
1750
|
+
* @experimental
|
|
1751
|
+
*/
|
|
1752
|
+
/** A probe that failed, with a reason fit for a log rather than a person. @experimental */
|
|
1753
|
+
interface AudioProbeFailure {
|
|
1754
|
+
ok: false;
|
|
1755
|
+
/** Why the file could not be read. Diagnostic, not a user-facing sentence. */
|
|
1756
|
+
reason: string;
|
|
1757
|
+
}
|
|
1758
|
+
/** What native learned, minus the display name only the host knows. @experimental */
|
|
1759
|
+
type AudioProbeResult = ({
|
|
1760
|
+
ok: true;
|
|
1761
|
+
} & Omit<AudioProbe, 'displayName'>) | AudioProbeFailure;
|
|
1762
|
+
/**
|
|
1763
|
+
* Open `uri` and report its audio.
|
|
1764
|
+
*
|
|
1765
|
+
* Never throws for an unreadable file: a pick that cannot be read is an answer,
|
|
1766
|
+
* and a host that had to wrap this in a `try` would be handling a normal
|
|
1767
|
+
* outcome as an exception. It throws only if the adapter is not installed,
|
|
1768
|
+
* which is a wiring mistake rather than a file.
|
|
1769
|
+
*
|
|
1770
|
+
* @experimental
|
|
1771
|
+
*/
|
|
1772
|
+
declare function probeAudioSource(uri: string): AudioProbeResult;
|
|
1773
|
+
|
|
1774
|
+
/**
|
|
1775
|
+
* Media the app owns, copied out of wherever it was picked from.
|
|
1776
|
+
*
|
|
1777
|
+
* A picker returns a reference, not a file. A `content://` grant is scoped to
|
|
1778
|
+
* the activity result that produced it; `copyToCacheDirectory` improves on that
|
|
1779
|
+
* by writing into **cache**, which Android reclaims whenever it wants storage
|
|
1780
|
+
* back. Either way a project persisted with a picker URI reopens with media
|
|
1781
|
+
* that has quietly gone, and nothing says so until an export fails.
|
|
1782
|
+
*
|
|
1783
|
+
* So ingestion copies into `filesDir` and the project stores that. See
|
|
1784
|
+
* `ManagedSources.kt` for why the name is the content hash and why nothing is
|
|
1785
|
+
* ever deleted implicitly.
|
|
1786
|
+
*
|
|
1787
|
+
* @experimental
|
|
1788
|
+
*/
|
|
1789
|
+
/** Where a managed source lives, and what it is. @experimental */
|
|
1790
|
+
interface ManagedSource {
|
|
1791
|
+
/** A `file://` URI in app-owned durable storage. Safe to persist. */
|
|
1792
|
+
uri: string;
|
|
1793
|
+
/** The content hash. Names the file; never derived from the user's filename. */
|
|
1794
|
+
token: string;
|
|
1795
|
+
byteLength: number;
|
|
1796
|
+
sha256: string;
|
|
1797
|
+
/**
|
|
1798
|
+
* Whether this call brought the file into existence.
|
|
1799
|
+
*
|
|
1800
|
+
* False means the bytes were already stored and something else may be
|
|
1801
|
+
* pointing at them — a caller unwinding its own work must not release a
|
|
1802
|
+
* source it did not create.
|
|
1803
|
+
*/
|
|
1804
|
+
created: boolean;
|
|
1805
|
+
}
|
|
1806
|
+
/** @experimental */
|
|
1807
|
+
interface ManagedSourceFailure {
|
|
1808
|
+
ok: false;
|
|
1809
|
+
/** Diagnostic. Never shown to a person: it may name a path. */
|
|
1810
|
+
reason: string;
|
|
1811
|
+
}
|
|
1812
|
+
/** @experimental */
|
|
1813
|
+
type IngestResult = ({
|
|
1814
|
+
ok: true;
|
|
1815
|
+
} & ManagedSource) | ManagedSourceFailure;
|
|
1816
|
+
/**
|
|
1817
|
+
* Copy a picked source into durable, app-owned storage.
|
|
1818
|
+
*
|
|
1819
|
+
* Never throws for an unreadable source: a pick that cannot be copied is an
|
|
1820
|
+
* answer. It throws only if the adapter is not installed.
|
|
1821
|
+
*
|
|
1822
|
+
* @experimental
|
|
1823
|
+
*/
|
|
1824
|
+
declare function ingestManagedSource(uri: string): IngestResult;
|
|
1825
|
+
/**
|
|
1826
|
+
* Drop one managed source.
|
|
1827
|
+
*
|
|
1828
|
+
* For unwinding an ingestion whose item was never created. Not for Undo: an
|
|
1829
|
+
* Undo that deleted the source would leave Redo restoring an item that points
|
|
1830
|
+
* at nothing.
|
|
1831
|
+
*
|
|
1832
|
+
* @experimental
|
|
1833
|
+
*/
|
|
1834
|
+
declare function releaseManagedSource(token: string): boolean;
|
|
1835
|
+
/**
|
|
1836
|
+
* Remove every managed source not named in `keepTokens`.
|
|
1837
|
+
*
|
|
1838
|
+
* The only thing that deletes media, and it is always given the full set of
|
|
1839
|
+
* what the project still references. Also removes partials a killed process
|
|
1840
|
+
* left behind.
|
|
1841
|
+
*
|
|
1842
|
+
* @experimental
|
|
1843
|
+
*/
|
|
1844
|
+
declare function sweepManagedSources(keepTokens: readonly string[]): {
|
|
1845
|
+
removed: number;
|
|
1846
|
+
kept: number;
|
|
1847
|
+
partials: number;
|
|
1848
|
+
/** Files that are not ours, counted so the accounting matches a listing. */
|
|
1849
|
+
skipped: number;
|
|
1850
|
+
};
|
|
1851
|
+
/**
|
|
1852
|
+
* Every filename in the managed directory, including partials and strays.
|
|
1853
|
+
*
|
|
1854
|
+
* Read-only accounting, so it is available in release for the same reason the
|
|
1855
|
+
* ledgers are: it is how a sweep is proven rather than asserted.
|
|
1856
|
+
* {@link managedSourceLedger} reports only well-formed sources, which cannot
|
|
1857
|
+
* show what a sweep left behind.
|
|
1858
|
+
*
|
|
1859
|
+
* @experimental
|
|
1860
|
+
*/
|
|
1861
|
+
declare function managedSourceList(): {
|
|
1862
|
+
name: string;
|
|
1863
|
+
byteLength: number;
|
|
1864
|
+
}[];
|
|
1865
|
+
/**
|
|
1866
|
+
* Gate-only: put a file in the managed directory with an exact name.
|
|
1867
|
+
*
|
|
1868
|
+
* Ingestion only ever produces well-formed, referenced sources, so the cases a
|
|
1869
|
+
* sweep exists to handle — an orphan, a stale partial, a file that is not ours
|
|
1870
|
+
* — cannot be reached through the normal API. Refused in a release build built
|
|
1871
|
+
* without diagnostics.
|
|
1872
|
+
*
|
|
1873
|
+
* @experimental
|
|
1874
|
+
*/
|
|
1875
|
+
declare function seedManagedSourceForGate(name: string, bytes: number): boolean;
|
|
1876
|
+
/**
|
|
1877
|
+
* Gate-only: sweep with a keep list handed over exactly as written.
|
|
1878
|
+
*
|
|
1879
|
+
* The production path serialises its own list, so an unparseable one can never
|
|
1880
|
+
* reach native — which makes "an unreadable keep list deletes nothing"
|
|
1881
|
+
* unprovable through it. Returns what native reported.
|
|
1882
|
+
*
|
|
1883
|
+
* @experimental
|
|
1884
|
+
*/
|
|
1885
|
+
declare function sweepManagedSourcesRawForGate(rawKeep: string): {
|
|
1886
|
+
ok: boolean;
|
|
1887
|
+
removed: number;
|
|
1888
|
+
kept: number;
|
|
1889
|
+
partials: number;
|
|
1890
|
+
skipped: number;
|
|
1891
|
+
};
|
|
1892
|
+
/** What is stored, for assertions. Tokens and sizes, never names. @experimental */
|
|
1893
|
+
declare function managedSourceLedger(): {
|
|
1894
|
+
count: number;
|
|
1895
|
+
bytes: number;
|
|
1896
|
+
items: {
|
|
1897
|
+
token: string;
|
|
1898
|
+
byteLength: number;
|
|
1899
|
+
}[];
|
|
1900
|
+
};
|
|
1901
|
+
/**
|
|
1902
|
+
* The token a managed URI names, or null if it is not one of ours.
|
|
1903
|
+
*
|
|
1904
|
+
* Lets a host collect the keep list for {@link sweepManagedSources} straight
|
|
1905
|
+
* from a project's sources without storing a second index that could drift.
|
|
1906
|
+
*
|
|
1907
|
+
* @experimental
|
|
1908
|
+
*/
|
|
1909
|
+
declare function managedSourceToken(uri: string): string | null;
|
|
1910
|
+
|
|
1911
|
+
/**
|
|
1912
|
+
* The audio a preview plays.
|
|
1913
|
+
*
|
|
1914
|
+
* ## Preparation, not playback
|
|
1915
|
+
*
|
|
1916
|
+
* A preview mix is produced by the same code an export runs: every contributing
|
|
1917
|
+
* source is decoded, re-timed by its clip's speed, ramped, summed and saturated
|
|
1918
|
+
* once, then encoded. That takes real time proportional to the project, and no
|
|
1919
|
+
* amount of naming it "preview" makes it realtime. A caller has to show that it
|
|
1920
|
+
* is preparing and must not pretend otherwise.
|
|
1921
|
+
*
|
|
1922
|
+
* ## Why not mix in the player
|
|
1923
|
+
*
|
|
1924
|
+
* Because then there would be two definitions of what a project sounds like,
|
|
1925
|
+
* and the one a person hears while editing would be the one that was never
|
|
1926
|
+
* tested against an export. Gain, fades, trims, speed and overlap are decided
|
|
1927
|
+
* by `compileAudioMixProgram` and executed by `AudioMixer` — here and in the
|
|
1928
|
+
* exporter, from the same plan.
|
|
1929
|
+
*
|
|
1930
|
+
* ## Superseding
|
|
1931
|
+
*
|
|
1932
|
+
* One job at a time. A newer request cancels the one in flight, which unwinds
|
|
1933
|
+
* at its next checkpoint, deletes its partial and reports `superseded`. A
|
|
1934
|
+
* superseded result is not a failure and must never be published: it is the
|
|
1935
|
+
* answer to a question nobody is asking any more.
|
|
1936
|
+
*
|
|
1937
|
+
* @experimental
|
|
1938
|
+
*/
|
|
1939
|
+
/** A prepared mix. @experimental */
|
|
1940
|
+
interface PreparedMix {
|
|
1941
|
+
ok: true;
|
|
1942
|
+
/** A `file://` URI in app-private storage. */
|
|
1943
|
+
uri: string;
|
|
1944
|
+
/** The content key: mix program, sources and mixer configuration. */
|
|
1945
|
+
key: string;
|
|
1946
|
+
/** True when nothing had to be encoded — the file was already there. */
|
|
1947
|
+
cached: boolean;
|
|
1948
|
+
byteLength: number;
|
|
1949
|
+
/**
|
|
1950
|
+
* Where the time went, when it was actually prepared.
|
|
1951
|
+
*
|
|
1952
|
+
* Milliseconds per stage and frames per source. Absent for a cached mix,
|
|
1953
|
+
* which did no work to attribute.
|
|
1954
|
+
*/
|
|
1955
|
+
profile?: {
|
|
1956
|
+
stageMs: Record<string, number>;
|
|
1957
|
+
counts: Record<string, number>;
|
|
1958
|
+
sources: Record<string, Record<string, number>>;
|
|
1959
|
+
};
|
|
1960
|
+
}
|
|
1961
|
+
/** @experimental */
|
|
1962
|
+
type PreviewMixResult = PreparedMix
|
|
1963
|
+
/** A newer request replaced this one. Not a failure; never publish it. */
|
|
1964
|
+
| {
|
|
1965
|
+
ok: false;
|
|
1966
|
+
superseded: true;
|
|
1967
|
+
key: string;
|
|
1968
|
+
}
|
|
1969
|
+
/** It ran and failed. The reason is diagnostic and may name a path. */
|
|
1970
|
+
| {
|
|
1971
|
+
ok: false;
|
|
1972
|
+
superseded?: false;
|
|
1973
|
+
key?: string;
|
|
1974
|
+
reason: string;
|
|
1975
|
+
};
|
|
1976
|
+
/**
|
|
1977
|
+
* Prepare the mix for a compiled program, or hand back the one already
|
|
1978
|
+
* prepared.
|
|
1979
|
+
*
|
|
1980
|
+
* `urisJson` maps span id to a resolved, durable source URI — the same
|
|
1981
|
+
* resolution an export performs, so the two agree about which file a span
|
|
1982
|
+
* means.
|
|
1983
|
+
*
|
|
1984
|
+
* Never rejects: a mix that cannot be produced is an outcome.
|
|
1985
|
+
*
|
|
1986
|
+
* @experimental
|
|
1987
|
+
*/
|
|
1988
|
+
declare function preparePreviewMix(jobId: string, programJson: string, urisJson: string): Promise<PreviewMixResult>;
|
|
1989
|
+
/**
|
|
1990
|
+
* Stop whatever is preparing.
|
|
1991
|
+
*
|
|
1992
|
+
* Safe to call when nothing is: cancelling twice, or cancelling after a job
|
|
1993
|
+
* already finished, is how a caller's teardown paths overlap on purpose.
|
|
1994
|
+
*
|
|
1995
|
+
* @experimental
|
|
1996
|
+
*/
|
|
1997
|
+
declare function cancelPreviewMix(): void;
|
|
1998
|
+
/**
|
|
1999
|
+
* The key a program would produce, without preparing it.
|
|
2000
|
+
*
|
|
2001
|
+
* Lets a caller ask "is this the mix I am already playing?" before starting a
|
|
2002
|
+
* job — which is what stops a selection change, or any edit that does not
|
|
2003
|
+
* change the sound, from rebuilding anything.
|
|
2004
|
+
*
|
|
2005
|
+
* @experimental
|
|
2006
|
+
*/
|
|
2007
|
+
declare function previewMixKey(programJson: string, urisJson: string): string;
|
|
2008
|
+
/** Drop one prepared mix. @experimental */
|
|
2009
|
+
declare function releasePreviewMix(key: string): boolean;
|
|
2010
|
+
/**
|
|
2011
|
+
* Keep only these mixes; remove the rest and every partial.
|
|
2012
|
+
*
|
|
2013
|
+
* The bound on the cache. A sweep that cannot read its own result removes
|
|
2014
|
+
* nothing, rather than assuming the directory should be empty.
|
|
2015
|
+
*
|
|
2016
|
+
* @experimental
|
|
2017
|
+
*/
|
|
2018
|
+
declare function sweepPreviewMixes(keepKeys: readonly string[]): {
|
|
2019
|
+
removed: number;
|
|
2020
|
+
kept: number;
|
|
2021
|
+
partials: number;
|
|
2022
|
+
skipped: number;
|
|
2023
|
+
};
|
|
2024
|
+
/** What is prepared, for assertions. @experimental */
|
|
2025
|
+
declare function previewMixLedger(): {
|
|
2026
|
+
count: number;
|
|
2027
|
+
bytes: number;
|
|
2028
|
+
items: {
|
|
2029
|
+
name: string;
|
|
2030
|
+
byteLength: number;
|
|
2031
|
+
}[];
|
|
2032
|
+
};
|
|
2033
|
+
|
|
2034
|
+
/**
|
|
2035
|
+
* Deciding what a preview should be doing about audio.
|
|
2036
|
+
*
|
|
2037
|
+
* Pure, and separate from the hook that acts on it, because every rule worth
|
|
2038
|
+
* getting right here is a rule about state rather than about React: whether a
|
|
2039
|
+
* project needs a mix at all, whether the mix on hand is the one the document
|
|
2040
|
+
* describes, whether a result that just arrived is still wanted, and whether
|
|
2041
|
+
* two players have drifted far enough to be worth correcting.
|
|
2042
|
+
*
|
|
2043
|
+
* ## Two sources of sound, never both
|
|
2044
|
+
*
|
|
2045
|
+
* Without music, the video player's own audio is the preview and nothing here
|
|
2046
|
+
* applies. With music, the mix covers the *whole* timeline — including the
|
|
2047
|
+
* audio of the clips themselves — so playing the clips' audio as well would
|
|
2048
|
+
* play everything twice. There is no state in which both are audible, and the
|
|
2049
|
+
* transition between them is the thing this file exists to sequence.
|
|
2050
|
+
*
|
|
2051
|
+
* @experimental
|
|
2052
|
+
*/
|
|
2053
|
+
/** What a preview's audio is doing. @experimental */
|
|
2054
|
+
type PreviewAudioStatus =
|
|
2055
|
+
/** No music in the project. The video player's own audio is the preview. */
|
|
2056
|
+
'source'
|
|
2057
|
+
/** Music exists and the mix for *this* document is not ready. */
|
|
2058
|
+
| 'preparing'
|
|
2059
|
+
/** The mix on hand matches the document. Source audio is muted. */
|
|
2060
|
+
| 'ready'
|
|
2061
|
+
/** The mix could not be prepared. Nothing is played rather than the wrong thing. */
|
|
2062
|
+
| 'failed';
|
|
2063
|
+
/** @experimental */
|
|
2064
|
+
interface PreviewAudioDecision {
|
|
2065
|
+
status: PreviewAudioStatus;
|
|
2066
|
+
/** The key the document wants. Empty when there is no music. */
|
|
2067
|
+
wantedKey: string;
|
|
2068
|
+
/** Whether a job should be started for `wantedKey`. */
|
|
2069
|
+
shouldPrepare: boolean;
|
|
2070
|
+
/** Whether the video player's own audio must be silenced. */
|
|
2071
|
+
muteSourceAudio: boolean;
|
|
2072
|
+
/** Whether playback may run right now. */
|
|
2073
|
+
canPlay: boolean;
|
|
2074
|
+
}
|
|
2075
|
+
/** @experimental */
|
|
2076
|
+
interface PreviewAudioInputs {
|
|
2077
|
+
/** The key the current document compiles to, or '' when it has no music. */
|
|
2078
|
+
wantedKey: string;
|
|
2079
|
+
/** The key of the mix already prepared and loaded, if any. */
|
|
2080
|
+
readyKey: string | null;
|
|
2081
|
+
/** The key a job is currently preparing, if any. */
|
|
2082
|
+
preparingKey: string | null;
|
|
2083
|
+
/** The key whose preparation failed, if any. */
|
|
2084
|
+
failedKey: string | null;
|
|
2085
|
+
}
|
|
2086
|
+
/**
|
|
2087
|
+
* Whether a project needs a mixed preview at all.
|
|
2088
|
+
*
|
|
2089
|
+
* True when anything in the compiled program is audible *and comes from an
|
|
2090
|
+
* audio item* — a project whose only sound is clip audio does not need a mix,
|
|
2091
|
+
* because the video player already plays exactly that and does it in realtime.
|
|
2092
|
+
*
|
|
2093
|
+
* @experimental
|
|
2094
|
+
*/
|
|
2095
|
+
declare function previewAudioNeeded(project: VideoProject): boolean;
|
|
2096
|
+
/**
|
|
2097
|
+
* What to do, given what is wanted and what is on hand.
|
|
2098
|
+
*
|
|
2099
|
+
* @experimental
|
|
2100
|
+
*/
|
|
2101
|
+
declare function decidePreviewAudio(inputs: PreviewAudioInputs): PreviewAudioDecision;
|
|
2102
|
+
/** The key a project's audio compiles to, given resolved sources. @experimental */
|
|
2103
|
+
declare function previewProgramJson(project: VideoProject): string;
|
|
2104
|
+
/**
|
|
2105
|
+
* The default tolerance.
|
|
2106
|
+
*
|
|
2107
|
+
* The drift this used to chase is prevented rather than corrected: `PreviewSync`
|
|
2108
|
+
* holds both transports through a stall and releases them aligned, which is
|
|
2109
|
+
* where all of it came from. A correcting seek during playback costs 200-340 ms
|
|
2110
|
+
* on a Pixel 7 — the same size as the error it was chasing, so it oscillated
|
|
2111
|
+
* between +264 ms and -198 ms — and realignment now happens while everything is
|
|
2112
|
+
* held, where a seek has nothing to chase.
|
|
2113
|
+
*
|
|
2114
|
+
* So this is back to the tolerance the gates were written against. It is a
|
|
2115
|
+
* backstop for something nobody has seen, not the mechanism.
|
|
2116
|
+
*
|
|
2117
|
+
* @experimental
|
|
2118
|
+
*/
|
|
2119
|
+
declare const PREVIEW_DRIFT_TOLERANCE_US = 60000;
|
|
2120
|
+
|
|
2121
|
+
/**
|
|
2122
|
+
* The paired player trace.
|
|
2123
|
+
*
|
|
2124
|
+
* Both players sampled on one monotonic clock, buffered natively and drained
|
|
2125
|
+
* afterwards. It exists because the question — do the picture and the sound
|
|
2126
|
+
* agree — needs samples closer together than a bridge hop, and far closer than
|
|
2127
|
+
* `uiautomator` can time.
|
|
2128
|
+
*
|
|
2129
|
+
* Diagnostics-only in native: a build without them refuses, and even with them
|
|
2130
|
+
* it does nothing until armed.
|
|
2131
|
+
*
|
|
2132
|
+
* @experimental
|
|
2133
|
+
*/
|
|
2134
|
+
/** One sample or event. @experimental */
|
|
2135
|
+
interface TraceEvent {
|
|
2136
|
+
seq: number;
|
|
2137
|
+
/** `System.nanoTime()`, the clock both players publish on. */
|
|
2138
|
+
atNs: number;
|
|
2139
|
+
type: string;
|
|
2140
|
+
v: Record<string, unknown>;
|
|
2141
|
+
}
|
|
2142
|
+
/** @experimental */
|
|
2143
|
+
interface TracePage {
|
|
2144
|
+
ok: boolean;
|
|
2145
|
+
/**
|
|
2146
|
+
* Whether the *run* is trustworthy.
|
|
2147
|
+
*
|
|
2148
|
+
* False when the sampler threw. The events before the fault are real and the
|
|
2149
|
+
* silence after it is not evidence of anything — a summary computed over a
|
|
2150
|
+
* truncated run reads exactly like one computed over a complete one.
|
|
2151
|
+
*/
|
|
2152
|
+
valid: boolean;
|
|
2153
|
+
/** Why it stopped being trustworthy, or null. */
|
|
2154
|
+
fault: string | null;
|
|
2155
|
+
events: TraceEvent[];
|
|
2156
|
+
nextSeq: number;
|
|
2157
|
+
total: number;
|
|
2158
|
+
/** Events overwritten before they could be read. Non-zero means a hole. */
|
|
2159
|
+
dropped: number;
|
|
2160
|
+
}
|
|
2161
|
+
/** Arm the trace and start sampling. @experimental */
|
|
2162
|
+
declare function armPreviewTrace(on: boolean, intervalMs?: number): boolean;
|
|
2163
|
+
/** Label the trace, so a harness can find its own cases. @experimental */
|
|
2164
|
+
declare function markPreviewTrace(label: string): void;
|
|
2165
|
+
/**
|
|
2166
|
+
* Drain a page of events.
|
|
2167
|
+
*
|
|
2168
|
+
* Paged rather than whole: a full trace is far past what a single log line
|
|
2169
|
+
* survives, and truncation in the middle is the failure this is meant to
|
|
2170
|
+
* detect rather than cause.
|
|
2171
|
+
*
|
|
2172
|
+
* @experimental
|
|
2173
|
+
*/
|
|
2174
|
+
declare function drainPreviewTrace(fromSeq: number, limit?: number): TracePage;
|
|
2175
|
+
/**
|
|
2176
|
+
* Every event, drained in pages until the buffer is exhausted.
|
|
2177
|
+
*
|
|
2178
|
+
* @experimental
|
|
2179
|
+
*/
|
|
2180
|
+
declare function drainAllPreviewTrace(limit?: number): TracePage;
|
|
2181
|
+
/**
|
|
2182
|
+
* The offset between the players, summarised over the samples that mattered.
|
|
2183
|
+
*
|
|
2184
|
+
* Only samples where **both** players were playing are counted: a paused
|
|
2185
|
+
* player's position does not advance, so including it would average in a
|
|
2186
|
+
* divergence that is not drift.
|
|
2187
|
+
*
|
|
2188
|
+
* @experimental
|
|
2189
|
+
*/
|
|
2190
|
+
declare function summariseOffsets(events: readonly TraceEvent[]): {
|
|
2191
|
+
samples: number;
|
|
2192
|
+
playingSamples: number;
|
|
2193
|
+
/** Samples whose presented frame was the same one as the tick before. */
|
|
2194
|
+
staleFrameSamples: number;
|
|
2195
|
+
/**
|
|
2196
|
+
* Samples with no presented frame at all.
|
|
2197
|
+
*
|
|
2198
|
+
* `FrameClock.latest` is null until the renderer consumes something, and
|
|
2199
|
+
* again after a generation bump. Counting these as "stale" hid them: a run
|
|
2200
|
+
* reported 56 stale of 506 while every sampled point in the same run had no
|
|
2201
|
+
* record whatsoever, because an absent field is not a false one.
|
|
2202
|
+
*/
|
|
2203
|
+
absentFrameSamples: number;
|
|
2204
|
+
/** Samples with a frame that had advanced since the previous tick. */
|
|
2205
|
+
freshFrameSamples: number;
|
|
2206
|
+
/** The longest run of consecutive stale samples. */
|
|
2207
|
+
longestStaleRun: number;
|
|
2208
|
+
minUs: number;
|
|
2209
|
+
maxUs: number;
|
|
2210
|
+
meanUs: number;
|
|
2211
|
+
worstAbsUs: number;
|
|
2212
|
+
};
|
|
2213
|
+
/**
|
|
2214
|
+
* What the stall coordinator has done.
|
|
2215
|
+
*
|
|
2216
|
+
* Holds, releases and realigning seeks. A gate asserts that seams produced
|
|
2217
|
+
* holds and that every hold resolved — a coordinator that never engaged and one
|
|
2218
|
+
* that engaged and never released look identical in an offset summary.
|
|
2219
|
+
*
|
|
2220
|
+
* @experimental
|
|
2221
|
+
*/
|
|
2222
|
+
declare function previewSyncCounters(): Record<string, unknown>;
|
|
2223
|
+
/**
|
|
2224
|
+
* Stop playback for a host reason — background, picker, unmount.
|
|
2225
|
+
*
|
|
2226
|
+
* Overrides a resume the coordinator is holding, so playback cannot restart at
|
|
2227
|
+
* a moment nobody asked for.
|
|
2228
|
+
*
|
|
2229
|
+
* @experimental
|
|
2230
|
+
*/
|
|
2231
|
+
declare function stopPreviewForHost(reason: string): void;
|
|
2232
|
+
/**
|
|
2233
|
+
* Frames offered, queued and drawn.
|
|
2234
|
+
*
|
|
2235
|
+
* The instrument-disruption check reads this with tracing off and again with it
|
|
2236
|
+
* on: if the drawn count advances at the same rate either way, the trace is not
|
|
2237
|
+
* changing what it measures. Three defects in this area were the measurement
|
|
2238
|
+
* rather than the product, so that check comes before the numbers do.
|
|
2239
|
+
*
|
|
2240
|
+
* @experimental
|
|
2241
|
+
*/
|
|
2242
|
+
declare function previewFramePipeline(): Record<string, number>;
|
|
2243
|
+
/** Which parts of the music preview exist. All true in production. @experimental */
|
|
2244
|
+
interface PreviewParts {
|
|
2245
|
+
/** Whether the drift check may act. Off measures without correcting. */
|
|
2246
|
+
driftCorrection: boolean;
|
|
2247
|
+
loadPlayer: boolean;
|
|
2248
|
+
startMix: boolean;
|
|
2249
|
+
positionPump: boolean;
|
|
2250
|
+
coordinator: boolean;
|
|
2251
|
+
}
|
|
2252
|
+
/**
|
|
2253
|
+
* Turn parts of the music preview off, to bisect a failure.
|
|
2254
|
+
*
|
|
2255
|
+
* Gate-only in native. Guessing between a second player, its position pump, the
|
|
2256
|
+
* stall coordinator and the mix actually playing is how the previous attempts
|
|
2257
|
+
* in this area became tuning; this is how one of them gets ruled out at a time.
|
|
2258
|
+
*
|
|
2259
|
+
* @experimental
|
|
2260
|
+
*/
|
|
2261
|
+
declare function setPreviewParts(parts: Partial<PreviewParts>): PreviewParts | null;
|
|
2262
|
+
/** @experimental */
|
|
2263
|
+
declare function previewParts(): PreviewParts | null;
|
|
2264
|
+
/**
|
|
2265
|
+
* Silence the video player's own audio.
|
|
2266
|
+
*
|
|
2267
|
+
* Exposed because a bisect control needs the *same* mute in every
|
|
2268
|
+
* configuration: without it, the video-only row would be the only one with
|
|
2269
|
+
* audible clip audio, and any difference against the others would be
|
|
2270
|
+
* confounded by a whole track the rest do not have.
|
|
2271
|
+
*
|
|
2272
|
+
* @experimental
|
|
2273
|
+
*/
|
|
2274
|
+
declare function muteSourceAudio(muted: boolean): void;
|
|
2275
|
+
/**
|
|
2276
|
+
* Every codec mix preparation owns, and what became of it.
|
|
2277
|
+
*
|
|
2278
|
+
* A `release()` that was called is not a release that finished, so both ends
|
|
2279
|
+
* are recorded along with the exception path. The preview video decoder is not
|
|
2280
|
+
* here: it belongs to the player and outlives any job, and counting it would
|
|
2281
|
+
* make "live after publication" always true.
|
|
2282
|
+
*
|
|
2283
|
+
* @experimental
|
|
2284
|
+
*/
|
|
2285
|
+
declare function previewCodecLedger(): {
|
|
2286
|
+
live: number;
|
|
2287
|
+
peakLive: number;
|
|
2288
|
+
total: number;
|
|
2289
|
+
entries: Record<string, unknown>[];
|
|
2290
|
+
};
|
|
2291
|
+
|
|
2292
|
+
/**
|
|
2293
|
+
* The offset, segmented by the marks a gate put in the run.
|
|
2294
|
+
*
|
|
2295
|
+
* ## Why a whole-run summary is not enough
|
|
2296
|
+
*
|
|
2297
|
+
* `summariseOffsets` answers "how bad did it get", and a gate needs "where".
|
|
2298
|
+
* Measured on a Pixel 7, a run whose worst excursion was 266 ms spent every
|
|
2299
|
+
* moment between its transitions inside ±40 ms: the whole-run worst described
|
|
2300
|
+
* three settling transients and nothing else, and a single number cannot tell
|
|
2301
|
+
* that from a steady 266 ms error. They are not the same defect and they do not
|
|
2302
|
+
* have the same fix.
|
|
2303
|
+
*
|
|
2304
|
+
* So a run is cut at its marks, and each segment is summarised on its own. The
|
|
2305
|
+
* gate verdict is still the run's worst — this localises a failure, it does not
|
|
2306
|
+
* soften one.
|
|
2307
|
+
*
|
|
2308
|
+
* ## Fresh and stale are reported, never traded
|
|
2309
|
+
*
|
|
2310
|
+
* A stale presented frame is what a person actually sees, so its offset is real
|
|
2311
|
+
* and is counted in `worstAbsUs`. But a stale frame's offset grows purely
|
|
2312
|
+
* because audio advanced while the picture did not, which is a different fault
|
|
2313
|
+
* from two clocks disagreeing — so `freshWorstAbsUs` is reported beside it
|
|
2314
|
+
* rather than instead of it. Neither is allowed to stand alone: quoting only
|
|
2315
|
+
* the fresh figure is how a run with a nine-second frozen picture passes a
|
|
2316
|
+
* synchronisation gate.
|
|
2317
|
+
*
|
|
2318
|
+
* @experimental
|
|
2319
|
+
*/
|
|
2320
|
+
interface PhaseSummary {
|
|
2321
|
+
/** The mark that opened this segment, or `'start'` before the first one. */
|
|
2322
|
+
label: string;
|
|
2323
|
+
index: number;
|
|
2324
|
+
/** Samples in the segment, whatever their state. */
|
|
2325
|
+
samples: number;
|
|
2326
|
+
/** Samples with both players playing — the only ones an offset means anything for. */
|
|
2327
|
+
playingSamples: number;
|
|
2328
|
+
/** Of those, the ones whose presented frame had advanced. */
|
|
2329
|
+
freshPlayingSamples: number;
|
|
2330
|
+
stalePlayingSamples: number;
|
|
2331
|
+
/** The longest unbroken run of stale samples, in samples. */
|
|
2332
|
+
longestStaleRun: number;
|
|
2333
|
+
minUs: number;
|
|
2334
|
+
maxUs: number;
|
|
2335
|
+
meanUs: number;
|
|
2336
|
+
/** Worst absolute offset over every playing sample, stale included. */
|
|
2337
|
+
worstAbsUs: number;
|
|
2338
|
+
/** Worst over fresh samples only. Reported beside `worstAbsUs`, never instead. */
|
|
2339
|
+
freshWorstAbsUs: number;
|
|
2340
|
+
/** Playing samples outside the tolerance. */
|
|
2341
|
+
overToleranceSamples: number;
|
|
2342
|
+
/**
|
|
2343
|
+
* How long the segment took to come inside tolerance and stay there.
|
|
2344
|
+
*
|
|
2345
|
+
* `-1` when it never did, which is the difference between a transient and a
|
|
2346
|
+
* failure that happens to start at a transition. Measured from the segment's
|
|
2347
|
+
* first playing sample, so it is a settling time rather than a wall clock.
|
|
2348
|
+
*/
|
|
2349
|
+
settledAfterUs: number;
|
|
2350
|
+
}
|
|
2351
|
+
/** @experimental */
|
|
2352
|
+
interface PhaseReport {
|
|
2353
|
+
toleranceUs: number;
|
|
2354
|
+
phases: PhaseSummary[];
|
|
2355
|
+
/** The run's worst, over every phase. This is the gate. */
|
|
2356
|
+
worstAbsUs: number;
|
|
2357
|
+
freshWorstAbsUs: number;
|
|
2358
|
+
overToleranceSamples: number;
|
|
2359
|
+
/** Whether every playing sample in the run was inside tolerance. */
|
|
2360
|
+
pass: boolean;
|
|
2361
|
+
}
|
|
2362
|
+
/**
|
|
2363
|
+
* The prefix a gate's own marks carry.
|
|
2364
|
+
*
|
|
2365
|
+
* Every mark is recorded as a `mark` event carrying its label, and the trace is
|
|
2366
|
+
* also full of system events — holds, releases, drift corrections, transport
|
|
2367
|
+
* transitions. Cutting on "not a sample" would start a new phase at every one
|
|
2368
|
+
* of those, and cutting on every mark would let an ad-hoc label reshape a gate's
|
|
2369
|
+
* phases. So only marks whose label carries this prefix cut the run.
|
|
2370
|
+
*
|
|
2371
|
+
* @experimental
|
|
2372
|
+
*/
|
|
2373
|
+
declare const PHASE_MARK_PREFIX = "phase:";
|
|
2374
|
+
/**
|
|
2375
|
+
* Summarise a drained run, cut at its mark events.
|
|
2376
|
+
*
|
|
2377
|
+
* @experimental
|
|
2378
|
+
*/
|
|
2379
|
+
declare function summarisePhases(events: readonly TraceEvent[], toleranceUs: number): PhaseReport;
|
|
2380
|
+
|
|
2381
|
+
/**
|
|
2382
|
+
* The shared-clock playback spike, from the host's side.
|
|
2383
|
+
*
|
|
2384
|
+
* ## What it is testing
|
|
2385
|
+
*
|
|
2386
|
+
* Whether one playback-clock authority can drive the picture and a prepared,
|
|
2387
|
+
* project-wide mix at once. The two-player preview cannot meet the 60 ms gate
|
|
2388
|
+
* and the reasons are structural: anchors polled at 50 ms make a single
|
|
2389
|
+
* comparison uncertain by up to ±56 ms against a 60 ms tolerance, the
|
|
2390
|
+
* correction loop runs at 500 ms while each corrective seek costs 200-400 ms,
|
|
2391
|
+
* and at a seam the project clock divides by a speed the player applies about
|
|
2392
|
+
* 450 ms later.
|
|
2393
|
+
*
|
|
2394
|
+
* ## Why the spec is built here
|
|
2395
|
+
*
|
|
2396
|
+
* `playlistOf` is the shared compiler, and it already decides every number this
|
|
2397
|
+
* needs: the source bounds a clip plays between, its speed, and where the
|
|
2398
|
+
* window sits on the project timeline. Passing its output through verbatim is
|
|
2399
|
+
* what keeps the adapter free of a second interpretation of the document —
|
|
2400
|
+
* native maps items onto media items and nothing else.
|
|
2401
|
+
*
|
|
2402
|
+
* Diagnostics-only, and it says nothing about live PCM gain or fades: it plays
|
|
2403
|
+
* the same prepared mix file the current preview plays.
|
|
2404
|
+
*
|
|
2405
|
+
* @experimental
|
|
2406
|
+
*/
|
|
2407
|
+
/** One clip, as the compiler described it. @experimental */
|
|
2408
|
+
interface CompositionSpikeItem {
|
|
2409
|
+
uri: string;
|
|
2410
|
+
/** Clipping bounds in **source** time, which is where speed is applied. */
|
|
2411
|
+
startUs: number;
|
|
2412
|
+
endUs: number;
|
|
2413
|
+
speed: number;
|
|
2414
|
+
}
|
|
2415
|
+
/** @experimental */
|
|
2416
|
+
interface CompositionSpikeSpec {
|
|
2417
|
+
/**
|
|
2418
|
+
* Hold the shared clock at each clip boundary.
|
|
2419
|
+
*
|
|
2420
|
+
* Off, and measured off for a reason: with `playWhenReady` false
|
|
2421
|
+
* `CompositionPlayer` does not advance across an item boundary, so the
|
|
2422
|
+
* incoming frame the hold waits for is never produced and the hold runs to
|
|
2423
|
+
* its bound. Holding the clock and preparing the next stream are mutually
|
|
2424
|
+
* exclusive on this player.
|
|
2425
|
+
*/
|
|
2426
|
+
seamHold?: boolean;
|
|
2427
|
+
/**
|
|
2428
|
+
* Prepare the incoming item with an explicit seek while the clock is held.
|
|
2429
|
+
*
|
|
2430
|
+
* Only meaningful with `seamHold`. Pausing natural traversal and seeking to a
|
|
2431
|
+
* target are different operations: the seek path resets the renderers, which
|
|
2432
|
+
* is what makes a paused player produce a frame, and is the same path the
|
|
2433
|
+
* existing seek readiness already passes on.
|
|
2434
|
+
*/
|
|
2435
|
+
seamSeek?: boolean;
|
|
2436
|
+
items: CompositionSpikeItem[];
|
|
2437
|
+
/** The prepared mix. Empty plays the picture with no sound at all. */
|
|
2438
|
+
mixUri: string;
|
|
2439
|
+
/**
|
|
2440
|
+
* The mix's duration.
|
|
2441
|
+
*
|
|
2442
|
+
* Required whenever `mixUri` is set: the player computes each sequence's
|
|
2443
|
+
* duration before playing, and an audio-only item cannot report one itself.
|
|
2444
|
+
*/
|
|
2445
|
+
mixDurationUs: number;
|
|
2446
|
+
width: number;
|
|
2447
|
+
height: number;
|
|
2448
|
+
/**
|
|
2449
|
+
* The request this attachment belongs to.
|
|
2450
|
+
*
|
|
2451
|
+
* Claimed before preparation starts. Native refuses an attachment whose id is
|
|
2452
|
+
* older than the newest claim, so a preparation that finishes late cannot put
|
|
2453
|
+
* a stale mix on screen for even one frame. Omit for runs with no race.
|
|
2454
|
+
*/
|
|
2455
|
+
requestId?: number;
|
|
2456
|
+
}
|
|
2457
|
+
/** @experimental */
|
|
2458
|
+
interface CompositionSpikeState {
|
|
2459
|
+
ok: boolean;
|
|
2460
|
+
attached: boolean;
|
|
2461
|
+
built: number;
|
|
2462
|
+
released: number;
|
|
2463
|
+
error: string | null;
|
|
2464
|
+
/**
|
|
2465
|
+
* Readiness latency, reported apart from synchronisation.
|
|
2466
|
+
*
|
|
2467
|
+
* A preparation pause is a latency figure and an A/V offset is an error;
|
|
2468
|
+
* averaging one into the other is how a slow start reads as drift.
|
|
2469
|
+
*/
|
|
2470
|
+
startupMs: number;
|
|
2471
|
+
seekMs: number;
|
|
2472
|
+
/** What the person asked for, independent of whether the transport is running. */
|
|
2473
|
+
intentPlaying?: boolean;
|
|
2474
|
+
/** A hold waiting for the first frame of the current epoch. */
|
|
2475
|
+
releasePending?: boolean;
|
|
2476
|
+
releases?: number;
|
|
2477
|
+
readinessTimeouts?: number;
|
|
2478
|
+
readinessFailure?: string | null;
|
|
2479
|
+
/** How long the last hold lasted before playback was released. */
|
|
2480
|
+
holdMs?: number;
|
|
2481
|
+
/**
|
|
2482
|
+
* Wall-clock milliseconds spent holding the clock at each clip boundary.
|
|
2483
|
+
*
|
|
2484
|
+
* Listed, never averaged. Each entry is a pause the person sees: the decoder
|
|
2485
|
+
* is torn down and rebuilt at a boundary and the clock waits rather than the
|
|
2486
|
+
* picture falling behind it. **This is not seamless playback**, which is why
|
|
2487
|
+
* it is reported apart from every synchronisation figure.
|
|
2488
|
+
*/
|
|
2489
|
+
seamHoldMs?: number[];
|
|
2490
|
+
/** Holds that reached their bound without a correctly associated frame. */
|
|
2491
|
+
seamHoldTimeouts?: number;
|
|
2492
|
+
seamHoldActive?: boolean;
|
|
2493
|
+
atNs?: number;
|
|
2494
|
+
positionUs?: number;
|
|
2495
|
+
playing?: boolean;
|
|
2496
|
+
playbackState?: number;
|
|
2497
|
+
durationUs?: number;
|
|
2498
|
+
presentedUs?: number | null;
|
|
2499
|
+
/** How old the presented frame was when it was read, so a stalled sample can be excluded. */
|
|
2500
|
+
presentedAgeNs?: number;
|
|
2501
|
+
/** How long the paired read itself took. Bounds the pairing error. */
|
|
2502
|
+
readSpanNs?: number;
|
|
2503
|
+
/** How stale the pairing was when it was read out. */
|
|
2504
|
+
pairingAgeNs?: number;
|
|
2505
|
+
}
|
|
2506
|
+
/**
|
|
2507
|
+
* Build the spike's spec from a project and a prepared mix.
|
|
2508
|
+
*
|
|
2509
|
+
* Unsupported clips are not silently dropped: `playlistOf` reports them, and a
|
|
2510
|
+
* spec built from a playlist missing a window would put every later seam in the
|
|
2511
|
+
* wrong place — which is exactly the class of error this spike exists to
|
|
2512
|
+
* measure.
|
|
2513
|
+
*
|
|
2514
|
+
* @experimental
|
|
2515
|
+
*/
|
|
2516
|
+
declare function compositionSpikeSpec(project: VideoProject, mix: {
|
|
2517
|
+
uri: string;
|
|
2518
|
+
durationUs: number;
|
|
2519
|
+
}, resolve: (source: {
|
|
2520
|
+
kind: string;
|
|
2521
|
+
uri?: string;
|
|
2522
|
+
}) => string | null): {
|
|
2523
|
+
spec: CompositionSpikeSpec;
|
|
2524
|
+
unsupported: string[];
|
|
2525
|
+
};
|
|
2526
|
+
/** Claim the next request id, before preparing. @experimental */
|
|
2527
|
+
declare function spikeClaim(): number;
|
|
2528
|
+
/** @experimental */
|
|
2529
|
+
declare function spikeStart(spec: CompositionSpikeSpec): Record<string, unknown>;
|
|
2530
|
+
/** @experimental */
|
|
2531
|
+
declare function spikePlay(): Record<string, unknown>;
|
|
2532
|
+
/** @experimental */
|
|
2533
|
+
declare function spikePause(): Record<string, unknown>;
|
|
2534
|
+
/** @experimental */
|
|
2535
|
+
declare function spikeSeek(positionUs: number): Record<string, unknown>;
|
|
2536
|
+
/** @experimental */
|
|
2537
|
+
declare function spikeState(): CompositionSpikeState;
|
|
2538
|
+
/**
|
|
2539
|
+
* One retained sample, unpacked.
|
|
2540
|
+
*
|
|
2541
|
+
* Packed across the boundary because a gate reads thousands of these, and named
|
|
2542
|
+
* here because a gate that indexes into an array of eighteen longs is one
|
|
2543
|
+
* transposition away from a confident wrong answer.
|
|
2544
|
+
*
|
|
2545
|
+
* @experimental
|
|
2546
|
+
*/
|
|
2547
|
+
interface SpikeSample {
|
|
2548
|
+
seq: number;
|
|
2549
|
+
atNs: number;
|
|
2550
|
+
positionUs: number;
|
|
2551
|
+
playing: boolean;
|
|
2552
|
+
ready: boolean;
|
|
2553
|
+
epoch: number;
|
|
2554
|
+
sinceEpochNs: number;
|
|
2555
|
+
frameId: number;
|
|
2556
|
+
frameGeneration: number;
|
|
2557
|
+
frameWindow: number;
|
|
2558
|
+
framePtsUs: number;
|
|
2559
|
+
/**
|
|
2560
|
+
* When Media3's renderer said this frame was about to go to the surface.
|
|
2561
|
+
*
|
|
2562
|
+
* The earliest boundary observable here, and the one that separates the
|
|
2563
|
+
* owners of a gap: nothing before this is the decoder or the item's
|
|
2564
|
+
* preparation, this to {@link SpikeSample.frameAvailableNs} is the video
|
|
2565
|
+
* graph, and that to {@link SpikeSample.frameConsumedNs} is this consumer.
|
|
2566
|
+
*/
|
|
2567
|
+
frameMetadataNs: number;
|
|
2568
|
+
/** The time Media3 *asked* the surface to present at. An intention. */
|
|
2569
|
+
frameReleaseNs: number;
|
|
2570
|
+
/** The time the consumer latched it. The last boundary observable here. */
|
|
2571
|
+
frameConsumedNs: number;
|
|
2572
|
+
frameAvailableNs: number;
|
|
2573
|
+
coalesced: number;
|
|
2574
|
+
offered: number;
|
|
2575
|
+
available: number;
|
|
2576
|
+
taken: number;
|
|
2577
|
+
readSpanNs: number;
|
|
2578
|
+
}
|
|
2579
|
+
/** @experimental */
|
|
2580
|
+
declare function spikeDrain(fromSeq: number, limit?: number): {
|
|
2581
|
+
ok: boolean;
|
|
2582
|
+
rows: SpikeSample[];
|
|
2583
|
+
nextSeq: number;
|
|
2584
|
+
total: number;
|
|
2585
|
+
dropped: number;
|
|
2586
|
+
};
|
|
2587
|
+
/** @experimental */
|
|
2588
|
+
declare function spikeRelease(): Record<string, unknown>;
|
|
2589
|
+
|
|
2590
|
+
/**
|
|
2591
|
+
* Where a shared-clock run left the tolerance, and why.
|
|
2592
|
+
*
|
|
2593
|
+
* ## What is being compared
|
|
2594
|
+
*
|
|
2595
|
+
* The audio clock's position and the presented frame's timestamp, brought to
|
|
2596
|
+
* one instant. The frame was latched at `frameConsumedNs`; the position was
|
|
2597
|
+
* read at `atNs`. Rolling the position back to the latch is what puts them on
|
|
2598
|
+
* the same monotonic instant — the composition runs at project rate 1x, so the
|
|
2599
|
+
* roll-back is the elapsed time itself.
|
|
2600
|
+
*
|
|
2601
|
+
* ## Which boundary this observes
|
|
2602
|
+
*
|
|
2603
|
+
* The **latch**, not the display. `frameConsumedNs` is when the consumer called
|
|
2604
|
+
* `updateTexImage`; `frameReleaseNs` is the time Media3 *asked* the surface to
|
|
2605
|
+
* present at, which is an intention and not an acknowledgement. Actual scanout
|
|
2606
|
+
* happens one compositor pass later and is not observed anywhere in this path.
|
|
2607
|
+
* So an offset here is "audio against latched picture", and the display adds a
|
|
2608
|
+
* further unmeasured delay common to every sample.
|
|
2609
|
+
*
|
|
2610
|
+
* ## Uncertainty is stated, never subtracted
|
|
2611
|
+
*
|
|
2612
|
+
* Comparing a 30 fps frame's timestamp against a continuous clock carries an
|
|
2613
|
+
* irreducible ±16.7 ms, and the paired read adds a few more. That is reported
|
|
2614
|
+
* beside the verdict. It is not deducted from an excursion to bring it under
|
|
2615
|
+
* the line: a 95 ms excursion is not a 78 ms excursion because the instrument
|
|
2616
|
+
* is imperfect.
|
|
2617
|
+
*
|
|
2618
|
+
* @experimental
|
|
2619
|
+
*/
|
|
2620
|
+
/** @experimental */
|
|
2621
|
+
declare const SPIKE_TOLERANCE_US = 60000;
|
|
2622
|
+
/**
|
|
2623
|
+
* Why one sample left the tolerance.
|
|
2624
|
+
*
|
|
2625
|
+
* Ordered by precedence, because more than one can be true and the first is the
|
|
2626
|
+
* one that explains the sample: a frame from before a seek is cross-epoch
|
|
2627
|
+
* whatever else is also true of it.
|
|
2628
|
+
*
|
|
2629
|
+
* @experimental
|
|
2630
|
+
*/
|
|
2631
|
+
type ExcursionCause =
|
|
2632
|
+
/**
|
|
2633
|
+
* The latched frame predates the moment playback was released.
|
|
2634
|
+
*
|
|
2635
|
+
* Not a filter and not a drift. The readiness gate releases on a frame whose
|
|
2636
|
+
* *position* matches the epoch, and that frame may have been latched long
|
|
2637
|
+
* before — measured at 13.9 s, the whole interval between attaching the
|
|
2638
|
+
* composition and the person pressing play. The picture was right; the
|
|
2639
|
+
* frame's wall-clock age simply includes a pause, and charging that to
|
|
2640
|
+
* synchronisation measures the pause.
|
|
2641
|
+
*
|
|
2642
|
+
* Reported rather than excluded, because the size of it is the size of the
|
|
2643
|
+
* interval in which the first sample after a release is not yet a
|
|
2644
|
+
* synchronisation reading.
|
|
2645
|
+
*/
|
|
2646
|
+
'pre-release-frame'
|
|
2647
|
+
/** The frame belongs to a different transition epoch or frame generation. */
|
|
2648
|
+
| 'cross-epoch'
|
|
2649
|
+
/** The frame was latched long enough ago that it is not a current reading. */
|
|
2650
|
+
| 'stale-pairing'
|
|
2651
|
+
/** The first frames after a seek or after readiness, before the picture caught up. */
|
|
2652
|
+
| 'first-frame-after-seek'
|
|
2653
|
+
/** The frame was latched well after the time it was asked to be presented at. */
|
|
2654
|
+
| 'delayed-presentation'
|
|
2655
|
+
/** The audio clock moved discontinuously between adjacent samples. */
|
|
2656
|
+
| 'audio-discontinuity'
|
|
2657
|
+
/** The mapped project time disagrees with the frame's own window and PTS. */
|
|
2658
|
+
| 'mapping'
|
|
2659
|
+
/** None of the above fitted. Deliberately not folded into a neighbour. */
|
|
2660
|
+
| 'unexplained';
|
|
2661
|
+
/** @experimental */
|
|
2662
|
+
interface Excursion {
|
|
2663
|
+
seq: number;
|
|
2664
|
+
/** Against the latch: observed, and earlier than display. */
|
|
2665
|
+
offsetUs: number;
|
|
2666
|
+
/** Against the time the frame was asked to appear. An intention. */
|
|
2667
|
+
offsetAtIntendedUs: number;
|
|
2668
|
+
cause: ExcursionCause;
|
|
2669
|
+
epoch: number;
|
|
2670
|
+
sinceEpochMs: number;
|
|
2671
|
+
frameWindow: number;
|
|
2672
|
+
framePtsUs: number;
|
|
2673
|
+
/** Latched minus asked-for. Large means the picture arrived late. */
|
|
2674
|
+
latchLagUs: number;
|
|
2675
|
+
/** How old the latched frame was when the position was read. */
|
|
2676
|
+
frameAgeUs: number;
|
|
2677
|
+
/** The audio clock's step from the previous sample, in microseconds. */
|
|
2678
|
+
positionStepUs: number;
|
|
2679
|
+
/** The wall-clock step from the previous sample. */
|
|
2680
|
+
wallStepUs: number;
|
|
2681
|
+
}
|
|
2682
|
+
/** @experimental */
|
|
2683
|
+
interface ExcursionReport {
|
|
2684
|
+
toleranceUs: number;
|
|
2685
|
+
/** Samples where playback was declared active. Only these are judged. */
|
|
2686
|
+
activeSamples: number;
|
|
2687
|
+
/** Samples before readiness, exposed rather than dropped. */
|
|
2688
|
+
preparingSamples: number;
|
|
2689
|
+
excursions: Excursion[];
|
|
2690
|
+
byCause: Record<string, number>;
|
|
2691
|
+
/**
|
|
2692
|
+
* Samples still showing a frame latched before the release.
|
|
2693
|
+
*
|
|
2694
|
+
* Reported apart, and *retained* in the acceptance set: they are real
|
|
2695
|
+
* playback, and if the picture never catches up they are the only thing that
|
|
2696
|
+
* says so.
|
|
2697
|
+
*/
|
|
2698
|
+
preReleaseSamples: number;
|
|
2699
|
+
preReleaseWorstUs: number;
|
|
2700
|
+
worstAbsUs: number;
|
|
2701
|
+
/** The same run judged at the intended-presentation boundary. */
|
|
2702
|
+
worstAtIntendedUs: number;
|
|
2703
|
+
overAtIntended: number;
|
|
2704
|
+
passAtIntended: boolean;
|
|
2705
|
+
/** Mean latch-minus-intended over the excursions. Negative means early. */
|
|
2706
|
+
meanLatchLagUs: number;
|
|
2707
|
+
pass: boolean;
|
|
2708
|
+
uncertainty: {
|
|
2709
|
+
/** Half a frame interval at the measured rate. Irreducible. */
|
|
2710
|
+
frameQuantumUs: number;
|
|
2711
|
+
/** The widest paired read in the run. */
|
|
2712
|
+
readSpanUs: number;
|
|
2713
|
+
positionQuantumUs: number;
|
|
2714
|
+
};
|
|
2715
|
+
}
|
|
2716
|
+
/**
|
|
2717
|
+
* The offset at each observable boundary.
|
|
2718
|
+
*
|
|
2719
|
+
* They are not the same instant and the difference is not noise. Measured on a
|
|
2720
|
+
* Pixel 7, the consumer latches a buffer about 34 ms *before* the time Media3
|
|
2721
|
+
* asked the surface to present it — `releaseOutputBuffer(index, releaseTimeNs)`
|
|
2722
|
+
* queues immediately with a desired present time in the future, and the
|
|
2723
|
+
* SurfaceTexture consumer takes it as soon as it is queued.
|
|
2724
|
+
*
|
|
2725
|
+
* So `atLatch` compares the audio clock against a frame that is in the texture
|
|
2726
|
+
* but has not been displayed, and is systematically early by that gap;
|
|
2727
|
+
* `atIntended` compares it against when that frame was *meant* to appear, which
|
|
2728
|
+
* is the best available estimate of what a viewer experiences and is still an
|
|
2729
|
+
* intention rather than an acknowledgement. The true display instant lies at or
|
|
2730
|
+
* after `atIntended`, and nothing in this process observes it.
|
|
2731
|
+
*
|
|
2732
|
+
* Both are returned because choosing one silently is how a boundary decision
|
|
2733
|
+
* becomes a verdict.
|
|
2734
|
+
*
|
|
2735
|
+
* @experimental
|
|
2736
|
+
*/
|
|
2737
|
+
declare function offsetAt(sample: SpikeSample,
|
|
2738
|
+
/**
|
|
2739
|
+
* When the current playing interval was released, if known.
|
|
2740
|
+
*
|
|
2741
|
+
* A frame latched while paused carries the pause in its age. Charging that to
|
|
2742
|
+
* synchronisation measured a 13.9 second "error" for a picture that was
|
|
2743
|
+
* showing exactly the right position. Clamping the reference to the release
|
|
2744
|
+
* instant charges only the time playback has actually been running — so at
|
|
2745
|
+
* release the offset is (audio position - displayed PTS), and it grows from
|
|
2746
|
+
* there if the audio advances while the picture stays put.
|
|
2747
|
+
*
|
|
2748
|
+
* That growth is the point: this corrects the baseline without removing
|
|
2749
|
+
* coverage. A frame that is never replaced still accrues the offset it
|
|
2750
|
+
* deserves.
|
|
2751
|
+
*/
|
|
2752
|
+
releaseAtNs?: number): {
|
|
2753
|
+
atLatch: number;
|
|
2754
|
+
atIntended: number;
|
|
2755
|
+
};
|
|
2756
|
+
/**
|
|
2757
|
+
* Classify a run's over-tolerance samples.
|
|
2758
|
+
*
|
|
2759
|
+
* `frameIntervalUs` is the source's nominal frame interval; half of it is the
|
|
2760
|
+
* irreducible quantisation of comparing a discrete frame against a continuous
|
|
2761
|
+
* clock.
|
|
2762
|
+
*
|
|
2763
|
+
* @experimental
|
|
2764
|
+
*/
|
|
2765
|
+
declare function classifyExcursions(samples: readonly SpikeSample[], options?: {
|
|
2766
|
+
toleranceUs?: number;
|
|
2767
|
+
frameIntervalUs?: number;
|
|
2768
|
+
}): ExcursionReport;
|
|
2769
|
+
/**
|
|
2770
|
+
* The samples around one excursion, for looking at rather than inferring from.
|
|
2771
|
+
*
|
|
2772
|
+
* @experimental
|
|
2773
|
+
*/
|
|
2774
|
+
declare function windowAround(samples: readonly SpikeSample[], seq: number, spanUs?: number): SpikeSample[];
|
|
2775
|
+
|
|
2776
|
+
/**
|
|
2777
|
+
* Score the picture where it is handed to composition.
|
|
2778
|
+
*
|
|
2779
|
+
* ## The boundary, stated
|
|
2780
|
+
*
|
|
2781
|
+
* **Submission**, not scanout. A frame is scored at the instant composition
|
|
2782
|
+
* begins consuming it; the compositor presents it some time later and nothing
|
|
2783
|
+
* in this process observes that. The delay is common to every frame, so it
|
|
2784
|
+
* shifts the whole series rather than spreading it — but it is not measured
|
|
2785
|
+
* here and is not subtracted from anything.
|
|
2786
|
+
*
|
|
2787
|
+
* ## Why an interval rather than a point
|
|
2788
|
+
*
|
|
2789
|
+
* A frame is not a sample of one instant; it is the correct picture for a span
|
|
2790
|
+
* of project time. Comparing a continuous audio clock against a single
|
|
2791
|
+
* timestamp charges the frame's own duration to synchronisation, which is why
|
|
2792
|
+
* the 0.5x clip — whose frames are correct for 66.7 ms each — looked worse than
|
|
2793
|
+
* the 1x clip at identical quality.
|
|
2794
|
+
*
|
|
2795
|
+
* So a submitted frame carries `[ptsUs, ptsUs + intervalUs)`, and the error is
|
|
2796
|
+
* how far outside that span the audio clock sits. Zero inside, and the distance
|
|
2797
|
+
* to the nearer edge outside. The interval comes from the measured cadence,
|
|
2798
|
+
* which for a clip of speed `s` and source interval `i` is `i / s` — 33.3 ms at
|
|
2799
|
+
* 1x, 66.7 ms at 0.5x.
|
|
2800
|
+
*
|
|
2801
|
+
* @experimental
|
|
2802
|
+
*/
|
|
2803
|
+
/** One submission, as the consumer recorded it. @experimental */
|
|
2804
|
+
interface SubmissionRecord {
|
|
2805
|
+
frameId: number;
|
|
2806
|
+
epoch: number;
|
|
2807
|
+
ptsUs: number;
|
|
2808
|
+
intervalUs: number;
|
|
2809
|
+
submittedNs: number;
|
|
2810
|
+
/** What the texture reported holding at acquisition. */
|
|
2811
|
+
texturePtsUs: number;
|
|
2812
|
+
/** Which ring slot carried it. */
|
|
2813
|
+
slot: number;
|
|
2814
|
+
occupancy: number;
|
|
2815
|
+
/**
|
|
2816
|
+
* The frame index read out of the slot's pixels when it was acquired.
|
|
2817
|
+
*
|
|
2818
|
+
* -1 when the probe was off or the band could not be read; `markerFailure`
|
|
2819
|
+
* says which.
|
|
2820
|
+
*/
|
|
2821
|
+
acquiredMarker: number;
|
|
2822
|
+
/** The frame index read out of the picture actually handed to composition. */
|
|
2823
|
+
submittedMarker: number;
|
|
2824
|
+
/** Empty when the band was read; otherwise why it was not. */
|
|
2825
|
+
markerFailure: string;
|
|
2826
|
+
/**
|
|
2827
|
+
* The picture's whole life, so one row can be attributed.
|
|
2828
|
+
*
|
|
2829
|
+
* When it was acquired, when it was due for presentation, when it was
|
|
2830
|
+
* promoted. Without these, a late submission cannot be separated into a late
|
|
2831
|
+
* arrival, a long wait for its due time, and a slow promotion.
|
|
2832
|
+
*/
|
|
2833
|
+
acquiredNs: number;
|
|
2834
|
+
dueNs: number;
|
|
2835
|
+
promotedNs: number;
|
|
2836
|
+
/**
|
|
2837
|
+
* Rendering error against the frame the submitted band names: background
|
|
2838
|
+
* level in 0..255, bar position in 0..1 of height. -1 when not measured.
|
|
2839
|
+
*/
|
|
2840
|
+
levelDelta: number;
|
|
2841
|
+
barDelta: number;
|
|
2842
|
+
}
|
|
2843
|
+
/** @experimental */
|
|
2844
|
+
interface ScoredSubmission {
|
|
2845
|
+
frameId: number;
|
|
2846
|
+
ptsUs: number;
|
|
2847
|
+
intervalUs: number;
|
|
2848
|
+
submittedNs: number;
|
|
2849
|
+
/** The audio clock at the submission instant, on the same monotonic timeline. */
|
|
2850
|
+
audioUs: number;
|
|
2851
|
+
/** 0 inside the interval; otherwise the distance to the nearer edge. */
|
|
2852
|
+
errorUs: number;
|
|
2853
|
+
/** Positive when the sound is past the frame's span — the picture is behind. */
|
|
2854
|
+
signedUs: number;
|
|
2855
|
+
occupancy: number;
|
|
2856
|
+
}
|
|
2857
|
+
/** @experimental */
|
|
2858
|
+
interface SubmissionReport {
|
|
2859
|
+
toleranceUs: number;
|
|
2860
|
+
scored: number;
|
|
2861
|
+
/** Submissions with no audio sample close enough to pair with. */
|
|
2862
|
+
unpaired: number;
|
|
2863
|
+
excursions: ScoredSubmission[];
|
|
2864
|
+
worstErrorUs: number;
|
|
2865
|
+
pass: boolean;
|
|
2866
|
+
/**
|
|
2867
|
+
* What this run established, as three outcomes.
|
|
2868
|
+
*
|
|
2869
|
+
* A score computed from nothing is not a pass. Zero scored submissions means
|
|
2870
|
+
* the run did not observe playback, and that has to be distinguishable from
|
|
2871
|
+
* a run that observed it and found it clean.
|
|
2872
|
+
*/
|
|
2873
|
+
verdict: 'pass' | 'fail' | 'invalid';
|
|
2874
|
+
verdictReason: string;
|
|
2875
|
+
/** Gaps between consecutive submitted timestamps: the delivered cadence. */
|
|
2876
|
+
cadenceUs: {
|
|
2877
|
+
median: number;
|
|
2878
|
+
min: number;
|
|
2879
|
+
max: number;
|
|
2880
|
+
samples: number;
|
|
2881
|
+
};
|
|
2882
|
+
/**
|
|
2883
|
+
* Wall-clock gaps between consecutive submissions.
|
|
2884
|
+
*
|
|
2885
|
+
* The tail is the point: a median at the output interval says nothing about
|
|
2886
|
+
* the frame that took four times as long, and that frame is the one somebody
|
|
2887
|
+
* sees. Taken from `submittedNs`, which is the instant composition began
|
|
2888
|
+
* consuming each picture.
|
|
2889
|
+
*/
|
|
2890
|
+
frameTimeUs: {
|
|
2891
|
+
p50: number;
|
|
2892
|
+
p95: number;
|
|
2893
|
+
p99: number;
|
|
2894
|
+
max: number;
|
|
2895
|
+
samples: number;
|
|
2896
|
+
};
|
|
2897
|
+
peakOccupancy: number;
|
|
2898
|
+
}
|
|
2899
|
+
/** @experimental */
|
|
2900
|
+
declare function scoreSubmissions(submissions: readonly SubmissionRecord[], samples: readonly SpikeSample[], options?: {
|
|
2901
|
+
toleranceUs?: number;
|
|
2902
|
+
pairWindowNs?: number;
|
|
2903
|
+
}): SubmissionReport;
|
|
2904
|
+
/** One submission, with its whole life and the clock it was scored against. @experimental */
|
|
2905
|
+
interface TraceRow {
|
|
2906
|
+
frameId: number;
|
|
2907
|
+
/** Offsets in ms from the window's first row, so the numbers are readable. */
|
|
2908
|
+
acquiredMs: number;
|
|
2909
|
+
dueMs: number;
|
|
2910
|
+
promotedMs: number;
|
|
2911
|
+
submittedMs: number;
|
|
2912
|
+
/**
|
|
2913
|
+
* Acquisition to due, due to promotion, promotion to submission, in us.
|
|
2914
|
+
*
|
|
2915
|
+
* A negative `waitUs` means the presentation time had already passed when the
|
|
2916
|
+
* frame was acquired — it was late before the ring saw it.
|
|
2917
|
+
*/
|
|
2918
|
+
waitUs: number;
|
|
2919
|
+
promoteUs: number;
|
|
2920
|
+
submitUs: number;
|
|
2921
|
+
ptsUs: number;
|
|
2922
|
+
audioUs: number;
|
|
2923
|
+
errorUs: number;
|
|
2924
|
+
slot: number;
|
|
2925
|
+
occupancy: number;
|
|
2926
|
+
marker: number;
|
|
2927
|
+
}
|
|
2928
|
+
/** @experimental */
|
|
2929
|
+
interface SubmissionTrace {
|
|
2930
|
+
/** The frame the window is centred on. */
|
|
2931
|
+
frameId: number;
|
|
2932
|
+
rows: TraceRow[];
|
|
2933
|
+
/**
|
|
2934
|
+
* Where the centre frame's budget went, as the largest single interval.
|
|
2935
|
+
*
|
|
2936
|
+
* `arrival` when it was acquired late, `due` when it sat waiting for its
|
|
2937
|
+
* presentation time, `promotion` when it was due and not chosen, `submission`
|
|
2938
|
+
* when promotion and submission diverged. Empty when the row is incomplete.
|
|
2939
|
+
*/
|
|
2940
|
+
spentOn: 'arrival' | 'due' | 'promotion' | 'submission' | 'unknown';
|
|
2941
|
+
/** The dominant leg's size, so a label cannot imply a magnitude it lacks. */
|
|
2942
|
+
spentUs: number;
|
|
2943
|
+
/** True when the window could not be built from complete records. */
|
|
2944
|
+
invalid: boolean;
|
|
2945
|
+
reason: string;
|
|
2946
|
+
}
|
|
2947
|
+
/**
|
|
2948
|
+
* The life of one submission and its neighbours.
|
|
2949
|
+
*
|
|
2950
|
+
* Built for attribution, not for scoring: a submission excursion says a picture
|
|
2951
|
+
* was wrong at an instant, and this says where its time went. Acquisition, due
|
|
2952
|
+
* time, promotion and submission are four stamps on one frame, and the largest
|
|
2953
|
+
* gap between consecutive ones is the interval to investigate.
|
|
2954
|
+
*
|
|
2955
|
+
* Incomplete rows are reported rather than dropped. A ring buffer that wrapped,
|
|
2956
|
+
* a record written before the ring carried these stamps, or a frame that never
|
|
2957
|
+
* reached submission all produce a window that cannot be attributed, and a
|
|
2958
|
+
* silently shortened window would read as a clean one.
|
|
2959
|
+
*
|
|
2960
|
+
* @experimental
|
|
2961
|
+
*/
|
|
2962
|
+
declare function traceSubmission(records: readonly SubmissionRecord[], scored: readonly ScoredSubmission[], frameId: number, span?: number): SubmissionTrace;
|
|
2963
|
+
/** @experimental */
|
|
2964
|
+
interface SeamStage {
|
|
2965
|
+
framePtsUs: number;
|
|
2966
|
+
/** Offsets in ms from the window's first sample. */
|
|
2967
|
+
metadataMs: number;
|
|
2968
|
+
availableMs: number;
|
|
2969
|
+
consumedMs: number;
|
|
2970
|
+
/** Renderer to queue, queue to latch, in us. -1 when a stamp is missing. */
|
|
2971
|
+
graphUs: number;
|
|
2972
|
+
latchUs: number;
|
|
2973
|
+
}
|
|
2974
|
+
/** @experimental */
|
|
2975
|
+
interface SeamTrace {
|
|
2976
|
+
/** The boundary this window is centred on, in project time. */
|
|
2977
|
+
atUs: number;
|
|
2978
|
+
/** The largest gap between consecutive frames reaching the renderer, in us. */
|
|
2979
|
+
outputGapUs: number;
|
|
2980
|
+
/** The same gap measured at acquisition. */
|
|
2981
|
+
acquireGapUs: number;
|
|
2982
|
+
/**
|
|
2983
|
+
* Who owns the gap.
|
|
2984
|
+
*
|
|
2985
|
+
* `graph` when the renderer produced and the queue did not follow.
|
|
2986
|
+
* `consumer` when it was queued and not latched. `production` when nothing
|
|
2987
|
+
* reached the renderer's output callback at all.
|
|
2988
|
+
*
|
|
2989
|
+
* That last one is deliberately broader than "the decoder was unprepared":
|
|
2990
|
+
* this stamp fires when the renderer is about to hand a frame to the surface,
|
|
2991
|
+
* so its absence covers a decoder that produced nothing *and* a renderer
|
|
2992
|
+
* blocked waiting for the sink to accept a new input stream. Separating those
|
|
2993
|
+
* two needs a stamp inside Media3 that is not exposed here. What it does
|
|
2994
|
+
* establish is that the graph and this consumer are not responsible, which is
|
|
2995
|
+
* what the acquisition gap alone could not say.
|
|
2996
|
+
*/
|
|
2997
|
+
owner: 'production' | 'graph' | 'consumer' | 'unknown';
|
|
2998
|
+
stages: SeamStage[];
|
|
2999
|
+
invalid: boolean;
|
|
3000
|
+
reason: string;
|
|
3001
|
+
}
|
|
3002
|
+
/**
|
|
3003
|
+
* Where output stops at a clip boundary.
|
|
3004
|
+
*
|
|
3005
|
+
* The acquisition gap says delivery stopped; it cannot say whether the decoder
|
|
3006
|
+
* produced nothing, the video graph held what it produced, or this consumer
|
|
3007
|
+
* failed to latch what was queued. Those are three owners and three different
|
|
3008
|
+
* fixes, and they are separated by one stamp: when the renderer said a frame
|
|
3009
|
+
* was about to go to the surface.
|
|
3010
|
+
*
|
|
3011
|
+
* Samples carrying no frame are skipped rather than treated as a gap — the
|
|
3012
|
+
* poller runs faster than the frame rate, so most samples repeat the last
|
|
3013
|
+
* frame, and counting those would report a stall on every tick.
|
|
3014
|
+
*
|
|
3015
|
+
* @experimental
|
|
3016
|
+
*/
|
|
3017
|
+
declare function traceSeam(samples: readonly SpikeSample[], atUs: number, spanUs?: number): SeamTrace;
|
|
3018
|
+
/** One frame, seen from both sides of the pairing. @experimental */
|
|
3019
|
+
interface StartupRow {
|
|
3020
|
+
frameId: number;
|
|
3021
|
+
/** What Media3 said this frame's presentation time was. */
|
|
3022
|
+
renderedPtsUs: number;
|
|
3023
|
+
/** What the submission record was written under. The same number, or a bug. */
|
|
3024
|
+
recordedPtsUs: number;
|
|
3025
|
+
/** What the pixels say the frame is. Independent of both. */
|
|
3026
|
+
marker: number;
|
|
3027
|
+
/** The source frame the marker's own index implies, at this clip's speed. */
|
|
3028
|
+
markerPtsUs: number;
|
|
3029
|
+
metadataMs: number;
|
|
3030
|
+
consumedMs: number;
|
|
3031
|
+
submittedMs: number;
|
|
3032
|
+
}
|
|
3033
|
+
/** @experimental */
|
|
3034
|
+
interface StartupTrace {
|
|
3035
|
+
rows: StartupRow[];
|
|
3036
|
+
/**
|
|
3037
|
+
* Frames the renderer never emitted, by source index.
|
|
3038
|
+
*
|
|
3039
|
+
* The discriminator. If the renderer emitted a presentation time and the
|
|
3040
|
+
* pixels under it belong to a different frame, the *association* is wrong. If
|
|
3041
|
+
* it never emitted that time at all, the frame was dropped or never produced.
|
|
3042
|
+
* Those are different defects and the pixels alone cannot tell them apart.
|
|
3043
|
+
*/
|
|
3044
|
+
missingSourceFrames: number[];
|
|
3045
|
+
/** Frames whose pixels disagree with the time they were rendered under. */
|
|
3046
|
+
misassociated: number[];
|
|
3047
|
+
invalid: boolean;
|
|
3048
|
+
reason: string;
|
|
3049
|
+
}
|
|
3050
|
+
/**
|
|
3051
|
+
* The first frames of a clip, from both sides of the pairing at once.
|
|
3052
|
+
*
|
|
3053
|
+
* Media3's presentation time and the marker in the pixels are independent
|
|
3054
|
+
* claims about the same frame. Joined by frame id, they separate "frames 2 to 6
|
|
3055
|
+
* were never produced" from "frame 7's pixels arrived labelled as frame 2" —
|
|
3056
|
+
* which look identical in a cadence result and need opposite fixes.
|
|
3057
|
+
*
|
|
3058
|
+
* @experimental
|
|
3059
|
+
*/
|
|
3060
|
+
declare function traceStartup(samples: readonly SpikeSample[], records: readonly SubmissionRecord[], options: {
|
|
3061
|
+
sourceFps: number;
|
|
3062
|
+
speed: number;
|
|
3063
|
+
count?: number;
|
|
3064
|
+
}): StartupTrace;
|
|
3065
|
+
|
|
3066
|
+
/**
|
|
3067
|
+
* Did the preview show the right frame at each output instant?
|
|
3068
|
+
*
|
|
3069
|
+
* ## The oracle this replaces
|
|
3070
|
+
*
|
|
3071
|
+
* The first version counted *source* frames and expected every one to appear.
|
|
3072
|
+
* That reported a 2x clip — 6 seconds of source occupying 3 seconds — as
|
|
3073
|
+
* delivering 98 of 180 frames with 58 missed intervals. The 180 was wrong: a
|
|
3074
|
+
* preview has its own output rate, and at 30 fps three seconds of timeline has
|
|
3075
|
+
* about 90 output instants, at each of which the resampling rule *skips* every
|
|
3076
|
+
* second source frame by design. Requiring all 180 asked the preview to run at
|
|
3077
|
+
* 60 fps and then scored it for refusing.
|
|
3078
|
+
*
|
|
3079
|
+
* So the oracle is the shared rule itself. `resampleClip` already decides which
|
|
3080
|
+
* source frame belongs at each output instant, at a stated target rate, and it
|
|
3081
|
+
* is the same function the exporter's contract is written against. Using it
|
|
3082
|
+
* here means the preview and the export cannot disagree about what "correct"
|
|
3083
|
+
* means.
|
|
3084
|
+
*
|
|
3085
|
+
* ## What is checked
|
|
3086
|
+
*
|
|
3087
|
+
* At each expected output instant, whether the picture on screen is the frame
|
|
3088
|
+
* the rule names. Not whether every source frame appeared — above 1x most of
|
|
3089
|
+
* them must not.
|
|
3090
|
+
*
|
|
3091
|
+
* @experimental
|
|
3092
|
+
*/
|
|
3093
|
+
/** One clip, as the playlist compiled it. @experimental */
|
|
3094
|
+
interface CadenceClip {
|
|
3095
|
+
name: string;
|
|
3096
|
+
/** Where the window sits on the project timeline. */
|
|
3097
|
+
timelineStartUs: number;
|
|
3098
|
+
timelineEndUs: number;
|
|
3099
|
+
/** Clipping bounds in source time. */
|
|
3100
|
+
sourceStartUs: number;
|
|
3101
|
+
sourceEndUs: number;
|
|
3102
|
+
speed: number;
|
|
3103
|
+
sourceFps: number;
|
|
3104
|
+
}
|
|
3105
|
+
/**
|
|
3106
|
+
* How far the preview may sit from the export's frame selection.
|
|
3107
|
+
*
|
|
3108
|
+
* The tolerance the rest of the video path is held to. A preview picture is
|
|
3109
|
+
* temporally accurate when the frame it shows is within this of the frame the
|
|
3110
|
+
* rule names, converted to project time — not when it is that exact frame.
|
|
3111
|
+
*
|
|
3112
|
+
* @experimental
|
|
3113
|
+
*/
|
|
3114
|
+
declare const PREVIEW_TOLERANCE_US = 60000;
|
|
3115
|
+
/**
|
|
3116
|
+
* What a run established, as three outcomes rather than two.
|
|
3117
|
+
*
|
|
3118
|
+
* `invalid` is not a soft failure — it is the absence of a measurement, and it
|
|
3119
|
+
* must never be reachable by a run that simply had the probe switched off. A
|
|
3120
|
+
* boolean cannot say that: `false` reads as "measured and wrong", and three
|
|
3121
|
+
* device runs once reported `wrong=0, worstOffset=0.0 ms` because a shifted
|
|
3122
|
+
* button meant nothing was scorable at all. That was a clean pass by absence of
|
|
3123
|
+
* evidence and one paste from being reported as a fix working.
|
|
3124
|
+
*
|
|
3125
|
+
* @experimental
|
|
3126
|
+
*/
|
|
3127
|
+
type Verdict = 'pass' | 'fail' | 'invalid';
|
|
3128
|
+
/** Rendering correctness, per clip. @experimental */
|
|
3129
|
+
interface CadenceParity {
|
|
3130
|
+
/** Submissions that carried a parity measurement. */
|
|
3131
|
+
measured: number;
|
|
3132
|
+
/** Worst background-level error, 0..255. */
|
|
3133
|
+
worstLevelDelta: number;
|
|
3134
|
+
/** Worst bar-position error, in 0..1 of height. */
|
|
3135
|
+
worstBarDelta: number;
|
|
3136
|
+
/** Submissions whose rendering was outside tolerance. */
|
|
3137
|
+
failures: number;
|
|
3138
|
+
pass: boolean;
|
|
3139
|
+
}
|
|
3140
|
+
/** @experimental */
|
|
3141
|
+
interface CadenceResult {
|
|
3142
|
+
clip: string;
|
|
3143
|
+
/** Output instants the rule names for the whole clip. */
|
|
3144
|
+
nominalInstants: number;
|
|
3145
|
+
/** Output instants the rule expects across the span actually covered. */
|
|
3146
|
+
expectedInstants: number;
|
|
3147
|
+
/**
|
|
3148
|
+
* Instants that could not be placed on the wall clock.
|
|
3149
|
+
*
|
|
3150
|
+
* No bracketing pair of player-clock samples inside one segment — the
|
|
3151
|
+
* transport was paused or seeking across them. Counted and excluded from
|
|
3152
|
+
* scoring rather than placed by extrapolation.
|
|
3153
|
+
*/
|
|
3154
|
+
unplaceableInstants: number;
|
|
3155
|
+
/**
|
|
3156
|
+
* How far the frame-derived clock sits from the player's, in microseconds.
|
|
3157
|
+
*
|
|
3158
|
+
* A diagnostic, never the anchor. Positive means pictures are handed over
|
|
3159
|
+
* later than the project time they carry — the pipeline's systematic
|
|
3160
|
+
* presentation delay.
|
|
3161
|
+
*
|
|
3162
|
+
* Anchoring on the frame-derived value would subtract this from every
|
|
3163
|
+
* measurement and call a uniformly late pipeline punctual, which is why
|
|
3164
|
+
* instants are placed from the player's clock and this is only reported.
|
|
3165
|
+
*/
|
|
3166
|
+
frameDerivedOffsetUs: number;
|
|
3167
|
+
/**
|
|
3168
|
+
* Instants before the first submission, and after the last.
|
|
3169
|
+
*
|
|
3170
|
+
* Named rather than absorbed. Scoring a clip over its nominal length would
|
|
3171
|
+
* count every instant a run never reached as a missed deadline; excluding
|
|
3172
|
+
* them silently would let a picture that arrived late or stopped early
|
|
3173
|
+
* disappear from coverage instead. They are excluded from the identity check
|
|
3174
|
+
* and reported here, and `leadingExcluded` in particular is the interval
|
|
3175
|
+
* where the previous clip was still on screen — which is the seam, and is
|
|
3176
|
+
* scored by the submission metric rather than by this one.
|
|
3177
|
+
*/
|
|
3178
|
+
leadingExcluded: number;
|
|
3179
|
+
trailingExcluded: number;
|
|
3180
|
+
/** Instants where some frame was on screen. */
|
|
3181
|
+
coveredInstants: number;
|
|
3182
|
+
/**
|
|
3183
|
+
* Instants showing a different source frame from the one the rule names.
|
|
3184
|
+
*
|
|
3185
|
+
* The real failure: a missed deadline is the picture being wrong at a moment
|
|
3186
|
+
* somebody was looking, not a source frame going unused.
|
|
3187
|
+
*/
|
|
3188
|
+
wrongFrameInstants: number;
|
|
3189
|
+
/** Instants with no frame at all — the picture had not caught up. */
|
|
3190
|
+
uncoveredInstants: number;
|
|
3191
|
+
/** Unique pictures submitted inside the span. */
|
|
3192
|
+
submittedUnique: number;
|
|
3193
|
+
/**
|
|
3194
|
+
* Source frames the rule says to skip.
|
|
3195
|
+
*
|
|
3196
|
+
* Above 1x this is most of them and is correct. Reported so a count of
|
|
3197
|
+
* "missing" frames cannot be read as loss.
|
|
3198
|
+
*/
|
|
3199
|
+
ruleSkippedSourceFrames: number;
|
|
3200
|
+
/** The interval between output instants, from the declared preview rate. */
|
|
3201
|
+
outputIntervalUs: number;
|
|
3202
|
+
medianHoldUs: number;
|
|
3203
|
+
maxHoldUs: number;
|
|
3204
|
+
/**
|
|
3205
|
+
* How long one picture should stay up, from the rule.
|
|
3206
|
+
*
|
|
3207
|
+
* At 2x a new source frame is due at every output instant; at 0.5x the rule
|
|
3208
|
+
* repeats each one, so a hold of two instants is correct rather than a stall.
|
|
3209
|
+
*/
|
|
3210
|
+
expectedHoldUs: number;
|
|
3211
|
+
/**
|
|
3212
|
+
* Pictures held past their turn.
|
|
3213
|
+
*
|
|
3214
|
+
* Counted from the holds rather than from the covered span, because a stall
|
|
3215
|
+
* at the end of a run produces no further submissions and would otherwise
|
|
3216
|
+
* simply shorten the span it was scored over.
|
|
3217
|
+
*/
|
|
3218
|
+
missedDeadlines: number;
|
|
3219
|
+
/**
|
|
3220
|
+
* How far off the shown frame was, counted in source frames.
|
|
3221
|
+
*
|
|
3222
|
+
* Keyed by `shown - expected`, over every scored instant whose identity was
|
|
3223
|
+
* established — the correct ones included, so the shape of the whole run is
|
|
3224
|
+
* visible rather than only its tail. This is what separates a picture that is
|
|
3225
|
+
* one source frame early from one that is a third of a second stale, and at
|
|
3226
|
+
* 2x those are 8.3 ms and 330 ms of project time: the same count of "wrong
|
|
3227
|
+
* instants" and completely different defects.
|
|
3228
|
+
*/
|
|
3229
|
+
offsets: Record<string, number>;
|
|
3230
|
+
/**
|
|
3231
|
+
* The largest offset, in project time rather than in source frames.
|
|
3232
|
+
*
|
|
3233
|
+
* The conversion that makes the count mean something. One source frame at
|
|
3234
|
+
* speed `s` is `1 / sourceFps / s` of project time — 16.7 ms at 2x, 66.7 ms
|
|
3235
|
+
* at 0.5x — so the same offset of three frames is 50 ms in one clip and 200
|
|
3236
|
+
* ms in another. Judged against the synchronisation tolerance, not against
|
|
3237
|
+
* zero: a preview consumes a live decoder at its own display rate and cannot
|
|
3238
|
+
* seek to an exact source frame per instant the way an export does.
|
|
3239
|
+
*/
|
|
3240
|
+
worstOffsetUs: number;
|
|
3241
|
+
/**
|
|
3242
|
+
* Rendering correctness, against the frame each picture's own band names.
|
|
3243
|
+
*
|
|
3244
|
+
* Separate from everything above it. The rest of this result is about *which*
|
|
3245
|
+
* frame was shown; this is about whether that frame was drawn correctly —
|
|
3246
|
+
* colour, orientation and scale. A picture one source frame early and
|
|
3247
|
+
* rendered perfectly scores zero here.
|
|
3248
|
+
*/
|
|
3249
|
+
parity: CadenceParity;
|
|
3250
|
+
/**
|
|
3251
|
+
* Instants that showed exactly the frame the export rule names.
|
|
3252
|
+
*
|
|
3253
|
+
* Reported, never required. It is the export contract's measure, kept here so
|
|
3254
|
+
* the preview's tolerance is visible as a decision rather than as the absence
|
|
3255
|
+
* of a check.
|
|
3256
|
+
*/
|
|
3257
|
+
exactInstants: number;
|
|
3258
|
+
/**
|
|
3259
|
+
* What this run established. @see Verdict
|
|
3260
|
+
*
|
|
3261
|
+
* `pass` only when identity was read from pixels, parity was measured, and
|
|
3262
|
+
* both were clean. Anything unmeasured is `invalid`.
|
|
3263
|
+
*/
|
|
3264
|
+
verdict: Verdict;
|
|
3265
|
+
/** Why, when the verdict is not a pass. */
|
|
3266
|
+
verdictReason: string;
|
|
3267
|
+
/** Whether the acquired-identity evidence was usable at all. */
|
|
3268
|
+
identity: CadenceIdentity;
|
|
3269
|
+
/**
|
|
3270
|
+
* Wall-clock gaps between this clip's submissions.
|
|
3271
|
+
*
|
|
3272
|
+
* Per clip because the aggregate mixes rates: a 0.5x clip legitimately holds
|
|
3273
|
+
* each picture for two output intervals, so a global p95 at 66.7 ms is the
|
|
3274
|
+
* 0.5x clip behaving correctly and says nothing about whether the 2x clip
|
|
3275
|
+
* stalled. Reported separately so a delay cannot hide inside a hold.
|
|
3276
|
+
*/
|
|
3277
|
+
frameTimeUs: {
|
|
3278
|
+
p50: number;
|
|
3279
|
+
p95: number;
|
|
3280
|
+
p99: number;
|
|
3281
|
+
max: number;
|
|
3282
|
+
samples: number;
|
|
3283
|
+
};
|
|
3284
|
+
/**
|
|
3285
|
+
* Every instant whose picture was not the one the rule names.
|
|
3286
|
+
*
|
|
3287
|
+
* Including the boundary instants the scored span excludes: those are the
|
|
3288
|
+
* seam, and leaving them out of the identity check is how a wrong picture at
|
|
3289
|
+
* a clip change disappears from both metrics at once.
|
|
3290
|
+
*/
|
|
3291
|
+
wrong: WrongInstant[];
|
|
3292
|
+
pass: boolean;
|
|
3293
|
+
}
|
|
3294
|
+
/**
|
|
3295
|
+
* Why an instant showed the wrong picture.
|
|
3296
|
+
*
|
|
3297
|
+
* The first three are decided in pixels and name one place to look. The fourth
|
|
3298
|
+
* is what the metadata alone can support, which is less than it appears: on
|
|
3299
|
+
* this device `SurfaceTexture` reports a system-clock value, so an instant
|
|
3300
|
+
* classified without the marker cannot separate a frame acquired wrongly from a
|
|
3301
|
+
* frame chosen wrongly, and says so rather than picking one.
|
|
3302
|
+
*
|
|
3303
|
+
* @experimental
|
|
3304
|
+
*/
|
|
3305
|
+
type WrongInstantCause =
|
|
3306
|
+
/**
|
|
3307
|
+
* The expected frame's pixels were not in the ring when the instant needed
|
|
3308
|
+
* them.
|
|
3309
|
+
*
|
|
3310
|
+
* Nothing downstream could have shown it. The fault is decode, acquisition or
|
|
3311
|
+
* the copy into the ring — and the test is by that instant, not by the end of
|
|
3312
|
+
* the clip, because every frame arrives eventually and a clip-global check
|
|
3313
|
+
* would charge a frame that had simply not arrived yet to selection.
|
|
3314
|
+
*/
|
|
3315
|
+
'not-delivered'
|
|
3316
|
+
/**
|
|
3317
|
+
* The expected frame's pixels were in the ring, and a different frame was
|
|
3318
|
+
* submitted.
|
|
3319
|
+
*
|
|
3320
|
+
* The fault is the choice, not the supply.
|
|
3321
|
+
*/
|
|
3322
|
+
| 'wrong-selection'
|
|
3323
|
+
/**
|
|
3324
|
+
* The submitted picture no longer held the frame its slot was filled with.
|
|
3325
|
+
*
|
|
3326
|
+
* A wrapped texture aliases storage the ring may redraw. This is that storage
|
|
3327
|
+
* changing under a picture composition still holds, and it is the one cause
|
|
3328
|
+
* here that is a correctness defect in the ring's own contract.
|
|
3329
|
+
*/
|
|
3330
|
+
| 'storage-aliased'
|
|
3331
|
+
/**
|
|
3332
|
+
* The wrong picture, and the evidence cannot say whether it was acquired or
|
|
3333
|
+
* chosen wrongly.
|
|
3334
|
+
*
|
|
3335
|
+
* What the metadata supports with the probe off. Not a third fault: a
|
|
3336
|
+
* placeholder for an attribution not yet made.
|
|
3337
|
+
*/
|
|
3338
|
+
| 'wrong-submitted-identity'
|
|
3339
|
+
/**
|
|
3340
|
+
* The right frame, submitted after the instant it belonged to.
|
|
3341
|
+
*
|
|
3342
|
+
* Only reachable without pixel identity. When the marker is readable, a frame
|
|
3343
|
+
* matching the rule makes the instant correct however late it was submitted,
|
|
3344
|
+
* so lateness stops being an identity fault and becomes a frame-time one.
|
|
3345
|
+
* Its evidence — the ring's own submission time and project timestamp — is
|
|
3346
|
+
* independent of the disputed pairing, which is why it survives there.
|
|
3347
|
+
*/
|
|
3348
|
+
| 'late-delivery'
|
|
3349
|
+
/** No submission covered the instant at all. */
|
|
3350
|
+
| 'uncovered';
|
|
3351
|
+
/** @experimental */
|
|
3352
|
+
interface WrongInstant {
|
|
3353
|
+
projectUs: number;
|
|
3354
|
+
/** From the shared rule: the source frame that belongs at this instant. */
|
|
3355
|
+
expectedSourceFrame: number;
|
|
3356
|
+
expectedSourcePtsUs: number;
|
|
3357
|
+
/** What was on screen, mapped back through the same speed. */
|
|
3358
|
+
shownSourceFrame: number;
|
|
3359
|
+
shownPtsUs: number;
|
|
3360
|
+
/** What the texture said it held when that frame was acquired. */
|
|
3361
|
+
acquiredTexturePtsUs: number;
|
|
3362
|
+
/** The frame index in the pixels of the slot, and of the submitted picture. */
|
|
3363
|
+
acquiredMarker: number;
|
|
3364
|
+
submittedMarker: number;
|
|
3365
|
+
slot: number;
|
|
3366
|
+
frameId: number;
|
|
3367
|
+
submittedNs: number;
|
|
3368
|
+
cause: WrongInstantCause;
|
|
3369
|
+
/** True when the instant lies outside the scored span. */
|
|
3370
|
+
boundary: boolean;
|
|
3371
|
+
}
|
|
3372
|
+
/** What the texture timestamp turned out to be. @experimental */
|
|
3373
|
+
type TimestampDomain =
|
|
3374
|
+
/** Comparable with the clip's media time. */
|
|
3375
|
+
'media'
|
|
3376
|
+
/** A system-clock value: `System.nanoTime()`, as measured on this device. */
|
|
3377
|
+
| 'system'
|
|
3378
|
+
/** Nothing was recorded, so there is nothing to establish. */
|
|
3379
|
+
| 'unknown';
|
|
3380
|
+
/** @experimental */
|
|
3381
|
+
interface CadenceIdentity {
|
|
3382
|
+
/**
|
|
3383
|
+
* The domain the texture timestamps were established to be in.
|
|
3384
|
+
*
|
|
3385
|
+
* Recorded rather than assumed, and recorded even when it is the one that
|
|
3386
|
+
* makes the check weaker. It was `media` by assumption for one whole run and
|
|
3387
|
+
* the assumption was wrong by forty-eight hours.
|
|
3388
|
+
*/
|
|
3389
|
+
timestampDomain: TimestampDomain;
|
|
3390
|
+
medianTextureSkewUs: number;
|
|
3391
|
+
/**
|
|
3392
|
+
* Whether identity came from the pixels.
|
|
3393
|
+
*
|
|
3394
|
+
* When false, the fallback compares submitted timestamps — which on a
|
|
3395
|
+
* `system` domain cannot uniquely associate a buffer with a frame. Every
|
|
3396
|
+
* instant is then counted `identityUnverified` and the clip cannot pass. A
|
|
3397
|
+
* fallback that quietly declared those instants correct would be restoring
|
|
3398
|
+
* confidence the evidence does not support.
|
|
3399
|
+
*/
|
|
3400
|
+
pixelIdentity: boolean;
|
|
3401
|
+
/** Instants whose identity could not be established either way. */
|
|
3402
|
+
identityUnverified: number;
|
|
3403
|
+
/** Distinct frame indices read out of the ring for this clip. */
|
|
3404
|
+
deliveredFrames: number;
|
|
3405
|
+
/** Bands that were read, and bands that failed. */
|
|
3406
|
+
markersRead: number;
|
|
3407
|
+
markersFailed: number;
|
|
3408
|
+
}
|
|
3409
|
+
/**
|
|
3410
|
+
* The preview's own output rate.
|
|
3411
|
+
*
|
|
3412
|
+
* Stated rather than inferred from what arrived: an oracle that takes its
|
|
3413
|
+
* expectation from the measurement cannot fail.
|
|
3414
|
+
*
|
|
3415
|
+
* @experimental
|
|
3416
|
+
*/
|
|
3417
|
+
declare const PREVIEW_OUTPUT_FPS = 30;
|
|
3418
|
+
/** @experimental */
|
|
3419
|
+
declare function checkCadence(clip: CadenceClip, submissions: readonly SubmissionRecord[], outputFps?: number,
|
|
3420
|
+
/**
|
|
3421
|
+
* How far the shown frame may be from the named one, in project time.
|
|
3422
|
+
*
|
|
3423
|
+
* Zero scores the export contract — exact selection, no tolerance — and the
|
|
3424
|
+
* default scores the preview's. @see PREVIEW_TOLERANCE_US
|
|
3425
|
+
*/
|
|
3426
|
+
toleranceUs?: number,
|
|
3427
|
+
/**
|
|
3428
|
+
* The player's own clock, paired with monotonic time.
|
|
3429
|
+
*
|
|
3430
|
+
* Where instants are placed on the wall clock. Empty means no instant can be
|
|
3431
|
+
* placed, and the result is `invalid` rather than scored against a guess.
|
|
3432
|
+
*/
|
|
3433
|
+
samples?: readonly SpikeSample[]): CadenceResult;
|
|
3434
|
+
|
|
3435
|
+
/**
|
|
3436
|
+
* Preview audio, bound to a screen's lifetime.
|
|
3437
|
+
*
|
|
3438
|
+
* The decisions live in `previewAudioSession.ts` and the mixing lives in
|
|
3439
|
+
* native; this is the part that acts — starting jobs, loading the player,
|
|
3440
|
+
* muting the other one, and refusing to publish an answer that has been
|
|
3441
|
+
* superseded.
|
|
3442
|
+
*
|
|
3443
|
+
* ## Epochs
|
|
3444
|
+
*
|
|
3445
|
+
* Every load, play and seek carries the epoch of the mix it belongs to. A
|
|
3446
|
+
* callback or a correction arriving with an older epoch is from a mix that has
|
|
3447
|
+
* since been replaced, and acting on it would move the picture to match audio
|
|
3448
|
+
* that is no longer playing. The epoch is bumped on every publish, so a late
|
|
3449
|
+
* result cannot masquerade as the current one.
|
|
3450
|
+
*
|
|
3451
|
+
* @experimental
|
|
3452
|
+
*/
|
|
3453
|
+
/** @experimental */
|
|
3454
|
+
interface UsePreviewAudioOptions {
|
|
3455
|
+
project: VideoProject;
|
|
3456
|
+
/** The same resolver the exporter is given. */
|
|
3457
|
+
resolveSource: (source: MediaSource, context: ExportSourceContext) => string | null | undefined;
|
|
3458
|
+
/** Off entirely — for a screen that never plays. */
|
|
3459
|
+
enabled?: boolean;
|
|
3460
|
+
/** How far apart the players may drift before audio is moved. */
|
|
3461
|
+
toleranceUs?: number;
|
|
3462
|
+
}
|
|
3463
|
+
/** @experimental */
|
|
3464
|
+
interface PreviewAudioBinding {
|
|
3465
|
+
status: PreviewAudioStatus;
|
|
3466
|
+
/** A sentence for a person when preparation failed. */
|
|
3467
|
+
error: string | null;
|
|
3468
|
+
/** Whether playback may run. False while preparing or after a failure. */
|
|
3469
|
+
canPlay: boolean;
|
|
3470
|
+
/** True once a mix matching the document is loaded. */
|
|
3471
|
+
ready: boolean;
|
|
3472
|
+
/** How long the last preparation took, in ms. Null when it was cached. */
|
|
3473
|
+
lastPrepareMs: number | null;
|
|
3474
|
+
/** Where that time went. Null for a cached mix. */
|
|
3475
|
+
lastProfile: unknown;
|
|
3476
|
+
/**
|
|
3477
|
+
* The prepared mix, for a composition that carries the sound itself.
|
|
3478
|
+
*
|
|
3479
|
+
* Null until one matching the document exists. The preview plays audio and
|
|
3480
|
+
* video from one composition on one clock, so what this hook contributes is
|
|
3481
|
+
* the prepared program — the file and how long it runs — rather than a second
|
|
3482
|
+
* player to keep in step with the first. `play`, `pause`, `seek`, `setRate`
|
|
3483
|
+
* and `checkDrift` are what is left of that second player and are no longer
|
|
3484
|
+
* part of the preview path.
|
|
3485
|
+
*/
|
|
3486
|
+
mix: {
|
|
3487
|
+
uri: string;
|
|
3488
|
+
durationUs: number;
|
|
3489
|
+
} | null;
|
|
3490
|
+
/** Cancel, release the player and drop the mix. */
|
|
3491
|
+
release(): void;
|
|
3492
|
+
}
|
|
3493
|
+
/** @experimental */
|
|
3494
|
+
declare function usePreviewAudio(options: UsePreviewAudioOptions): PreviewAudioBinding;
|
|
3495
|
+
|
|
3496
|
+
/**
|
|
3497
|
+
* One isolation control for the preview-audio hook, and nothing else.
|
|
3498
|
+
*
|
|
3499
|
+
* ## What it is for
|
|
3500
|
+
*
|
|
3501
|
+
* The picture stops advancing when a mix is loaded, and the minimal failing
|
|
3502
|
+
* configuration loads a *cached* mix — no decode, no encode, no codecs at all.
|
|
3503
|
+
* What is left in common is this hook being active, and being active means two
|
|
3504
|
+
* separable things: the native calls it makes, and the React state it
|
|
3505
|
+
* publishes. Five `useState` values re-render the editor while a surface, a
|
|
3506
|
+
* texture consumer and a composition callback are all being re-established.
|
|
3507
|
+
*
|
|
3508
|
+
* Suppressing the publications while leaving every native call in place
|
|
3509
|
+
* separates those two. If the freeze survives suppression it belongs to the
|
|
3510
|
+
* native side; if it disappears, the trigger is a render-driven ownership
|
|
3511
|
+
* transition and the fix belongs in whatever that render disturbs.
|
|
3512
|
+
*
|
|
3513
|
+
* ## What it is not
|
|
3514
|
+
*
|
|
3515
|
+
* Not a fix, and not reachable from a production build. Suppressed, the hook
|
|
3516
|
+
* cannot report readiness, so audio never plays — the control is deliberately
|
|
3517
|
+
* useless for anything except attribution. Anyone tempted to leave it on to
|
|
3518
|
+
* "reduce renders" is turning a diagnostic into a silent feature regression.
|
|
3519
|
+
*
|
|
3520
|
+
* @experimental
|
|
3521
|
+
*/
|
|
3522
|
+
/** @experimental */
|
|
3523
|
+
interface PreviewAudioDiagnostics {
|
|
3524
|
+
/** Drop the hook's React state publications, keeping every native call. */
|
|
3525
|
+
suppressPublications: boolean;
|
|
3526
|
+
/** Log each publication with its value. Off by default; noisy under a mix. */
|
|
3527
|
+
logPublications: boolean;
|
|
3528
|
+
/** Publications made, and publications suppressed. */
|
|
3529
|
+
published: number;
|
|
3530
|
+
suppressed: number;
|
|
3531
|
+
}
|
|
3532
|
+
/** @experimental */
|
|
3533
|
+
declare const previewAudioDiagnostics: PreviewAudioDiagnostics;
|
|
3534
|
+
|
|
3535
|
+
/**
|
|
3536
|
+
* Turning a compiled mix program's spans into files to decode.
|
|
3537
|
+
*
|
|
3538
|
+
* Shared by the exporter and by preview preparation, deliberately. The two must
|
|
3539
|
+
* agree about which file a span means, or a preview would be mixed from one set
|
|
3540
|
+
* of sources and the export from another — and the difference would only be
|
|
3541
|
+
* audible in the finished video.
|
|
3542
|
+
*
|
|
3543
|
+
* @experimental
|
|
3544
|
+
*/
|
|
3545
|
+
/**
|
|
3546
|
+
* What a span is, so a host can resolve it as what it actually is.
|
|
3547
|
+
*
|
|
3548
|
+
* A span carved from a clip's own audio resolves as that clip — the same
|
|
3549
|
+
* context the video pass used, so the same media resolves the same way. A span
|
|
3550
|
+
* from a music item resolves as that item. The id alone cannot say which, and a
|
|
3551
|
+
* host that guessed would resolve the first thing that matched.
|
|
3552
|
+
*/
|
|
3553
|
+
declare function mixSpanContext(project: VideoProject, spanId: string): ExportSourceContext | null;
|
|
3554
|
+
/** @experimental */
|
|
3555
|
+
interface ResolvedMixSources {
|
|
3556
|
+
/** Span id to readable URI, for every span that needs decoding. */
|
|
3557
|
+
uris: Record<string, string>;
|
|
3558
|
+
/** One sentence per span that could not be resolved. Empty means usable. */
|
|
3559
|
+
unresolved: string[];
|
|
3560
|
+
}
|
|
3561
|
+
/**
|
|
3562
|
+
* Resolve every span that needs decoding.
|
|
3563
|
+
*
|
|
3564
|
+
* Silent spans are skipped: they occupy their time and contribute no samples,
|
|
3565
|
+
* so asking a host to resolve one would be asking for a file that is not
|
|
3566
|
+
* needed — and a host that returned nothing would look like a failure.
|
|
3567
|
+
*
|
|
3568
|
+
* @experimental
|
|
3569
|
+
*/
|
|
3570
|
+
declare function resolveMixSources(project: VideoProject, program: AudioMixProgram, resolveSource: (source: MediaSource, context: ExportSourceContext) => string | null | undefined): ResolvedMixSources;
|
|
3571
|
+
|
|
3572
|
+
/**
|
|
3573
|
+
* The AAC encoder's frozen behaviour, re-measurable on demand.
|
|
3574
|
+
*
|
|
3575
|
+
* The encode loop is 16% of a preview preparation and was 89% of it before it
|
|
3576
|
+
* was rewritten. That rewrite was checked against a baseline of natively
|
|
3577
|
+
* generated PCM fixtures — inputs that are formulas rather than files, so the
|
|
3578
|
+
* two halves of the comparison cannot drift apart — and the check was exact:
|
|
3579
|
+
* access-unit count, payload bytes, payload hash, CSD, timestamps, decoded
|
|
3580
|
+
* frame count, RMS and transient positions all matched.
|
|
3581
|
+
*
|
|
3582
|
+
* This is the door to that. Diagnostics-gated in native, so a release build
|
|
3583
|
+
* without them refuses.
|
|
3584
|
+
*
|
|
3585
|
+
* @experimental
|
|
3586
|
+
*/
|
|
3587
|
+
/** @experimental */
|
|
3588
|
+
interface EncoderGateResult {
|
|
3589
|
+
ok: boolean;
|
|
3590
|
+
fixture?: string;
|
|
3591
|
+
/** Compressed shape. */
|
|
3592
|
+
units?: number;
|
|
3593
|
+
payloadBytes?: number;
|
|
3594
|
+
payloadHash?: number;
|
|
3595
|
+
csd0?: number;
|
|
3596
|
+
firstUs?: number;
|
|
3597
|
+
lastUs?: number;
|
|
3598
|
+
headTimestamps?: number[];
|
|
3599
|
+
/** Decoded result. */
|
|
3600
|
+
decodedFrames?: number;
|
|
3601
|
+
decodedRms?: number;
|
|
3602
|
+
peak?: number;
|
|
3603
|
+
markerPositionsMs?: number[];
|
|
3604
|
+
expectedMarkersMs?: number[];
|
|
3605
|
+
/** What it cost. */
|
|
3606
|
+
encodeMs?: number;
|
|
3607
|
+
inputFrames?: number;
|
|
3608
|
+
fileBytes?: number;
|
|
3609
|
+
error?: string;
|
|
3610
|
+
}
|
|
3611
|
+
/** The fixtures the gate can run. @experimental */
|
|
3612
|
+
declare const ENCODER_GATE_FIXTURES: readonly ["short", "acceptance", "silence", "saturated", "markers"];
|
|
3613
|
+
/**
|
|
3614
|
+
* Encode one fixture and describe the result completely.
|
|
3615
|
+
*
|
|
3616
|
+
* Never throws: a gate that is unavailable is an answer.
|
|
3617
|
+
*
|
|
3618
|
+
* @experimental
|
|
3619
|
+
*/
|
|
3620
|
+
declare function runEncoderGate(fixture: string, maxInputSize?: number): EncoderGateResult;
|
|
3621
|
+
|
|
3622
|
+
export { type AndroidThumbnailOptions, AndroidVideoPlayer, type AudioProbeFailure, type AudioProbeResult, CONVERGED_RUN, type CadenceClip, type CadenceResult, type CompositionPlayback, type CompositionPlaybackState, type CompositionSpikeItem, type CompositionSpikeSpec, type CompositionSpikeState, DEFAULT_CEILING_BYTES, DEFAULT_RISE_BYTES, DEFAULT_WINDOW_SAMPLES, type Distribution, ENCODER_GATE_FIXTURES, EXPORT_STAGES, type EncoderGateResult, type Excursion, type ExcursionCause, type ExcursionReport, type ExportJob, type ExportJobs, type ExportJobsOptions, type ExportStage, type IngestResult, MAX_OBSERVED_HEARTBEAT_GAP_MS, MAX_RECOVERY_SAMPLES, type ManagedSource, type ManagedSourceFailure, Media3, type Media3ExporterOptions, type Media3Playback, type Media3State, MediaSaveCancelled, MediaSaveFailed, type MemoryGuard, NO_SURFACE, type OverlayPreparationResult, type OverlayResourceDescriptor, type OverlayStagingSink, PHASE_MARK_PREFIX, PREVIEW_DRIFT_TOLERANCE_US, PREVIEW_OUTPUT_FPS, PREVIEW_TOLERANCE_US, type Phase, type PhaseReport, type PhaseState, type PhaseSummary, type PlaybackLifecycle, type PlaybackLifecycleEffects, type PlaybackStopReason, type PlaylistIntent, type PlaylistSession, type PreparedMix, type PreviewAudioBinding, type PreviewAudioDecision, type PreviewAudioDiagnostics, type PreviewAudioInputs, type PreviewAudioStatus, type PreviewMixResult, type PreviewParts, type ProgressPlan, type ProgressPlanInput, type ProgressReporter, type ProgressReporterOptions, RECOVERY_GRACE_MS, type RecoverySample, type ResolvedMixSources, SPIKE_TOLERANCE_US, STALL_GRACE_MS, type ScoredSubmission, type SeamStage, type SeamTrace, type ShardPlan, type SpikeSample, type StagedRaster, type StallKind, type StallVerdict, type StallWatchdog, type StallWatchdogOptions, type StartupRow, type StartupTrace, type SubmissionRecord, type SubmissionReport, type SubmissionTrace, type SurfaceCreateResult, type SurfaceOwner, type SurfaceOwnerEffects, type SurfaceReadiness, type SyncReport, type SyncTrace, type TickSample, type TraceEvent, type TracePage, type TraceRow, type UseCompositionPlaybackOptions, type UseMedia3PlaybackOptions, type UsePreviewAudioOptions, type Verdict, VideoExportRejected, type VideoSurface, type VideoSurfaceHandle, type VideoSurfaceOptions, afterPrepare, armPreviewTrace, beginPhase, cancelPreviewMix, checkCadence, classifyExcursions, classifyStall, compositionSpikeSpec, createAndroidThumbnailProvider, createExportJobs, createMedia3VideoExporter, createMediaStoreSaver, createMemoryGuard, createOverlayStagingSink, createPlaybackLifecycle, createProgressReporter, createStallWatchdog, createSurfaceOwner, createSyncTrace, decidePreviewAudio, describeTrip, distribution, drainAllPreviewTrace, drainPreviewTrace, expectedPtsAt, idlePlaylistSession, ingestManagedSource, installOverlayStaging, isCurrentGeneration, managedSourceLedger, managedSourceList, managedSourceToken, markPreviewTrace, measureStagingStall, mixSpanContext, muteSourceAudio, nextOwnerToken, observe, observeMemory, offsetAt, planProgressWeights, preparePreviewMix, previewAudioDiagnostics, previewAudioNeeded, previewCodecLedger, previewFramePipeline, previewMixKey, previewMixLedger, previewParts, previewProgramJson, previewSyncCounters, probeAudioSource, record as recordTick, recoveryFrames, releaseManagedSource, releasePreviewMix, resetSurfaceOwnerForTests, resolveMixSources, retireSurface, runEncoderGate, scoreSubmissions, seedManagedSourceForGate, setPreviewParts, shareSavedMedia, shouldPrepare, shouldRecover, spikeClaim, spikeDrain, spikePause, spikePlay, spikeRelease, spikeSeek, spikeStart, spikeState, stopPreviewForHost, summariseOffsets, summarisePhases, surfaceOwner, sweepManagedSources, sweepManagedSourcesRawForGate, sweepOverlayStaging, sweepPendingMediaSaves, sweepPreviewMixes, syncReport, thumbnailCounters, traceSeam, traceStartup, traceSubmission, useCompositionPlayback, useMedia3Playback, usePreviewAudio, useVideoSurface, videoSurfaceSlot, windowAround };
|