@pictor-sdk/video-android 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +40 -0
- package/android/CMakeLists.txt +30 -0
- package/android/build.gradle +119 -0
- package/android/src/main/AndroidManifest.xml +1 -0
- package/android/src/main/cpp/pictor_overlaystage.cpp +372 -0
- package/android/src/main/cpp/pictor_publish.cpp +87 -0
- package/android/src/main/cpp/pictor_videosurface.cpp +735 -0
- package/android/src/main/java/dev/pictor/video/AacPriming.kt +261 -0
- package/android/src/main/java/dev/pictor/video/AudioMixer.kt +1112 -0
- package/android/src/main/java/dev/pictor/video/AudioProbe.kt +162 -0
- package/android/src/main/java/dev/pictor/video/CadenceShaderProgram.kt +337 -0
- package/android/src/main/java/dev/pictor/video/CodecLedger.kt +126 -0
- package/android/src/main/java/dev/pictor/video/CompositionSpike.kt +978 -0
- package/android/src/main/java/dev/pictor/video/Diagnostics.kt +54 -0
- package/android/src/main/java/dev/pictor/video/EncoderGate.kt +476 -0
- package/android/src/main/java/dev/pictor/video/ExportGate.kt +1974 -0
- package/android/src/main/java/dev/pictor/video/ExportHeartbeat.kt +72 -0
- package/android/src/main/java/dev/pictor/video/ExportProgress.kt +103 -0
- package/android/src/main/java/dev/pictor/video/FrameClock.kt +591 -0
- package/android/src/main/java/dev/pictor/video/FramePending.kt +181 -0
- package/android/src/main/java/dev/pictor/video/GateAudio.kt +456 -0
- package/android/src/main/java/dev/pictor/video/GateDestination.kt +445 -0
- package/android/src/main/java/dev/pictor/video/GateFrames.kt +662 -0
- package/android/src/main/java/dev/pictor/video/GateMux.kt +185 -0
- package/android/src/main/java/dev/pictor/video/ManagedSources.kt +333 -0
- package/android/src/main/java/dev/pictor/video/Media3Player.kt +934 -0
- package/android/src/main/java/dev/pictor/video/MediaStoreSave.kt +419 -0
- package/android/src/main/java/dev/pictor/video/OverlayResources.kt +169 -0
- package/android/src/main/java/dev/pictor/video/PictorVideoModule.kt +1612 -0
- package/android/src/main/java/dev/pictor/video/PictorVideoPackage.kt +26 -0
- package/android/src/main/java/dev/pictor/video/PlaybackAnchor.kt +161 -0
- package/android/src/main/java/dev/pictor/video/PreviewAudioPlayer.kt +291 -0
- package/android/src/main/java/dev/pictor/video/PreviewConfig.kt +96 -0
- package/android/src/main/java/dev/pictor/video/PreviewMix.kt +396 -0
- package/android/src/main/java/dev/pictor/video/PreviewSync.kt +374 -0
- package/android/src/main/java/dev/pictor/video/PreviewTrace.kt +478 -0
- package/android/src/main/java/dev/pictor/video/PrimingTrim.kt +337 -0
- package/android/src/main/java/dev/pictor/video/RepeatingEncoderFactory.kt +97 -0
- package/android/src/main/java/dev/pictor/video/SceneProgram.kt +169 -0
- package/android/src/main/java/dev/pictor/video/SceneProgramNative.kt +299 -0
- package/android/src/main/java/dev/pictor/video/SkiaPassThroughEffect.kt +695 -0
- package/android/src/main/java/dev/pictor/video/SurfaceAttachment.kt +110 -0
- package/android/src/main/java/dev/pictor/video/ThumbnailDecoder.kt +171 -0
- package/android/src/main/java/dev/pictor/video/VideoSurface.kt +394 -0
- package/app.plugin.js +43 -0
- package/dist/index.d.mts +3622 -0
- package/dist/index.d.ts +3622 -0
- package/dist/index.js +3737 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +3625 -0
- package/dist/index.mjs.map +1 -0
- package/package.json +82 -0
- package/plugin/gradleProperties.d.ts +19 -0
- package/plugin/gradleProperties.js +64 -0
- package/react-native.config.js +13 -0
- package/src/NativePictorVideo.ts +574 -0
- package/src/audioProbe.ts +72 -0
- package/src/compositionSpike.ts +305 -0
- package/src/convergence.ts +112 -0
- package/src/encoderGate.ts +66 -0
- package/src/exportDestination.ts +149 -0
- package/src/exportJob.ts +232 -0
- package/src/exportProgress.ts +391 -0
- package/src/exportProgressPump.ts +190 -0
- package/src/exporter.ts +655 -0
- package/src/index.ts +276 -0
- package/src/managedSources.ts +252 -0
- package/src/media3.ts +412 -0
- package/src/mediaStore.ts +177 -0
- package/src/memoryGuard.ts +118 -0
- package/src/mixSources.ts +82 -0
- package/src/overlayStaging.ts +216 -0
- package/src/playbackLifecycle.ts +137 -0
- package/src/player.ts +39 -0
- package/src/playlistSession.ts +125 -0
- package/src/previewAudioDiagnostics.ts +67 -0
- package/src/previewAudioSession.ts +168 -0
- package/src/previewCadence.ts +950 -0
- package/src/previewMix.ts +212 -0
- package/src/previewTrace.ts +348 -0
- package/src/spikeExcursions.ts +405 -0
- package/src/submissionScoring.ts +680 -0
- package/src/surfaceOwner.ts +264 -0
- package/src/surfaceRecovery.ts +121 -0
- package/src/syncTrace.ts +189 -0
- package/src/thumbnailProvider.ts +163 -0
- package/src/tracePhases.ts +217 -0
- package/src/useCompositionPlayback.ts +327 -0
- package/src/useMedia3Playback.ts +492 -0
- package/src/usePreviewAudio.ts +305 -0
- package/src/useVideoSurface.ts +282 -0
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import {
|
|
2
|
+
createThumbnailPolicy,
|
|
3
|
+
thumbnailKey,
|
|
4
|
+
type MediaSource,
|
|
5
|
+
type AbortLike,
|
|
6
|
+
type Thumbnail,
|
|
7
|
+
type ThumbnailProvider,
|
|
8
|
+
type ThumbnailRequest,
|
|
9
|
+
} from '@pictor-sdk/video-core'
|
|
10
|
+
import { requireNative } from './player'
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* `ThumbnailProvider` over Media3's retriever, with the core policy on top.
|
|
14
|
+
*
|
|
15
|
+
* The division is the point: `@pictor-sdk/video-core` owns deduplication, the
|
|
16
|
+
* bounded cache and visible-range ordering, all of which are pure; this file
|
|
17
|
+
* owns the part that needs a codec. A host that writes its own provider — for a
|
|
18
|
+
* remote thumbnail service, say — reuses the policy and replaces only this.
|
|
19
|
+
*
|
|
20
|
+
* Results arrive as base64 JPEG rather than as a native image handle. A strip
|
|
21
|
+
* thumbnail is a few kilobytes and it is drawn by React Native's own image
|
|
22
|
+
* pipeline, so there is nothing here to leak and no ownership to transfer. The
|
|
23
|
+
* `AHardwareBuffer` lesson applies to full-resolution frames at 30 per second,
|
|
24
|
+
* not to a 64-pixel square that changes when the user scrolls.
|
|
25
|
+
*
|
|
26
|
+
* @experimental
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
/** @experimental */
|
|
30
|
+
export interface AndroidThumbnailOptions {
|
|
31
|
+
/**
|
|
32
|
+
* How many decoded thumbnails to keep.
|
|
33
|
+
*
|
|
34
|
+
* 240 covers several screens of a dense strip at 64 px — enough that
|
|
35
|
+
* scrolling back does not re-decode, bounded enough that a long project
|
|
36
|
+
* cannot grow without limit.
|
|
37
|
+
*/
|
|
38
|
+
capacity?: number
|
|
39
|
+
/** How often to check whether a decode has finished, in milliseconds. */
|
|
40
|
+
pollMs?: number
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const DEFAULT_CAPACITY = 240
|
|
44
|
+
const DEFAULT_POLL_MS = 16
|
|
45
|
+
|
|
46
|
+
/** @experimental */
|
|
47
|
+
export function createAndroidThumbnailProvider(
|
|
48
|
+
options: AndroidThumbnailOptions = {},
|
|
49
|
+
): ThumbnailProvider {
|
|
50
|
+
const policy = createThumbnailPolicy<string>({
|
|
51
|
+
capacity: options.capacity ?? DEFAULT_CAPACITY,
|
|
52
|
+
})
|
|
53
|
+
const pollMs = options.pollMs ?? DEFAULT_POLL_MS
|
|
54
|
+
/** Tokens in flight, by request key, so a cancel can reach the native side. */
|
|
55
|
+
const tokens = new Map<string, number>()
|
|
56
|
+
|
|
57
|
+
function uriOf(source: MediaSource): string {
|
|
58
|
+
if (typeof source === 'string') return source
|
|
59
|
+
if (source && typeof source === 'object' && 'uri' in source) {
|
|
60
|
+
const uri = (source as { uri?: unknown }).uri
|
|
61
|
+
if (typeof uri === 'string') return uri
|
|
62
|
+
}
|
|
63
|
+
throw new Error(
|
|
64
|
+
`@pictor-sdk/video-android: a thumbnail source is not a URI. Normalise host media to { kind: 'uri', uri } first. Received: ${JSON.stringify(source)}`,
|
|
65
|
+
)
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
return {
|
|
69
|
+
async request(request: ThumbnailRequest, signal?: AbortLike): Promise<Thumbnail | null> {
|
|
70
|
+
const key = thumbnailKey(request)
|
|
71
|
+
const cached = policy.get(request)
|
|
72
|
+
if (cached !== undefined) {
|
|
73
|
+
return { image: cached, sourceTimeUs: request.sourceTimeUs, size: { width: 0, height: 0 } }
|
|
74
|
+
}
|
|
75
|
+
if (signal?.aborted) return null
|
|
76
|
+
|
|
77
|
+
const native = requireNative()
|
|
78
|
+
policy.begin(request)
|
|
79
|
+
const token = native.thumbnailRequest(
|
|
80
|
+
uriOf(request.source),
|
|
81
|
+
request.sourceTimeUs,
|
|
82
|
+
request.widthPx,
|
|
83
|
+
)
|
|
84
|
+
tokens.set(key, token)
|
|
85
|
+
|
|
86
|
+
const stop = () => {
|
|
87
|
+
native.thumbnailCancel(token)
|
|
88
|
+
tokens.delete(key)
|
|
89
|
+
policy.cancel(request)
|
|
90
|
+
}
|
|
91
|
+
signal?.addEventListener('abort', stop, { once: true })
|
|
92
|
+
|
|
93
|
+
try {
|
|
94
|
+
// Poll rather than await a bridged Promise: the native side is a token
|
|
95
|
+
// API precisely so a request can be abandoned, and a Promise that
|
|
96
|
+
// cannot be cancelled would decode frames nobody will see.
|
|
97
|
+
for (;;) {
|
|
98
|
+
if (signal?.aborted) return null
|
|
99
|
+
const data = native.thumbnailTake(token)
|
|
100
|
+
if (data !== null && data !== undefined) {
|
|
101
|
+
tokens.delete(key)
|
|
102
|
+
const evicted = policy.store(request, `data:image/jpeg;base64,${data}`)
|
|
103
|
+
// Base64 strings are garbage-collected; eviction is bookkeeping
|
|
104
|
+
// rather than disposal, and the returned values are ignored on
|
|
105
|
+
// purpose. A provider handing back native images would dispose here.
|
|
106
|
+
void evicted
|
|
107
|
+
return {
|
|
108
|
+
image: policy.get(request)!,
|
|
109
|
+
sourceTimeUs: request.sourceTimeUs,
|
|
110
|
+
size: { width: request.widthPx, height: 0 },
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
const error = native.thumbnailError(token)
|
|
114
|
+
if (error !== null && error !== undefined) {
|
|
115
|
+
tokens.delete(key)
|
|
116
|
+
policy.cancel(request)
|
|
117
|
+
return null
|
|
118
|
+
}
|
|
119
|
+
if (!native.thumbnailRunning(token)) {
|
|
120
|
+
// Neither a result nor an error, and not running: the frame could
|
|
121
|
+
// not be produced. A source can legitimately have no frame at a
|
|
122
|
+
// time — past the end of a variable-rate clip, for instance.
|
|
123
|
+
tokens.delete(key)
|
|
124
|
+
policy.cancel(request)
|
|
125
|
+
return null
|
|
126
|
+
}
|
|
127
|
+
await new Promise<void>((resolve) => setTimeout(() => resolve(), pollMs))
|
|
128
|
+
}
|
|
129
|
+
} finally {
|
|
130
|
+
signal?.removeEventListener('abort', stop)
|
|
131
|
+
}
|
|
132
|
+
},
|
|
133
|
+
|
|
134
|
+
releaseSource(source: MediaSource) {
|
|
135
|
+
policy.evictSource(source)
|
|
136
|
+
requireNative().thumbnailReleaseSource(uriOf(source))
|
|
137
|
+
},
|
|
138
|
+
|
|
139
|
+
release() {
|
|
140
|
+
for (const token of tokens.values()) requireNative().thumbnailCancel(token)
|
|
141
|
+
tokens.clear()
|
|
142
|
+
policy.clear()
|
|
143
|
+
requireNative().thumbnailReleaseAll()
|
|
144
|
+
},
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** Decode counters, for gates and diagnostics. @experimental */
|
|
149
|
+
export function thumbnailCounters(): {
|
|
150
|
+
decoded: number
|
|
151
|
+
cancelled: number
|
|
152
|
+
failed: number
|
|
153
|
+
running: number
|
|
154
|
+
retrievers: number
|
|
155
|
+
} {
|
|
156
|
+
return requireNative().thumbnailCounters() as {
|
|
157
|
+
decoded: number
|
|
158
|
+
cancelled: number
|
|
159
|
+
failed: number
|
|
160
|
+
running: number
|
|
161
|
+
retrievers: number
|
|
162
|
+
}
|
|
163
|
+
}
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
import type { TraceEvent } from './previewTrace'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The offset, segmented by the marks a gate put in the run.
|
|
5
|
+
*
|
|
6
|
+
* ## Why a whole-run summary is not enough
|
|
7
|
+
*
|
|
8
|
+
* `summariseOffsets` answers "how bad did it get", and a gate needs "where".
|
|
9
|
+
* Measured on a Pixel 7, a run whose worst excursion was 266 ms spent every
|
|
10
|
+
* moment between its transitions inside ±40 ms: the whole-run worst described
|
|
11
|
+
* three settling transients and nothing else, and a single number cannot tell
|
|
12
|
+
* that from a steady 266 ms error. They are not the same defect and they do not
|
|
13
|
+
* have the same fix.
|
|
14
|
+
*
|
|
15
|
+
* So a run is cut at its marks, and each segment is summarised on its own. The
|
|
16
|
+
* gate verdict is still the run's worst — this localises a failure, it does not
|
|
17
|
+
* soften one.
|
|
18
|
+
*
|
|
19
|
+
* ## Fresh and stale are reported, never traded
|
|
20
|
+
*
|
|
21
|
+
* A stale presented frame is what a person actually sees, so its offset is real
|
|
22
|
+
* and is counted in `worstAbsUs`. But a stale frame's offset grows purely
|
|
23
|
+
* because audio advanced while the picture did not, which is a different fault
|
|
24
|
+
* from two clocks disagreeing — so `freshWorstAbsUs` is reported beside it
|
|
25
|
+
* rather than instead of it. Neither is allowed to stand alone: quoting only
|
|
26
|
+
* the fresh figure is how a run with a nine-second frozen picture passes a
|
|
27
|
+
* synchronisation gate.
|
|
28
|
+
*
|
|
29
|
+
* @experimental
|
|
30
|
+
*/
|
|
31
|
+
export interface PhaseSummary {
|
|
32
|
+
/** The mark that opened this segment, or `'start'` before the first one. */
|
|
33
|
+
label: string
|
|
34
|
+
index: number
|
|
35
|
+
/** Samples in the segment, whatever their state. */
|
|
36
|
+
samples: number
|
|
37
|
+
/** Samples with both players playing — the only ones an offset means anything for. */
|
|
38
|
+
playingSamples: number
|
|
39
|
+
/** Of those, the ones whose presented frame had advanced. */
|
|
40
|
+
freshPlayingSamples: number
|
|
41
|
+
stalePlayingSamples: number
|
|
42
|
+
/** The longest unbroken run of stale samples, in samples. */
|
|
43
|
+
longestStaleRun: number
|
|
44
|
+
minUs: number
|
|
45
|
+
maxUs: number
|
|
46
|
+
meanUs: number
|
|
47
|
+
/** Worst absolute offset over every playing sample, stale included. */
|
|
48
|
+
worstAbsUs: number
|
|
49
|
+
/** Worst over fresh samples only. Reported beside `worstAbsUs`, never instead. */
|
|
50
|
+
freshWorstAbsUs: number
|
|
51
|
+
/** Playing samples outside the tolerance. */
|
|
52
|
+
overToleranceSamples: number
|
|
53
|
+
/**
|
|
54
|
+
* How long the segment took to come inside tolerance and stay there.
|
|
55
|
+
*
|
|
56
|
+
* `-1` when it never did, which is the difference between a transient and a
|
|
57
|
+
* failure that happens to start at a transition. Measured from the segment's
|
|
58
|
+
* first playing sample, so it is a settling time rather than a wall clock.
|
|
59
|
+
*/
|
|
60
|
+
settledAfterUs: number
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** @experimental */
|
|
64
|
+
export interface PhaseReport {
|
|
65
|
+
toleranceUs: number
|
|
66
|
+
phases: PhaseSummary[]
|
|
67
|
+
/** The run's worst, over every phase. This is the gate. */
|
|
68
|
+
worstAbsUs: number
|
|
69
|
+
freshWorstAbsUs: number
|
|
70
|
+
overToleranceSamples: number
|
|
71
|
+
/** Whether every playing sample in the run was inside tolerance. */
|
|
72
|
+
pass: boolean
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* The prefix a gate's own marks carry.
|
|
77
|
+
*
|
|
78
|
+
* Every mark is recorded as a `mark` event carrying its label, and the trace is
|
|
79
|
+
* also full of system events — holds, releases, drift corrections, transport
|
|
80
|
+
* transitions. Cutting on "not a sample" would start a new phase at every one
|
|
81
|
+
* of those, and cutting on every mark would let an ad-hoc label reshape a gate's
|
|
82
|
+
* phases. So only marks whose label carries this prefix cut the run.
|
|
83
|
+
*
|
|
84
|
+
* @experimental
|
|
85
|
+
*/
|
|
86
|
+
export const PHASE_MARK_PREFIX = 'phase:'
|
|
87
|
+
|
|
88
|
+
function num(v: unknown): number {
|
|
89
|
+
return typeof v === 'number' ? v : Number(v)
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function empty(label: string, index: number): PhaseSummary {
|
|
93
|
+
return {
|
|
94
|
+
label,
|
|
95
|
+
index,
|
|
96
|
+
samples: 0,
|
|
97
|
+
playingSamples: 0,
|
|
98
|
+
freshPlayingSamples: 0,
|
|
99
|
+
stalePlayingSamples: 0,
|
|
100
|
+
longestStaleRun: 0,
|
|
101
|
+
minUs: 0,
|
|
102
|
+
maxUs: 0,
|
|
103
|
+
meanUs: 0,
|
|
104
|
+
worstAbsUs: 0,
|
|
105
|
+
freshWorstAbsUs: 0,
|
|
106
|
+
overToleranceSamples: 0,
|
|
107
|
+
settledAfterUs: 0,
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Summarise a drained run, cut at its mark events.
|
|
113
|
+
*
|
|
114
|
+
* @experimental
|
|
115
|
+
*/
|
|
116
|
+
export function summarisePhases(
|
|
117
|
+
events: readonly TraceEvent[],
|
|
118
|
+
toleranceUs: number,
|
|
119
|
+
): PhaseReport {
|
|
120
|
+
const phases: PhaseSummary[] = []
|
|
121
|
+
let current = empty('start', 0)
|
|
122
|
+
// Accumulators kept outside the summary so the published shape stays plain
|
|
123
|
+
// data — a gate serialises this straight into a log line.
|
|
124
|
+
let sum = 0
|
|
125
|
+
let staleRun = 0
|
|
126
|
+
let firstPlayingNs = 0
|
|
127
|
+
let lastOverNs = 0
|
|
128
|
+
let sawPlaying = false
|
|
129
|
+
let stillOver = false
|
|
130
|
+
|
|
131
|
+
const close = () => {
|
|
132
|
+
current.meanUs = current.playingSamples > 0 ? Math.round(sum / current.playingSamples) : 0
|
|
133
|
+
/*
|
|
134
|
+
Settling is measured to the *last* excursion, not the first return.
|
|
135
|
+
|
|
136
|
+
A transient that goes out, comes back and goes out again has not settled
|
|
137
|
+
at the first return, and reporting that would turn the worst case into the
|
|
138
|
+
best-looking number available. -1 for a phase still outside tolerance at
|
|
139
|
+
its final sample: that is not a transient at all.
|
|
140
|
+
*/
|
|
141
|
+
current.settledAfterUs =
|
|
142
|
+
current.overToleranceSamples === 0
|
|
143
|
+
? 0
|
|
144
|
+
: stillOver
|
|
145
|
+
? -1
|
|
146
|
+
: Math.round((lastOverNs - firstPlayingNs) / 1000)
|
|
147
|
+
phases.push(current)
|
|
148
|
+
sum = 0
|
|
149
|
+
staleRun = 0
|
|
150
|
+
firstPlayingNs = 0
|
|
151
|
+
lastOverNs = 0
|
|
152
|
+
sawPlaying = false
|
|
153
|
+
stillOver = false
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
for (const e of events) {
|
|
157
|
+
if (e.type !== 'sample') {
|
|
158
|
+
const label = e.type === 'mark' ? String(e.v.label ?? '') : ''
|
|
159
|
+
if (label.startsWith(PHASE_MARK_PREFIX)) {
|
|
160
|
+
close()
|
|
161
|
+
current = empty(label.slice(PHASE_MARK_PREFIX.length), phases.length)
|
|
162
|
+
}
|
|
163
|
+
continue
|
|
164
|
+
}
|
|
165
|
+
current.samples += 1
|
|
166
|
+
if (e.v.videoPlaying !== true || e.v.audioPlaying !== true) {
|
|
167
|
+
staleRun = 0
|
|
168
|
+
continue
|
|
169
|
+
}
|
|
170
|
+
const offset = num(e.v.offsetUs)
|
|
171
|
+
if (!Number.isFinite(offset)) continue
|
|
172
|
+
current.playingSamples += 1
|
|
173
|
+
if (!sawPlaying) {
|
|
174
|
+
sawPlaying = true
|
|
175
|
+
firstPlayingNs = num(e.atNs)
|
|
176
|
+
}
|
|
177
|
+
sum += offset
|
|
178
|
+
if (current.playingSamples === 1) {
|
|
179
|
+
current.minUs = offset
|
|
180
|
+
current.maxUs = offset
|
|
181
|
+
} else {
|
|
182
|
+
current.minUs = Math.min(current.minUs, offset)
|
|
183
|
+
current.maxUs = Math.max(current.maxUs, offset)
|
|
184
|
+
}
|
|
185
|
+
const abs = Math.abs(offset)
|
|
186
|
+
if (abs > current.worstAbsUs) current.worstAbsUs = abs
|
|
187
|
+
if (abs > toleranceUs) {
|
|
188
|
+
current.overToleranceSamples += 1
|
|
189
|
+
lastOverNs = num(e.atNs)
|
|
190
|
+
stillOver = true
|
|
191
|
+
} else {
|
|
192
|
+
stillOver = false
|
|
193
|
+
}
|
|
194
|
+
if (e.v.pfFresh === true) {
|
|
195
|
+
current.freshPlayingSamples += 1
|
|
196
|
+
if (abs > current.freshWorstAbsUs) current.freshWorstAbsUs = abs
|
|
197
|
+
staleRun = 0
|
|
198
|
+
} else {
|
|
199
|
+
current.stalePlayingSamples += 1
|
|
200
|
+
staleRun += 1
|
|
201
|
+
if (staleRun > current.longestStaleRun) current.longestStaleRun = staleRun
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
close()
|
|
205
|
+
|
|
206
|
+
const worstAbsUs = phases.reduce((w, p) => Math.max(w, p.worstAbsUs), 0)
|
|
207
|
+
const freshWorstAbsUs = phases.reduce((w, p) => Math.max(w, p.freshWorstAbsUs), 0)
|
|
208
|
+
const overToleranceSamples = phases.reduce((n, p) => n + p.overToleranceSamples, 0)
|
|
209
|
+
return {
|
|
210
|
+
toleranceUs,
|
|
211
|
+
phases,
|
|
212
|
+
worstAbsUs,
|
|
213
|
+
freshWorstAbsUs,
|
|
214
|
+
overToleranceSamples,
|
|
215
|
+
pass: overToleranceSamples === 0,
|
|
216
|
+
}
|
|
217
|
+
}
|
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
|
|
2
|
+
import {
|
|
3
|
+
playIntent,
|
|
4
|
+
playlistOf,
|
|
5
|
+
projectDuration,
|
|
6
|
+
type PlayIntent,
|
|
7
|
+
type UnsupportedSource,
|
|
8
|
+
type VideoProject,
|
|
9
|
+
} from '@pictor-sdk/video-core'
|
|
10
|
+
import {
|
|
11
|
+
compositionSpikeSpec,
|
|
12
|
+
type CompositionSpikeState,
|
|
13
|
+
spikePause,
|
|
14
|
+
spikePlay,
|
|
15
|
+
spikeRelease,
|
|
16
|
+
spikeSeek,
|
|
17
|
+
spikeStart,
|
|
18
|
+
spikeState,
|
|
19
|
+
} from './compositionSpike'
|
|
20
|
+
import { createPlaybackLifecycle, type PlaybackLifecycle } from './playbackLifecycle'
|
|
21
|
+
import type { Media3Playback } from './useMedia3Playback'
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* The preview, on one clock.
|
|
25
|
+
*
|
|
26
|
+
* ## What this replaces
|
|
27
|
+
*
|
|
28
|
+
* Two players kept in step: a video player and a separate player for the
|
|
29
|
+
* prepared audio mix, with a drift check correcting between them. That
|
|
30
|
+
* architecture was measured to its limit and could not hold synchronisation
|
|
31
|
+
* across a clip seam — the correction is a seek, a seek costs a decode, and a
|
|
32
|
+
* decode at the moment of a seam is what the correction was trying to cover.
|
|
33
|
+
*
|
|
34
|
+
* A `CompositionPlayer` has one clock, one playback thread and one audio graph.
|
|
35
|
+
* The prepared mix becomes an audio-only sequence beside the video sequence, so
|
|
36
|
+
* the two cannot drift: there is nothing to drift against.
|
|
37
|
+
*
|
|
38
|
+
* ## What it does not fix
|
|
39
|
+
*
|
|
40
|
+
* The clip seam still costs a picture. Media3 tears down the video decoder at
|
|
41
|
+
* an item boundary and builds another, and for 119 to 168 ms nothing reaches
|
|
42
|
+
* the renderer while the clock runs on — the audio is continuous and the
|
|
43
|
+
* picture holds. That behaviour was watched on a device and accepted for v1;
|
|
44
|
+
* see `docs/research/preview-acceptance.md` for what was accepted and what was
|
|
45
|
+
* explicitly not.
|
|
46
|
+
*
|
|
47
|
+
* ## Why the same contract
|
|
48
|
+
*
|
|
49
|
+
* It returns {@link Media3Playback}, unchanged. The editor, the transport and
|
|
50
|
+
* the gates were all written against it, and swapping the engine underneath is
|
|
51
|
+
* a smaller and more reviewable change than rewriting every caller to a shape
|
|
52
|
+
* that would end up equivalent.
|
|
53
|
+
*
|
|
54
|
+
* @experimental
|
|
55
|
+
*/
|
|
56
|
+
export interface UseCompositionPlaybackOptions {
|
|
57
|
+
project: VideoProject
|
|
58
|
+
/** Timeline time the host wants shown. */
|
|
59
|
+
playheadUs: number
|
|
60
|
+
/**
|
|
61
|
+
* The prepared audio program, from `usePreviewAudio`.
|
|
62
|
+
*
|
|
63
|
+
* Null while it is being prepared, after it has failed, and — legitimately —
|
|
64
|
+
* when the document has no audio at all.
|
|
65
|
+
*
|
|
66
|
+
* Those are different situations and the hook has to tell them apart. A
|
|
67
|
+
* document *with* audio waits: building the composition without its sound
|
|
68
|
+
* would play a silent preview and gain the music a second later, which is two
|
|
69
|
+
* timelines to anyone watching. A document *without* audio has nothing to
|
|
70
|
+
* wait for, and refusing to build would leave the picture dead.
|
|
71
|
+
*/
|
|
72
|
+
mix: { uri: string; durationUs: number } | null
|
|
73
|
+
/** Turns a document source into something the player can open. */
|
|
74
|
+
resolveSource: (source: { kind: string; uri?: string }) => string | null
|
|
75
|
+
/** @see UseMedia3PlaybackOptions.surfaceGeneration */
|
|
76
|
+
surfaceGeneration?: number
|
|
77
|
+
enabled?: boolean
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** How often the composition's state is read, in milliseconds. */
|
|
81
|
+
const POLL_MS = 100
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* What a composition knows about itself.
|
|
85
|
+
*
|
|
86
|
+
* Deliberately not `Media3State`. That type is the shape the video player's
|
|
87
|
+
* native state comes back in — forty fields about queue depth, anchors, native
|
|
88
|
+
* heap and window indices — and a composition has none of them. Synthesising
|
|
89
|
+
* one would be inventing forty numbers to satisfy a type, and a reader would
|
|
90
|
+
* have no way to tell the invented ones from the measured ones.
|
|
91
|
+
*
|
|
92
|
+
* @experimental
|
|
93
|
+
*/
|
|
94
|
+
export interface CompositionPlaybackState {
|
|
95
|
+
playing: boolean
|
|
96
|
+
ready: boolean
|
|
97
|
+
buffering: boolean
|
|
98
|
+
ended: boolean
|
|
99
|
+
released: boolean
|
|
100
|
+
/** Project time, which for a single composition is the only time there is. */
|
|
101
|
+
positionUs: number
|
|
102
|
+
durationUs: number
|
|
103
|
+
/** The presentation size, or 0 while nothing has been measured. */
|
|
104
|
+
width: number
|
|
105
|
+
height: number
|
|
106
|
+
error: string | null
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function composedState(snapshot: CompositionSpikeState): CompositionPlaybackState {
|
|
110
|
+
const playbackState = snapshot.playbackState ?? 1
|
|
111
|
+
return {
|
|
112
|
+
playing: snapshot.playing === true,
|
|
113
|
+
ready: playbackState === 3,
|
|
114
|
+
buffering: playbackState === 2,
|
|
115
|
+
ended: playbackState === 4,
|
|
116
|
+
// `released` counts releases; a positive count with nothing attached is the
|
|
117
|
+
// released state this means.
|
|
118
|
+
released: snapshot.attached !== true && (snapshot.released ?? 0) > 0,
|
|
119
|
+
positionUs: snapshot.positionUs ?? 0,
|
|
120
|
+
durationUs: snapshot.durationUs ?? 0,
|
|
121
|
+
/*
|
|
122
|
+
Zero, and honestly so.
|
|
123
|
+
|
|
124
|
+
A composition renders into the presentation surface at the size the host
|
|
125
|
+
gave it; it does not report a decoded size the way the video player did.
|
|
126
|
+
The host already knows the size it asked for, and a fabricated number here
|
|
127
|
+
would be worse than none.
|
|
128
|
+
*/
|
|
129
|
+
width: 0,
|
|
130
|
+
height: 0,
|
|
131
|
+
error: snapshot.error ?? null,
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** @experimental */
|
|
136
|
+
export type CompositionPlayback = Omit<Media3Playback, 'state'> & {
|
|
137
|
+
state: CompositionPlaybackState | null
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** @experimental */
|
|
141
|
+
export function useCompositionPlayback(
|
|
142
|
+
options: UseCompositionPlaybackOptions,
|
|
143
|
+
): CompositionPlayback {
|
|
144
|
+
const { project, mix, enabled = true } = options
|
|
145
|
+
const surfaceGeneration = options.surfaceGeneration ?? 0
|
|
146
|
+
const resolveSource = useRef(options.resolveSource)
|
|
147
|
+
resolveSource.current = options.resolveSource
|
|
148
|
+
|
|
149
|
+
const [state, setState] = useState<CompositionPlaybackState | null>(null)
|
|
150
|
+
const [playing, setPlaying] = useState(false)
|
|
151
|
+
const playheadRef = useRef(options.playheadUs)
|
|
152
|
+
playheadRef.current = options.playheadUs
|
|
153
|
+
|
|
154
|
+
const playlist = useMemo(() => playlistOf(project), [project])
|
|
155
|
+
const items = playlist.items
|
|
156
|
+
const durationUs = useMemo(() => projectDuration(project), [project])
|
|
157
|
+
|
|
158
|
+
/*
|
|
159
|
+
What the composition is made of, as a string.
|
|
160
|
+
|
|
161
|
+
Every edit makes a new project object and most of them — a selection, a
|
|
162
|
+
playhead move — change nothing the player should be playing. The mix is part
|
|
163
|
+
of it: a rebuilt mix is a different composition even when the video is
|
|
164
|
+
identical, and starting the old one would play the sound the user just
|
|
165
|
+
changed.
|
|
166
|
+
*/
|
|
167
|
+
// A document with audio has a mix to wait for; one without does not.
|
|
168
|
+
const needsMix = project.audio.length > 0
|
|
169
|
+
const signature = useMemo(() => {
|
|
170
|
+
if (needsMix && mix === null) return ''
|
|
171
|
+
const parts = items.map(
|
|
172
|
+
(i) => `${JSON.stringify(i.source)}|${i.startUs}|${i.endUs}|${i.speed}|${i.muted}`,
|
|
173
|
+
)
|
|
174
|
+
const sound = mix === null ? 'silent' : `${mix.uri}:${mix.durationUs}`
|
|
175
|
+
return `${parts.join(';')}#${playlist.unsupported.length}@${sound}`
|
|
176
|
+
}, [items, playlist.unsupported.length, mix, needsMix])
|
|
177
|
+
|
|
178
|
+
const [unsupported, setUnsupported] = useState<readonly UnsupportedSource[]>([])
|
|
179
|
+
const lastSeekRef = useRef<Media3Playback['lastSeek']>(null)
|
|
180
|
+
|
|
181
|
+
/*
|
|
182
|
+
This screen's ownership, unchanged from the path it replaces.
|
|
183
|
+
|
|
184
|
+
The player outlives any screen: leaving the editor while playing left the
|
|
185
|
+
sound running, and backgrounding did the same. One lifecycle per hook
|
|
186
|
+
instance, and a screen mounted later starts from nothing.
|
|
187
|
+
*/
|
|
188
|
+
const lifecycleRef = useRef<PlaybackLifecycle | null>(null)
|
|
189
|
+
if (lifecycleRef.current === null) {
|
|
190
|
+
lifecycleRef.current = createPlaybackLifecycle({
|
|
191
|
+
pause: () => void spikePause(),
|
|
192
|
+
release: () => void spikeRelease(),
|
|
193
|
+
})
|
|
194
|
+
}
|
|
195
|
+
const lifecycle = lifecycleRef.current
|
|
196
|
+
const [lifecycleEpoch, setLifecycleEpoch] = useState(lifecycle.epoch)
|
|
197
|
+
|
|
198
|
+
const loaded = useRef('')
|
|
199
|
+
useEffect(() => {
|
|
200
|
+
if (!enabled || signature === '') return
|
|
201
|
+
if (surfaceGeneration === 0) return
|
|
202
|
+
if (loaded.current === signature) return
|
|
203
|
+
/*
|
|
204
|
+
An empty URI is a composition with no audio sequence, which the adapter
|
|
205
|
+
builds and plays. It is not a missing mix — that case never reaches here,
|
|
206
|
+
because the signature is empty until one exists.
|
|
207
|
+
*/
|
|
208
|
+
const { spec, unsupported: refused } = compositionSpikeSpec(
|
|
209
|
+
project,
|
|
210
|
+
mix ?? { uri: '', durationUs: 0 },
|
|
211
|
+
(s) => resolveSource.current(s),
|
|
212
|
+
)
|
|
213
|
+
setUnsupported(
|
|
214
|
+
refused.map((reason) => ({ clipId: reason.split(':')[0] ?? '', reason }) as UnsupportedSource),
|
|
215
|
+
)
|
|
216
|
+
if (spec.items.length === 0) return
|
|
217
|
+
const started = spikeStart(spec) as { ok?: boolean }
|
|
218
|
+
// A refusal is not a load. Recording it as one is what left an editor
|
|
219
|
+
// waiting for an edit the person had already made.
|
|
220
|
+
if (started.ok === true) loaded.current = signature
|
|
221
|
+
}, [enabled, signature, surfaceGeneration, project, mix])
|
|
222
|
+
|
|
223
|
+
useEffect(() => {
|
|
224
|
+
return () => {
|
|
225
|
+
lifecycle.teardown('unmount')
|
|
226
|
+
}
|
|
227
|
+
}, [lifecycle])
|
|
228
|
+
|
|
229
|
+
useEffect(() => {
|
|
230
|
+
if (!enabled) return
|
|
231
|
+
// Captured now and checked on every tick: an interval is not cleared the
|
|
232
|
+
// instant a screen goes, and a poll landing afterwards must not write into
|
|
233
|
+
// one that is no longer there.
|
|
234
|
+
const epoch = lifecycle.epoch
|
|
235
|
+
const id = setInterval(() => {
|
|
236
|
+
if (!lifecycle.accepts(epoch)) return
|
|
237
|
+
const next = composedState(spikeState())
|
|
238
|
+
/*
|
|
239
|
+
Only a change of meaning re-renders.
|
|
240
|
+
|
|
241
|
+
Replacing the state object every tick re-renders the editor ten times a
|
|
242
|
+
second to move a line, and at ten hertz the line moves visibly in steps.
|
|
243
|
+
A host that wants the position drives it from `positionUs()`.
|
|
244
|
+
*/
|
|
245
|
+
setState((previous: CompositionPlaybackState | null) =>
|
|
246
|
+
previous !== null &&
|
|
247
|
+
previous.playing === next.playing &&
|
|
248
|
+
previous.ready === next.ready &&
|
|
249
|
+
previous.buffering === next.buffering &&
|
|
250
|
+
previous.ended === next.ended &&
|
|
251
|
+
previous.released === next.released &&
|
|
252
|
+
previous.error === next.error
|
|
253
|
+
? previous
|
|
254
|
+
: next,
|
|
255
|
+
)
|
|
256
|
+
setPlaying(next.playing)
|
|
257
|
+
}, POLL_MS)
|
|
258
|
+
return () => clearInterval(id)
|
|
259
|
+
}, [enabled, lifecycle, lifecycleEpoch])
|
|
260
|
+
|
|
261
|
+
const seek = useCallback((timeUs: number) => {
|
|
262
|
+
const at = Math.max(0, Math.round(timeUs))
|
|
263
|
+
lastSeekRef.current = { globalUs: at, windowIndex: 0, localUs: at }
|
|
264
|
+
spikeSeek(at)
|
|
265
|
+
}, [])
|
|
266
|
+
|
|
267
|
+
const positionUs = useCallback((): number | null => {
|
|
268
|
+
const snapshot = spikeState()
|
|
269
|
+
if (!snapshot.attached) return null
|
|
270
|
+
const at = snapshot.positionUs
|
|
271
|
+
return typeof at === 'number' ? at : null
|
|
272
|
+
}, [])
|
|
273
|
+
|
|
274
|
+
const play = useCallback((): PlayIntent => {
|
|
275
|
+
const snapshot = spikeState()
|
|
276
|
+
const intent = playIntent({
|
|
277
|
+
positionUs: typeof snapshot.positionUs === 'number' ? snapshot.positionUs : 0,
|
|
278
|
+
durationUs,
|
|
279
|
+
playbackState: typeof snapshot.playbackState === 'number' ? snapshot.playbackState : 1,
|
|
280
|
+
ended: snapshot.playbackState === 4,
|
|
281
|
+
})
|
|
282
|
+
if (intent.seekToUs !== null) seek(intent.seekToUs)
|
|
283
|
+
if (!intent.play) return intent
|
|
284
|
+
// Refused once the screen has given the player back: a late press from a
|
|
285
|
+
// control that has not unmounted yet must not start a released player.
|
|
286
|
+
if (lifecycle.released) return intent
|
|
287
|
+
spikePlay()
|
|
288
|
+
setPlaying(true)
|
|
289
|
+
return intent
|
|
290
|
+
}, [durationUs, lifecycle, seek])
|
|
291
|
+
|
|
292
|
+
useEffect(() => {
|
|
293
|
+
setLifecycleEpoch(lifecycle.epoch)
|
|
294
|
+
}, [lifecycle, surfaceGeneration])
|
|
295
|
+
|
|
296
|
+
return {
|
|
297
|
+
playing,
|
|
298
|
+
play,
|
|
299
|
+
get lastSeek() {
|
|
300
|
+
return lastSeekRef.current
|
|
301
|
+
},
|
|
302
|
+
positionUs,
|
|
303
|
+
effectiveRate: 1,
|
|
304
|
+
pause: useCallback(() => {
|
|
305
|
+
spikePause()
|
|
306
|
+
setPlaying(false)
|
|
307
|
+
}, []),
|
|
308
|
+
/*
|
|
309
|
+
Rate is the document's, not the transport's.
|
|
310
|
+
|
|
311
|
+
Clip speed is compiled into the composition and into the mix, and a
|
|
312
|
+
preview-wide rate on top of it would be a second speed nobody asked for.
|
|
313
|
+
The contract keeps the method; this path has nothing to do with it.
|
|
314
|
+
*/
|
|
315
|
+
setRate: useCallback(() => {}, []),
|
|
316
|
+
seek,
|
|
317
|
+
state,
|
|
318
|
+
items,
|
|
319
|
+
unsupported,
|
|
320
|
+
divergenceUs: 0,
|
|
321
|
+
lifecycle,
|
|
322
|
+
// One player, so there is no half to stall: the recovery this reported was
|
|
323
|
+
// the video player running while the audio player had stopped, or the
|
|
324
|
+
// reverse. @see docs/research/preview-acceptance.md
|
|
325
|
+
stall: null,
|
|
326
|
+
}
|
|
327
|
+
}
|