@pictor-sdk/video-android 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +40 -0
  3. package/android/CMakeLists.txt +30 -0
  4. package/android/build.gradle +119 -0
  5. package/android/src/main/AndroidManifest.xml +1 -0
  6. package/android/src/main/cpp/pictor_overlaystage.cpp +372 -0
  7. package/android/src/main/cpp/pictor_publish.cpp +87 -0
  8. package/android/src/main/cpp/pictor_videosurface.cpp +735 -0
  9. package/android/src/main/java/dev/pictor/video/AacPriming.kt +261 -0
  10. package/android/src/main/java/dev/pictor/video/AudioMixer.kt +1112 -0
  11. package/android/src/main/java/dev/pictor/video/AudioProbe.kt +162 -0
  12. package/android/src/main/java/dev/pictor/video/CadenceShaderProgram.kt +337 -0
  13. package/android/src/main/java/dev/pictor/video/CodecLedger.kt +126 -0
  14. package/android/src/main/java/dev/pictor/video/CompositionSpike.kt +978 -0
  15. package/android/src/main/java/dev/pictor/video/Diagnostics.kt +54 -0
  16. package/android/src/main/java/dev/pictor/video/EncoderGate.kt +476 -0
  17. package/android/src/main/java/dev/pictor/video/ExportGate.kt +1974 -0
  18. package/android/src/main/java/dev/pictor/video/ExportHeartbeat.kt +72 -0
  19. package/android/src/main/java/dev/pictor/video/ExportProgress.kt +103 -0
  20. package/android/src/main/java/dev/pictor/video/FrameClock.kt +591 -0
  21. package/android/src/main/java/dev/pictor/video/FramePending.kt +181 -0
  22. package/android/src/main/java/dev/pictor/video/GateAudio.kt +456 -0
  23. package/android/src/main/java/dev/pictor/video/GateDestination.kt +445 -0
  24. package/android/src/main/java/dev/pictor/video/GateFrames.kt +662 -0
  25. package/android/src/main/java/dev/pictor/video/GateMux.kt +185 -0
  26. package/android/src/main/java/dev/pictor/video/ManagedSources.kt +333 -0
  27. package/android/src/main/java/dev/pictor/video/Media3Player.kt +934 -0
  28. package/android/src/main/java/dev/pictor/video/MediaStoreSave.kt +419 -0
  29. package/android/src/main/java/dev/pictor/video/OverlayResources.kt +169 -0
  30. package/android/src/main/java/dev/pictor/video/PictorVideoModule.kt +1612 -0
  31. package/android/src/main/java/dev/pictor/video/PictorVideoPackage.kt +26 -0
  32. package/android/src/main/java/dev/pictor/video/PlaybackAnchor.kt +161 -0
  33. package/android/src/main/java/dev/pictor/video/PreviewAudioPlayer.kt +291 -0
  34. package/android/src/main/java/dev/pictor/video/PreviewConfig.kt +96 -0
  35. package/android/src/main/java/dev/pictor/video/PreviewMix.kt +396 -0
  36. package/android/src/main/java/dev/pictor/video/PreviewSync.kt +374 -0
  37. package/android/src/main/java/dev/pictor/video/PreviewTrace.kt +478 -0
  38. package/android/src/main/java/dev/pictor/video/PrimingTrim.kt +337 -0
  39. package/android/src/main/java/dev/pictor/video/RepeatingEncoderFactory.kt +97 -0
  40. package/android/src/main/java/dev/pictor/video/SceneProgram.kt +169 -0
  41. package/android/src/main/java/dev/pictor/video/SceneProgramNative.kt +299 -0
  42. package/android/src/main/java/dev/pictor/video/SkiaPassThroughEffect.kt +695 -0
  43. package/android/src/main/java/dev/pictor/video/SurfaceAttachment.kt +110 -0
  44. package/android/src/main/java/dev/pictor/video/ThumbnailDecoder.kt +171 -0
  45. package/android/src/main/java/dev/pictor/video/VideoSurface.kt +394 -0
  46. package/app.plugin.js +43 -0
  47. package/dist/index.d.mts +3622 -0
  48. package/dist/index.d.ts +3622 -0
  49. package/dist/index.js +3737 -0
  50. package/dist/index.js.map +1 -0
  51. package/dist/index.mjs +3625 -0
  52. package/dist/index.mjs.map +1 -0
  53. package/package.json +82 -0
  54. package/plugin/gradleProperties.d.ts +19 -0
  55. package/plugin/gradleProperties.js +64 -0
  56. package/react-native.config.js +13 -0
  57. package/src/NativePictorVideo.ts +574 -0
  58. package/src/audioProbe.ts +72 -0
  59. package/src/compositionSpike.ts +305 -0
  60. package/src/convergence.ts +112 -0
  61. package/src/encoderGate.ts +66 -0
  62. package/src/exportDestination.ts +149 -0
  63. package/src/exportJob.ts +232 -0
  64. package/src/exportProgress.ts +391 -0
  65. package/src/exportProgressPump.ts +190 -0
  66. package/src/exporter.ts +655 -0
  67. package/src/index.ts +276 -0
  68. package/src/managedSources.ts +252 -0
  69. package/src/media3.ts +412 -0
  70. package/src/mediaStore.ts +177 -0
  71. package/src/memoryGuard.ts +118 -0
  72. package/src/mixSources.ts +82 -0
  73. package/src/overlayStaging.ts +216 -0
  74. package/src/playbackLifecycle.ts +137 -0
  75. package/src/player.ts +39 -0
  76. package/src/playlistSession.ts +125 -0
  77. package/src/previewAudioDiagnostics.ts +67 -0
  78. package/src/previewAudioSession.ts +168 -0
  79. package/src/previewCadence.ts +950 -0
  80. package/src/previewMix.ts +212 -0
  81. package/src/previewTrace.ts +348 -0
  82. package/src/spikeExcursions.ts +405 -0
  83. package/src/submissionScoring.ts +680 -0
  84. package/src/surfaceOwner.ts +264 -0
  85. package/src/surfaceRecovery.ts +121 -0
  86. package/src/syncTrace.ts +189 -0
  87. package/src/thumbnailProvider.ts +163 -0
  88. package/src/tracePhases.ts +217 -0
  89. package/src/useCompositionPlayback.ts +327 -0
  90. package/src/useMedia3Playback.ts +492 -0
  91. package/src/usePreviewAudio.ts +305 -0
  92. package/src/useVideoSurface.ts +282 -0
@@ -0,0 +1,163 @@
1
+ import {
2
+ createThumbnailPolicy,
3
+ thumbnailKey,
4
+ type MediaSource,
5
+ type AbortLike,
6
+ type Thumbnail,
7
+ type ThumbnailProvider,
8
+ type ThumbnailRequest,
9
+ } from '@pictor-sdk/video-core'
10
+ import { requireNative } from './player'
11
+
12
+ /**
13
+ * `ThumbnailProvider` over Media3's retriever, with the core policy on top.
14
+ *
15
+ * The division is the point: `@pictor-sdk/video-core` owns deduplication, the
16
+ * bounded cache and visible-range ordering, all of which are pure; this file
17
+ * owns the part that needs a codec. A host that writes its own provider — for a
18
+ * remote thumbnail service, say — reuses the policy and replaces only this.
19
+ *
20
+ * Results arrive as base64 JPEG rather than as a native image handle. A strip
21
+ * thumbnail is a few kilobytes and it is drawn by React Native's own image
22
+ * pipeline, so there is nothing here to leak and no ownership to transfer. The
23
+ * `AHardwareBuffer` lesson applies to full-resolution frames at 30 per second,
24
+ * not to a 64-pixel square that changes when the user scrolls.
25
+ *
26
+ * @experimental
27
+ */
28
+
29
+ /** @experimental */
30
+ export interface AndroidThumbnailOptions {
31
+ /**
32
+ * How many decoded thumbnails to keep.
33
+ *
34
+ * 240 covers several screens of a dense strip at 64 px — enough that
35
+ * scrolling back does not re-decode, bounded enough that a long project
36
+ * cannot grow without limit.
37
+ */
38
+ capacity?: number
39
+ /** How often to check whether a decode has finished, in milliseconds. */
40
+ pollMs?: number
41
+ }
42
+
43
+ const DEFAULT_CAPACITY = 240
44
+ const DEFAULT_POLL_MS = 16
45
+
46
+ /** @experimental */
47
+ export function createAndroidThumbnailProvider(
48
+ options: AndroidThumbnailOptions = {},
49
+ ): ThumbnailProvider {
50
+ const policy = createThumbnailPolicy<string>({
51
+ capacity: options.capacity ?? DEFAULT_CAPACITY,
52
+ })
53
+ const pollMs = options.pollMs ?? DEFAULT_POLL_MS
54
+ /** Tokens in flight, by request key, so a cancel can reach the native side. */
55
+ const tokens = new Map<string, number>()
56
+
57
+ function uriOf(source: MediaSource): string {
58
+ if (typeof source === 'string') return source
59
+ if (source && typeof source === 'object' && 'uri' in source) {
60
+ const uri = (source as { uri?: unknown }).uri
61
+ if (typeof uri === 'string') return uri
62
+ }
63
+ throw new Error(
64
+ `@pictor-sdk/video-android: a thumbnail source is not a URI. Normalise host media to { kind: 'uri', uri } first. Received: ${JSON.stringify(source)}`,
65
+ )
66
+ }
67
+
68
+ return {
69
+ async request(request: ThumbnailRequest, signal?: AbortLike): Promise<Thumbnail | null> {
70
+ const key = thumbnailKey(request)
71
+ const cached = policy.get(request)
72
+ if (cached !== undefined) {
73
+ return { image: cached, sourceTimeUs: request.sourceTimeUs, size: { width: 0, height: 0 } }
74
+ }
75
+ if (signal?.aborted) return null
76
+
77
+ const native = requireNative()
78
+ policy.begin(request)
79
+ const token = native.thumbnailRequest(
80
+ uriOf(request.source),
81
+ request.sourceTimeUs,
82
+ request.widthPx,
83
+ )
84
+ tokens.set(key, token)
85
+
86
+ const stop = () => {
87
+ native.thumbnailCancel(token)
88
+ tokens.delete(key)
89
+ policy.cancel(request)
90
+ }
91
+ signal?.addEventListener('abort', stop, { once: true })
92
+
93
+ try {
94
+ // Poll rather than await a bridged Promise: the native side is a token
95
+ // API precisely so a request can be abandoned, and a Promise that
96
+ // cannot be cancelled would decode frames nobody will see.
97
+ for (;;) {
98
+ if (signal?.aborted) return null
99
+ const data = native.thumbnailTake(token)
100
+ if (data !== null && data !== undefined) {
101
+ tokens.delete(key)
102
+ const evicted = policy.store(request, `data:image/jpeg;base64,${data}`)
103
+ // Base64 strings are garbage-collected; eviction is bookkeeping
104
+ // rather than disposal, and the returned values are ignored on
105
+ // purpose. A provider handing back native images would dispose here.
106
+ void evicted
107
+ return {
108
+ image: policy.get(request)!,
109
+ sourceTimeUs: request.sourceTimeUs,
110
+ size: { width: request.widthPx, height: 0 },
111
+ }
112
+ }
113
+ const error = native.thumbnailError(token)
114
+ if (error !== null && error !== undefined) {
115
+ tokens.delete(key)
116
+ policy.cancel(request)
117
+ return null
118
+ }
119
+ if (!native.thumbnailRunning(token)) {
120
+ // Neither a result nor an error, and not running: the frame could
121
+ // not be produced. A source can legitimately have no frame at a
122
+ // time — past the end of a variable-rate clip, for instance.
123
+ tokens.delete(key)
124
+ policy.cancel(request)
125
+ return null
126
+ }
127
+ await new Promise<void>((resolve) => setTimeout(() => resolve(), pollMs))
128
+ }
129
+ } finally {
130
+ signal?.removeEventListener('abort', stop)
131
+ }
132
+ },
133
+
134
+ releaseSource(source: MediaSource) {
135
+ policy.evictSource(source)
136
+ requireNative().thumbnailReleaseSource(uriOf(source))
137
+ },
138
+
139
+ release() {
140
+ for (const token of tokens.values()) requireNative().thumbnailCancel(token)
141
+ tokens.clear()
142
+ policy.clear()
143
+ requireNative().thumbnailReleaseAll()
144
+ },
145
+ }
146
+ }
147
+
148
+ /** Decode counters, for gates and diagnostics. @experimental */
149
+ export function thumbnailCounters(): {
150
+ decoded: number
151
+ cancelled: number
152
+ failed: number
153
+ running: number
154
+ retrievers: number
155
+ } {
156
+ return requireNative().thumbnailCounters() as {
157
+ decoded: number
158
+ cancelled: number
159
+ failed: number
160
+ running: number
161
+ retrievers: number
162
+ }
163
+ }
@@ -0,0 +1,217 @@
1
+ import type { TraceEvent } from './previewTrace'
2
+
3
+ /**
4
+ * The offset, segmented by the marks a gate put in the run.
5
+ *
6
+ * ## Why a whole-run summary is not enough
7
+ *
8
+ * `summariseOffsets` answers "how bad did it get", and a gate needs "where".
9
+ * Measured on a Pixel 7, a run whose worst excursion was 266 ms spent every
10
+ * moment between its transitions inside ±40 ms: the whole-run worst described
11
+ * three settling transients and nothing else, and a single number cannot tell
12
+ * that from a steady 266 ms error. They are not the same defect and they do not
13
+ * have the same fix.
14
+ *
15
+ * So a run is cut at its marks, and each segment is summarised on its own. The
16
+ * gate verdict is still the run's worst — this localises a failure, it does not
17
+ * soften one.
18
+ *
19
+ * ## Fresh and stale are reported, never traded
20
+ *
21
+ * A stale presented frame is what a person actually sees, so its offset is real
22
+ * and is counted in `worstAbsUs`. But a stale frame's offset grows purely
23
+ * because audio advanced while the picture did not, which is a different fault
24
+ * from two clocks disagreeing — so `freshWorstAbsUs` is reported beside it
25
+ * rather than instead of it. Neither is allowed to stand alone: quoting only
26
+ * the fresh figure is how a run with a nine-second frozen picture passes a
27
+ * synchronisation gate.
28
+ *
29
+ * @experimental
30
+ */
31
+ export interface PhaseSummary {
32
+ /** The mark that opened this segment, or `'start'` before the first one. */
33
+ label: string
34
+ index: number
35
+ /** Samples in the segment, whatever their state. */
36
+ samples: number
37
+ /** Samples with both players playing — the only ones an offset means anything for. */
38
+ playingSamples: number
39
+ /** Of those, the ones whose presented frame had advanced. */
40
+ freshPlayingSamples: number
41
+ stalePlayingSamples: number
42
+ /** The longest unbroken run of stale samples, in samples. */
43
+ longestStaleRun: number
44
+ minUs: number
45
+ maxUs: number
46
+ meanUs: number
47
+ /** Worst absolute offset over every playing sample, stale included. */
48
+ worstAbsUs: number
49
+ /** Worst over fresh samples only. Reported beside `worstAbsUs`, never instead. */
50
+ freshWorstAbsUs: number
51
+ /** Playing samples outside the tolerance. */
52
+ overToleranceSamples: number
53
+ /**
54
+ * How long the segment took to come inside tolerance and stay there.
55
+ *
56
+ * `-1` when it never did, which is the difference between a transient and a
57
+ * failure that happens to start at a transition. Measured from the segment's
58
+ * first playing sample, so it is a settling time rather than a wall clock.
59
+ */
60
+ settledAfterUs: number
61
+ }
62
+
63
+ /** @experimental */
64
+ export interface PhaseReport {
65
+ toleranceUs: number
66
+ phases: PhaseSummary[]
67
+ /** The run's worst, over every phase. This is the gate. */
68
+ worstAbsUs: number
69
+ freshWorstAbsUs: number
70
+ overToleranceSamples: number
71
+ /** Whether every playing sample in the run was inside tolerance. */
72
+ pass: boolean
73
+ }
74
+
75
+ /**
76
+ * The prefix a gate's own marks carry.
77
+ *
78
+ * Every mark is recorded as a `mark` event carrying its label, and the trace is
79
+ * also full of system events — holds, releases, drift corrections, transport
80
+ * transitions. Cutting on "not a sample" would start a new phase at every one
81
+ * of those, and cutting on every mark would let an ad-hoc label reshape a gate's
82
+ * phases. So only marks whose label carries this prefix cut the run.
83
+ *
84
+ * @experimental
85
+ */
86
+ export const PHASE_MARK_PREFIX = 'phase:'
87
+
88
+ function num(v: unknown): number {
89
+ return typeof v === 'number' ? v : Number(v)
90
+ }
91
+
92
+ function empty(label: string, index: number): PhaseSummary {
93
+ return {
94
+ label,
95
+ index,
96
+ samples: 0,
97
+ playingSamples: 0,
98
+ freshPlayingSamples: 0,
99
+ stalePlayingSamples: 0,
100
+ longestStaleRun: 0,
101
+ minUs: 0,
102
+ maxUs: 0,
103
+ meanUs: 0,
104
+ worstAbsUs: 0,
105
+ freshWorstAbsUs: 0,
106
+ overToleranceSamples: 0,
107
+ settledAfterUs: 0,
108
+ }
109
+ }
110
+
111
+ /**
112
+ * Summarise a drained run, cut at its mark events.
113
+ *
114
+ * @experimental
115
+ */
116
+ export function summarisePhases(
117
+ events: readonly TraceEvent[],
118
+ toleranceUs: number,
119
+ ): PhaseReport {
120
+ const phases: PhaseSummary[] = []
121
+ let current = empty('start', 0)
122
+ // Accumulators kept outside the summary so the published shape stays plain
123
+ // data — a gate serialises this straight into a log line.
124
+ let sum = 0
125
+ let staleRun = 0
126
+ let firstPlayingNs = 0
127
+ let lastOverNs = 0
128
+ let sawPlaying = false
129
+ let stillOver = false
130
+
131
+ const close = () => {
132
+ current.meanUs = current.playingSamples > 0 ? Math.round(sum / current.playingSamples) : 0
133
+ /*
134
+ Settling is measured to the *last* excursion, not the first return.
135
+
136
+ A transient that goes out, comes back and goes out again has not settled
137
+ at the first return, and reporting that would turn the worst case into the
138
+ best-looking number available. -1 for a phase still outside tolerance at
139
+ its final sample: that is not a transient at all.
140
+ */
141
+ current.settledAfterUs =
142
+ current.overToleranceSamples === 0
143
+ ? 0
144
+ : stillOver
145
+ ? -1
146
+ : Math.round((lastOverNs - firstPlayingNs) / 1000)
147
+ phases.push(current)
148
+ sum = 0
149
+ staleRun = 0
150
+ firstPlayingNs = 0
151
+ lastOverNs = 0
152
+ sawPlaying = false
153
+ stillOver = false
154
+ }
155
+
156
+ for (const e of events) {
157
+ if (e.type !== 'sample') {
158
+ const label = e.type === 'mark' ? String(e.v.label ?? '') : ''
159
+ if (label.startsWith(PHASE_MARK_PREFIX)) {
160
+ close()
161
+ current = empty(label.slice(PHASE_MARK_PREFIX.length), phases.length)
162
+ }
163
+ continue
164
+ }
165
+ current.samples += 1
166
+ if (e.v.videoPlaying !== true || e.v.audioPlaying !== true) {
167
+ staleRun = 0
168
+ continue
169
+ }
170
+ const offset = num(e.v.offsetUs)
171
+ if (!Number.isFinite(offset)) continue
172
+ current.playingSamples += 1
173
+ if (!sawPlaying) {
174
+ sawPlaying = true
175
+ firstPlayingNs = num(e.atNs)
176
+ }
177
+ sum += offset
178
+ if (current.playingSamples === 1) {
179
+ current.minUs = offset
180
+ current.maxUs = offset
181
+ } else {
182
+ current.minUs = Math.min(current.minUs, offset)
183
+ current.maxUs = Math.max(current.maxUs, offset)
184
+ }
185
+ const abs = Math.abs(offset)
186
+ if (abs > current.worstAbsUs) current.worstAbsUs = abs
187
+ if (abs > toleranceUs) {
188
+ current.overToleranceSamples += 1
189
+ lastOverNs = num(e.atNs)
190
+ stillOver = true
191
+ } else {
192
+ stillOver = false
193
+ }
194
+ if (e.v.pfFresh === true) {
195
+ current.freshPlayingSamples += 1
196
+ if (abs > current.freshWorstAbsUs) current.freshWorstAbsUs = abs
197
+ staleRun = 0
198
+ } else {
199
+ current.stalePlayingSamples += 1
200
+ staleRun += 1
201
+ if (staleRun > current.longestStaleRun) current.longestStaleRun = staleRun
202
+ }
203
+ }
204
+ close()
205
+
206
+ const worstAbsUs = phases.reduce((w, p) => Math.max(w, p.worstAbsUs), 0)
207
+ const freshWorstAbsUs = phases.reduce((w, p) => Math.max(w, p.freshWorstAbsUs), 0)
208
+ const overToleranceSamples = phases.reduce((n, p) => n + p.overToleranceSamples, 0)
209
+ return {
210
+ toleranceUs,
211
+ phases,
212
+ worstAbsUs,
213
+ freshWorstAbsUs,
214
+ overToleranceSamples,
215
+ pass: overToleranceSamples === 0,
216
+ }
217
+ }
@@ -0,0 +1,327 @@
1
+ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
2
+ import {
3
+ playIntent,
4
+ playlistOf,
5
+ projectDuration,
6
+ type PlayIntent,
7
+ type UnsupportedSource,
8
+ type VideoProject,
9
+ } from '@pictor-sdk/video-core'
10
+ import {
11
+ compositionSpikeSpec,
12
+ type CompositionSpikeState,
13
+ spikePause,
14
+ spikePlay,
15
+ spikeRelease,
16
+ spikeSeek,
17
+ spikeStart,
18
+ spikeState,
19
+ } from './compositionSpike'
20
+ import { createPlaybackLifecycle, type PlaybackLifecycle } from './playbackLifecycle'
21
+ import type { Media3Playback } from './useMedia3Playback'
22
+
23
+ /**
24
+ * The preview, on one clock.
25
+ *
26
+ * ## What this replaces
27
+ *
28
+ * Two players kept in step: a video player and a separate player for the
29
+ * prepared audio mix, with a drift check correcting between them. That
30
+ * architecture was measured to its limit and could not hold synchronisation
31
+ * across a clip seam — the correction is a seek, a seek costs a decode, and a
32
+ * decode at the moment of a seam is what the correction was trying to cover.
33
+ *
34
+ * A `CompositionPlayer` has one clock, one playback thread and one audio graph.
35
+ * The prepared mix becomes an audio-only sequence beside the video sequence, so
36
+ * the two cannot drift: there is nothing to drift against.
37
+ *
38
+ * ## What it does not fix
39
+ *
40
+ * The clip seam still costs a picture. Media3 tears down the video decoder at
41
+ * an item boundary and builds another, and for 119 to 168 ms nothing reaches
42
+ * the renderer while the clock runs on — the audio is continuous and the
43
+ * picture holds. That behaviour was watched on a device and accepted for v1;
44
+ * see `docs/research/preview-acceptance.md` for what was accepted and what was
45
+ * explicitly not.
46
+ *
47
+ * ## Why the same contract
48
+ *
49
+ * It returns {@link Media3Playback}, unchanged. The editor, the transport and
50
+ * the gates were all written against it, and swapping the engine underneath is
51
+ * a smaller and more reviewable change than rewriting every caller to a shape
52
+ * that would end up equivalent.
53
+ *
54
+ * @experimental
55
+ */
56
+ export interface UseCompositionPlaybackOptions {
57
+ project: VideoProject
58
+ /** Timeline time the host wants shown. */
59
+ playheadUs: number
60
+ /**
61
+ * The prepared audio program, from `usePreviewAudio`.
62
+ *
63
+ * Null while it is being prepared, after it has failed, and — legitimately —
64
+ * when the document has no audio at all.
65
+ *
66
+ * Those are different situations and the hook has to tell them apart. A
67
+ * document *with* audio waits: building the composition without its sound
68
+ * would play a silent preview and gain the music a second later, which is two
69
+ * timelines to anyone watching. A document *without* audio has nothing to
70
+ * wait for, and refusing to build would leave the picture dead.
71
+ */
72
+ mix: { uri: string; durationUs: number } | null
73
+ /** Turns a document source into something the player can open. */
74
+ resolveSource: (source: { kind: string; uri?: string }) => string | null
75
+ /** @see UseMedia3PlaybackOptions.surfaceGeneration */
76
+ surfaceGeneration?: number
77
+ enabled?: boolean
78
+ }
79
+
80
+ /** How often the composition's state is read, in milliseconds. */
81
+ const POLL_MS = 100
82
+
83
+ /**
84
+ * What a composition knows about itself.
85
+ *
86
+ * Deliberately not `Media3State`. That type is the shape the video player's
87
+ * native state comes back in — forty fields about queue depth, anchors, native
88
+ * heap and window indices — and a composition has none of them. Synthesising
89
+ * one would be inventing forty numbers to satisfy a type, and a reader would
90
+ * have no way to tell the invented ones from the measured ones.
91
+ *
92
+ * @experimental
93
+ */
94
+ export interface CompositionPlaybackState {
95
+ playing: boolean
96
+ ready: boolean
97
+ buffering: boolean
98
+ ended: boolean
99
+ released: boolean
100
+ /** Project time, which for a single composition is the only time there is. */
101
+ positionUs: number
102
+ durationUs: number
103
+ /** The presentation size, or 0 while nothing has been measured. */
104
+ width: number
105
+ height: number
106
+ error: string | null
107
+ }
108
+
109
+ function composedState(snapshot: CompositionSpikeState): CompositionPlaybackState {
110
+ const playbackState = snapshot.playbackState ?? 1
111
+ return {
112
+ playing: snapshot.playing === true,
113
+ ready: playbackState === 3,
114
+ buffering: playbackState === 2,
115
+ ended: playbackState === 4,
116
+ // `released` counts releases; a positive count with nothing attached is the
117
+ // released state this means.
118
+ released: snapshot.attached !== true && (snapshot.released ?? 0) > 0,
119
+ positionUs: snapshot.positionUs ?? 0,
120
+ durationUs: snapshot.durationUs ?? 0,
121
+ /*
122
+ Zero, and honestly so.
123
+
124
+ A composition renders into the presentation surface at the size the host
125
+ gave it; it does not report a decoded size the way the video player did.
126
+ The host already knows the size it asked for, and a fabricated number here
127
+ would be worse than none.
128
+ */
129
+ width: 0,
130
+ height: 0,
131
+ error: snapshot.error ?? null,
132
+ }
133
+ }
134
+
135
+ /** @experimental */
136
+ export type CompositionPlayback = Omit<Media3Playback, 'state'> & {
137
+ state: CompositionPlaybackState | null
138
+ }
139
+
140
+ /** @experimental */
141
+ export function useCompositionPlayback(
142
+ options: UseCompositionPlaybackOptions,
143
+ ): CompositionPlayback {
144
+ const { project, mix, enabled = true } = options
145
+ const surfaceGeneration = options.surfaceGeneration ?? 0
146
+ const resolveSource = useRef(options.resolveSource)
147
+ resolveSource.current = options.resolveSource
148
+
149
+ const [state, setState] = useState<CompositionPlaybackState | null>(null)
150
+ const [playing, setPlaying] = useState(false)
151
+ const playheadRef = useRef(options.playheadUs)
152
+ playheadRef.current = options.playheadUs
153
+
154
+ const playlist = useMemo(() => playlistOf(project), [project])
155
+ const items = playlist.items
156
+ const durationUs = useMemo(() => projectDuration(project), [project])
157
+
158
+ /*
159
+ What the composition is made of, as a string.
160
+
161
+ Every edit makes a new project object and most of them — a selection, a
162
+ playhead move — change nothing the player should be playing. The mix is part
163
+ of it: a rebuilt mix is a different composition even when the video is
164
+ identical, and starting the old one would play the sound the user just
165
+ changed.
166
+ */
167
+ // A document with audio has a mix to wait for; one without does not.
168
+ const needsMix = project.audio.length > 0
169
+ const signature = useMemo(() => {
170
+ if (needsMix && mix === null) return ''
171
+ const parts = items.map(
172
+ (i) => `${JSON.stringify(i.source)}|${i.startUs}|${i.endUs}|${i.speed}|${i.muted}`,
173
+ )
174
+ const sound = mix === null ? 'silent' : `${mix.uri}:${mix.durationUs}`
175
+ return `${parts.join(';')}#${playlist.unsupported.length}@${sound}`
176
+ }, [items, playlist.unsupported.length, mix, needsMix])
177
+
178
+ const [unsupported, setUnsupported] = useState<readonly UnsupportedSource[]>([])
179
+ const lastSeekRef = useRef<Media3Playback['lastSeek']>(null)
180
+
181
+ /*
182
+ This screen's ownership, unchanged from the path it replaces.
183
+
184
+ The player outlives any screen: leaving the editor while playing left the
185
+ sound running, and backgrounding did the same. One lifecycle per hook
186
+ instance, and a screen mounted later starts from nothing.
187
+ */
188
+ const lifecycleRef = useRef<PlaybackLifecycle | null>(null)
189
+ if (lifecycleRef.current === null) {
190
+ lifecycleRef.current = createPlaybackLifecycle({
191
+ pause: () => void spikePause(),
192
+ release: () => void spikeRelease(),
193
+ })
194
+ }
195
+ const lifecycle = lifecycleRef.current
196
+ const [lifecycleEpoch, setLifecycleEpoch] = useState(lifecycle.epoch)
197
+
198
+ const loaded = useRef('')
199
+ useEffect(() => {
200
+ if (!enabled || signature === '') return
201
+ if (surfaceGeneration === 0) return
202
+ if (loaded.current === signature) return
203
+ /*
204
+ An empty URI is a composition with no audio sequence, which the adapter
205
+ builds and plays. It is not a missing mix — that case never reaches here,
206
+ because the signature is empty until one exists.
207
+ */
208
+ const { spec, unsupported: refused } = compositionSpikeSpec(
209
+ project,
210
+ mix ?? { uri: '', durationUs: 0 },
211
+ (s) => resolveSource.current(s),
212
+ )
213
+ setUnsupported(
214
+ refused.map((reason) => ({ clipId: reason.split(':')[0] ?? '', reason }) as UnsupportedSource),
215
+ )
216
+ if (spec.items.length === 0) return
217
+ const started = spikeStart(spec) as { ok?: boolean }
218
+ // A refusal is not a load. Recording it as one is what left an editor
219
+ // waiting for an edit the person had already made.
220
+ if (started.ok === true) loaded.current = signature
221
+ }, [enabled, signature, surfaceGeneration, project, mix])
222
+
223
+ useEffect(() => {
224
+ return () => {
225
+ lifecycle.teardown('unmount')
226
+ }
227
+ }, [lifecycle])
228
+
229
+ useEffect(() => {
230
+ if (!enabled) return
231
+ // Captured now and checked on every tick: an interval is not cleared the
232
+ // instant a screen goes, and a poll landing afterwards must not write into
233
+ // one that is no longer there.
234
+ const epoch = lifecycle.epoch
235
+ const id = setInterval(() => {
236
+ if (!lifecycle.accepts(epoch)) return
237
+ const next = composedState(spikeState())
238
+ /*
239
+ Only a change of meaning re-renders.
240
+
241
+ Replacing the state object every tick re-renders the editor ten times a
242
+ second to move a line, and at ten hertz the line moves visibly in steps.
243
+ A host that wants the position drives it from `positionUs()`.
244
+ */
245
+ setState((previous: CompositionPlaybackState | null) =>
246
+ previous !== null &&
247
+ previous.playing === next.playing &&
248
+ previous.ready === next.ready &&
249
+ previous.buffering === next.buffering &&
250
+ previous.ended === next.ended &&
251
+ previous.released === next.released &&
252
+ previous.error === next.error
253
+ ? previous
254
+ : next,
255
+ )
256
+ setPlaying(next.playing)
257
+ }, POLL_MS)
258
+ return () => clearInterval(id)
259
+ }, [enabled, lifecycle, lifecycleEpoch])
260
+
261
+ const seek = useCallback((timeUs: number) => {
262
+ const at = Math.max(0, Math.round(timeUs))
263
+ lastSeekRef.current = { globalUs: at, windowIndex: 0, localUs: at }
264
+ spikeSeek(at)
265
+ }, [])
266
+
267
+ const positionUs = useCallback((): number | null => {
268
+ const snapshot = spikeState()
269
+ if (!snapshot.attached) return null
270
+ const at = snapshot.positionUs
271
+ return typeof at === 'number' ? at : null
272
+ }, [])
273
+
274
+ const play = useCallback((): PlayIntent => {
275
+ const snapshot = spikeState()
276
+ const intent = playIntent({
277
+ positionUs: typeof snapshot.positionUs === 'number' ? snapshot.positionUs : 0,
278
+ durationUs,
279
+ playbackState: typeof snapshot.playbackState === 'number' ? snapshot.playbackState : 1,
280
+ ended: snapshot.playbackState === 4,
281
+ })
282
+ if (intent.seekToUs !== null) seek(intent.seekToUs)
283
+ if (!intent.play) return intent
284
+ // Refused once the screen has given the player back: a late press from a
285
+ // control that has not unmounted yet must not start a released player.
286
+ if (lifecycle.released) return intent
287
+ spikePlay()
288
+ setPlaying(true)
289
+ return intent
290
+ }, [durationUs, lifecycle, seek])
291
+
292
+ useEffect(() => {
293
+ setLifecycleEpoch(lifecycle.epoch)
294
+ }, [lifecycle, surfaceGeneration])
295
+
296
+ return {
297
+ playing,
298
+ play,
299
+ get lastSeek() {
300
+ return lastSeekRef.current
301
+ },
302
+ positionUs,
303
+ effectiveRate: 1,
304
+ pause: useCallback(() => {
305
+ spikePause()
306
+ setPlaying(false)
307
+ }, []),
308
+ /*
309
+ Rate is the document's, not the transport's.
310
+
311
+ Clip speed is compiled into the composition and into the mix, and a
312
+ preview-wide rate on top of it would be a second speed nobody asked for.
313
+ The contract keeps the method; this path has nothing to do with it.
314
+ */
315
+ setRate: useCallback(() => {}, []),
316
+ seek,
317
+ state,
318
+ items,
319
+ unsupported,
320
+ divergenceUs: 0,
321
+ lifecycle,
322
+ // One player, so there is no half to stall: the recovery this reported was
323
+ // the video player running while the audio player had stopped, or the
324
+ // reverse. @see docs/research/preview-acceptance.md
325
+ stall: null,
326
+ }
327
+ }