@bycrux/editor 0.11.2 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (268) hide show
  1. package/package.json +3 -1
  2. package/src/ControlsInfoModal.tsx +251 -61
  3. package/src/__tests__/ControlsInfoModal.test.tsx +43 -0
  4. package/src/__tests__/adapter.test.ts +59 -1
  5. package/src/__tests__/schema-assignability.test.ts +93 -0
  6. package/src/__tests__/schema.test.ts +15 -0
  7. package/src/__tests__/video-adapter-contract.test.ts +128 -5
  8. package/src/carousel/AddElementMenu.tsx +10 -4
  9. package/src/carousel/CarouselEditor.tsx +46 -8
  10. package/src/carousel/CarouselRenderModal.tsx +15 -10
  11. package/src/carousel/OverlayPicker.tsx +10 -3
  12. package/src/carousel/SlidePropertyPanel.tsx +43 -21
  13. package/src/components/FilmstripScrubber.tsx +277 -0
  14. package/src/components/__tests__/FilmstripScrubber.test.tsx +216 -0
  15. package/src/engine/__tests__/audio-clock.test.ts +655 -0
  16. package/src/engine/__tests__/audio-worklet-source.test.ts +316 -0
  17. package/src/engine/__tests__/batch-planner.test.ts +298 -0
  18. package/src/engine/__tests__/decode-worker-source.test.ts +249 -0
  19. package/src/engine/__tests__/demux-ranged.test.ts +495 -0
  20. package/src/engine/__tests__/demux-truncated.test.ts +136 -0
  21. package/src/engine/__tests__/demux.test.ts +305 -0
  22. package/src/engine/__tests__/eligibility.test.ts +146 -0
  23. package/src/engine/__tests__/engine-recovery.test.ts +263 -0
  24. package/src/engine/__tests__/engine.test.ts +326 -0
  25. package/src/engine/__tests__/frame-server-ranged.test.ts +261 -0
  26. package/src/engine/__tests__/frame-server.test.ts +326 -0
  27. package/src/engine/__tests__/media-loader-ranged.test.ts +289 -0
  28. package/src/engine/__tests__/media-loader.test.ts +100 -0
  29. package/src/engine/__tests__/scheduler-crop.test.ts +353 -0
  30. package/src/engine/__tests__/scheduler.test.ts +1310 -0
  31. package/src/engine/__tests__/scrub-resolve.test.ts +96 -0
  32. package/src/engine/__tests__/scrub-source.test.ts +195 -0
  33. package/src/engine/__tests__/source-host.test.ts +455 -0
  34. package/src/engine/__tests__/time-stretch.test.ts +182 -0
  35. package/src/engine/audio-clock.ts +1179 -0
  36. package/src/engine/audio-worklet-source.ts +160 -0
  37. package/src/engine/batch-planner.ts +308 -0
  38. package/src/engine/debug-hud.tsx +54 -0
  39. package/src/engine/decode-worker-source.ts +142 -0
  40. package/src/engine/demux.ts +900 -0
  41. package/src/engine/eligibility.ts +140 -0
  42. package/src/engine/frame-server.ts +644 -0
  43. package/src/engine/index.ts +950 -0
  44. package/src/engine/media-loader.ts +380 -0
  45. package/src/engine/mp4box.d.ts +214 -0
  46. package/src/engine/scheduler.ts +1324 -0
  47. package/src/engine/scrub-resolve.ts +66 -0
  48. package/src/engine/scrub-source.ts +496 -0
  49. package/src/engine/time-stretch.ts +224 -0
  50. package/src/index.ts +83 -1
  51. package/src/preview/OverlayPreview.tsx +2 -24
  52. package/src/schema.ts +146 -3
  53. package/src/state/__tests__/use-project-sync.test.tsx +56 -0
  54. package/src/state/use-project-sync.ts +17 -0
  55. package/src/test-setup.ts +32 -0
  56. package/src/text/FontPicker.tsx +85 -51
  57. package/src/text/TextFormattingToolbar.tsx +6 -1
  58. package/src/text/__tests__/FontPicker.options.test.ts +43 -0
  59. package/src/text/__tests__/TextFormattingToolbar.test.tsx +4 -4
  60. package/src/theme.ts +108 -3
  61. package/src/types.ts +601 -22
  62. package/src/ui/Loader.tsx +64 -0
  63. package/src/ui/NumberField.tsx +254 -0
  64. package/src/ui/Slider.tsx +128 -0
  65. package/src/ui/Tooltip.tsx +118 -0
  66. package/src/ui/__tests__/NumberField.test.tsx +298 -0
  67. package/src/ui/__tests__/Slider.test.tsx +150 -0
  68. package/src/ui/__tests__/Tooltip.test.tsx +105 -0
  69. package/src/ui/__tests__/usePersistentState.test.tsx +112 -0
  70. package/src/ui/badge.tsx +12 -4
  71. package/src/ui/index.ts +6 -0
  72. package/src/ui/input.tsx +1 -1
  73. package/src/ui/select.tsx +1 -1
  74. package/src/ui/switch.tsx +12 -2
  75. package/src/ui/textarea.tsx +1 -1
  76. package/src/ui/usePersistentState.ts +63 -0
  77. package/src/video/AudioPolishModal.tsx +983 -0
  78. package/src/video/CaptionListPanel.test.tsx +944 -0
  79. package/src/video/CaptionListPanel.tsx +1106 -0
  80. package/src/video/CaptionRegenModal.tsx +37 -9
  81. package/src/video/CaptionSpecimen.tsx +160 -0
  82. package/src/video/CaptionStyleGallery.tsx +466 -0
  83. package/src/video/CommandPalette.tsx +165 -0
  84. package/src/video/ImageToneMenu.tsx +137 -0
  85. package/src/video/OverlayInspector.tsx +977 -0
  86. package/src/video/RenderModal.tsx +1035 -62
  87. package/src/video/VersionCompare.tsx +258 -0
  88. package/src/video/VersionPanel.tsx +117 -42
  89. package/src/video/VideoEditor.tsx +2229 -243
  90. package/src/video/__tests__/AudioPolishModal.test.tsx +825 -0
  91. package/src/video/__tests__/CaptionListPanel.font.test.tsx +397 -0
  92. package/src/video/__tests__/CaptionListPanel.generate.test.tsx +153 -0
  93. package/src/video/__tests__/CaptionRegenModal.test.tsx +29 -0
  94. package/src/video/__tests__/CaptionSpecimen.test.tsx +213 -0
  95. package/src/video/__tests__/CaptionStyleGallery.test.tsx +362 -0
  96. package/src/video/__tests__/CommandPalette.test.tsx +119 -0
  97. package/src/video/__tests__/OverlayInspector.test.tsx +1367 -0
  98. package/src/video/__tests__/RenderModal.exportControls.test.tsx +319 -0
  99. package/src/video/__tests__/RenderModal.options.test.tsx +562 -0
  100. package/src/video/__tests__/RenderModal.progress.test.ts +65 -0
  101. package/src/video/__tests__/VersionCompare.test.tsx +182 -0
  102. package/src/video/__tests__/VersionPanel.test.tsx +279 -0
  103. package/src/video/__tests__/VideoEditor.audioPolish.test.tsx +241 -0
  104. package/src/video/__tests__/VideoEditor.captionDelete.test.tsx +147 -0
  105. package/src/video/__tests__/VideoEditor.captionGesture.test.tsx +170 -0
  106. package/src/video/__tests__/VideoEditor.captionSeam.test.tsx +134 -0
  107. package/src/video/__tests__/VideoEditor.clipKeyframes.test.tsx +59 -0
  108. package/src/video/__tests__/VideoEditor.context.test.tsx +44 -0
  109. package/src/video/__tests__/VideoEditor.editFocus.test.tsx +55 -0
  110. package/src/video/__tests__/VideoEditor.keymap.test.tsx +725 -0
  111. package/src/video/__tests__/VideoEditor.layout.test.tsx +338 -0
  112. package/src/video/__tests__/VideoEditor.propertiesPanel.test.tsx +765 -0
  113. package/src/video/__tests__/VideoEditor.rippleDeleteCaptions.test.tsx +159 -0
  114. package/src/video/__tests__/VideoEditor.sourcePreview.test.tsx +122 -0
  115. package/src/video/__tests__/VideoEditor.test.tsx +403 -50
  116. package/src/video/__tests__/audioMagnet.test.ts +158 -0
  117. package/src/video/__tests__/audioPolish.test.ts +1202 -0
  118. package/src/video/__tests__/captionActiveWord.test.ts +107 -0
  119. package/src/video/__tests__/captionLanes.test.ts +262 -0
  120. package/src/video/__tests__/captionPositioning.test.tsx +8 -3
  121. package/src/video/__tests__/captionWordFloor.test.ts +228 -0
  122. package/src/video/__tests__/clipboard-ops.test.ts +430 -0
  123. package/src/video/__tests__/cuts.insert.test.ts +244 -0
  124. package/src/video/__tests__/cuts.test.ts +1213 -34
  125. package/src/video/__tests__/export-limits.test.ts +195 -0
  126. package/src/video/__tests__/hover-scrub.test.ts +163 -0
  127. package/src/video/__tests__/keyframeOps.canKeyframeProp.test.ts +61 -0
  128. package/src/video/__tests__/keyframeOps.test.ts +643 -0
  129. package/src/video/__tests__/keymap.test.tsx +171 -0
  130. package/src/video/__tests__/render-progress.test.tsx +20 -4
  131. package/src/video/__tests__/shuttle.test.ts +210 -0
  132. package/src/video/__tests__/source-preview.test.ts +60 -0
  133. package/src/video/__tests__/timecode.test.ts +77 -0
  134. package/src/video/__tests__/use-report-context.test.tsx +101 -0
  135. package/src/video/audioMagnet.ts +72 -0
  136. package/src/video/audioPolish.ts +774 -0
  137. package/src/video/captionActiveWord.ts +74 -0
  138. package/src/video/captionLanes.ts +202 -0
  139. package/src/video/captionRepair.ts +9 -5
  140. package/src/video/captionStyleDefaults.ts +100 -0
  141. package/src/video/captionWordFloor.ts +83 -0
  142. package/src/video/clipboard-ops.ts +377 -0
  143. package/src/video/cuts.ts +713 -37
  144. package/src/video/export-limits.ts +102 -0
  145. package/src/video/hover-scrub.ts +102 -0
  146. package/src/video/imageTone.ts +58 -0
  147. package/src/video/imageToneExamples.ts +10 -0
  148. package/src/video/keyframeOps.ts +384 -0
  149. package/src/video/keymap.ts +146 -0
  150. package/src/video/panels/ClipPropertiesPanel.tsx +706 -0
  151. package/src/video/panels/LeftPanelTabs.tsx +187 -0
  152. package/src/video/panels/OverlayContentPanel.tsx +351 -0
  153. package/src/video/panels/TabNav.tsx +60 -0
  154. package/src/video/panels/__tests__/ClipPropertiesPanel.test.tsx +645 -0
  155. package/src/video/panels/__tests__/LeftPanelTabs.test.tsx +189 -0
  156. package/src/video/panels/__tests__/OverlayContentPanel.test.tsx +222 -0
  157. package/src/video/panels/__tests__/TabNav.test.tsx +71 -0
  158. package/src/video/preview/CaptionPreview.tsx +57 -69
  159. package/src/video/preview/EngineSurface.tsx +103 -0
  160. package/src/video/preview/OverlayItemsLayer.tsx +315 -103
  161. package/src/video/preview/PreviewPlayer.tsx +337 -56
  162. package/src/video/preview/SocialPreviewMenu.tsx +214 -0
  163. package/src/video/preview/SocialSafeZoneOverlay.tsx +478 -0
  164. package/src/video/preview/__tests__/CaptionPreview.fonts.test.tsx +97 -0
  165. package/src/video/preview/__tests__/EngineSurface.test.tsx +114 -0
  166. package/src/video/preview/__tests__/OverlayItemsLayer.edit.test.tsx +4 -2
  167. package/src/video/preview/__tests__/OverlayItemsLayer.keyframes.test.tsx +363 -0
  168. package/src/video/preview/__tests__/OverlayItemsLayer.selection.test.tsx +329 -0
  169. package/src/video/preview/__tests__/PreviewPlayer.engine.test.tsx +139 -0
  170. package/src/video/preview/__tests__/SocialPreviewMenu.test.tsx +121 -0
  171. package/src/video/preview/__tests__/SocialSafeZoneOverlay.test.tsx +165 -0
  172. package/src/video/preview/__tests__/captionDragState.test.ts +120 -1
  173. package/src/video/preview/__tests__/latencyCompensation.test.tsx +451 -0
  174. package/src/video/preview/__tests__/proxySupport.test.ts +75 -0
  175. package/src/video/preview/__tests__/transformStyle.test.ts +29 -1
  176. package/src/video/preview/__tests__/useDragOverlay.perAxis.test.ts +197 -0
  177. package/src/video/preview/__tests__/useEnginePlayback.test.tsx +530 -0
  178. package/src/video/preview/__tests__/useVideoPlayback.corpus.test.ts +313 -0
  179. package/src/video/preview/__tests__/useVideoPlayback.test.ts +38 -5
  180. package/src/video/preview/__tests__/useVideoPlayback.trackAudio.test.ts +278 -0
  181. package/src/video/preview/audio-context.ts +111 -0
  182. package/src/video/preview/captionDragState.ts +91 -1
  183. package/src/video/preview/proxySupport.ts +86 -0
  184. package/src/video/preview/transformStyle.ts +22 -12
  185. package/src/video/preview/useDragOverlay.ts +92 -18
  186. package/src/video/preview/useEnginePlayback.ts +625 -0
  187. package/src/video/preview/useVideoPlayback.ts +211 -167
  188. package/src/video/sdrCurves.ts +56 -0
  189. package/src/video/shuttle.ts +159 -0
  190. package/src/video/source-preview.ts +66 -0
  191. package/src/video/timecode.ts +59 -0
  192. package/src/video/timeline/EditableSegment.tsx +1 -1
  193. package/src/video/timeline/Scrubber.tsx +46 -174
  194. package/src/video/timeline/SpeedControl.tsx +95 -0
  195. package/src/video/timeline/Timeline.tsx +1061 -328
  196. package/src/video/timeline/TimelineContext.ts +17 -10
  197. package/src/video/timeline/TrackGutter.tsx +560 -0
  198. package/src/video/timeline/TrackSettingsPopover.tsx +228 -0
  199. package/src/video/timeline/VolumeControl.tsx +113 -0
  200. package/src/video/timeline/__tests__/Timeline.backgroundClick.test.tsx +110 -0
  201. package/src/video/timeline/__tests__/Timeline.crossfade.test.tsx +104 -0
  202. package/src/video/timeline/__tests__/Timeline.fadeCurveMenu.test.tsx +174 -0
  203. package/src/video/timeline/__tests__/Timeline.keyframeDelete.test.tsx +273 -0
  204. package/src/video/timeline/__tests__/Timeline.keyframeFollow.test.tsx +215 -0
  205. package/src/video/timeline/__tests__/Timeline.keyframeMenu.test.tsx +253 -0
  206. package/src/video/timeline/__tests__/Timeline.keymap.test.tsx +372 -0
  207. package/src/video/timeline/__tests__/Timeline.subcutRegen.test.tsx +351 -0
  208. package/src/video/timeline/__tests__/TrackGutter.test.tsx +372 -0
  209. package/src/video/timeline/__tests__/_canvasSelect.test.tsx +273 -0
  210. package/src/video/timeline/__tests__/_canvasSelect.ts +414 -0
  211. package/src/video/timeline/__tests__/dragdrop-math.test.ts +135 -0
  212. package/src/video/timeline/__tests__/effectiveItemAudio.test.ts +50 -0
  213. package/src/video/timeline/__tests__/enabledTrackItems.test.ts +164 -0
  214. package/src/video/timeline/__tests__/moveItemAcrossTracks.test.ts +363 -0
  215. package/src/video/timeline/__tests__/multiSelectOps.test.ts +447 -0
  216. package/src/video/timeline/__tests__/placement.test.ts +278 -0
  217. package/src/video/timeline/__tests__/resizeWindowedItem.test.ts +140 -0
  218. package/src/video/timeline/__tests__/timeline-model.test.ts +576 -0
  219. package/src/video/timeline/__tests__/visualItemLabel.test.ts +69 -0
  220. package/src/video/timeline/canvas/TimelineCanvas.tsx +1447 -0
  221. package/src/video/timeline/canvas/__tests__/TimelineCanvas.drop.test.tsx +439 -0
  222. package/src/video/timeline/canvas/__tests__/TimelineCanvas.edgeScroll.test.tsx +346 -0
  223. package/src/video/timeline/canvas/__tests__/TimelineCanvas.panefill.test.tsx +122 -0
  224. package/src/video/timeline/canvas/__tests__/TimelineCanvas.pendingDrops.test.tsx +316 -0
  225. package/src/video/timeline/canvas/__tests__/TimelineCanvas.pointer.test.tsx +606 -0
  226. package/src/video/timeline/canvas/__tests__/TimelineCanvas.test.tsx +407 -0
  227. package/src/video/timeline/canvas/__tests__/clip-bands.test.ts +77 -0
  228. package/src/video/timeline/canvas/__tests__/draw.test.ts +2198 -0
  229. package/src/video/timeline/canvas/__tests__/fade-curve.test.ts +187 -0
  230. package/src/video/timeline/canvas/__tests__/filmstrips.test.ts +561 -0
  231. package/src/video/timeline/canvas/__tests__/hit-test.test.ts +818 -0
  232. package/src/video/timeline/canvas/__tests__/pending-drop.test.ts +210 -0
  233. package/src/video/timeline/canvas/__tests__/pointer-machine.test.ts +3358 -0
  234. package/src/video/timeline/canvas/__tests__/snap.test.ts +257 -0
  235. package/src/video/timeline/canvas/__tests__/viewport.test.ts +399 -0
  236. package/src/video/timeline/canvas/__tests__/waveforms.test.ts +946 -0
  237. package/src/video/timeline/canvas/clip-bands.ts +56 -0
  238. package/src/video/timeline/canvas/draw.ts +2187 -0
  239. package/src/video/timeline/canvas/fade-curve.ts +111 -0
  240. package/src/video/timeline/canvas/filmstrips.ts +418 -0
  241. package/src/video/timeline/canvas/hit-test.ts +501 -0
  242. package/src/video/timeline/canvas/keyframe-strip.ts +73 -0
  243. package/src/video/timeline/canvas/pointer-machine.ts +1828 -0
  244. package/src/video/timeline/canvas/snap.ts +232 -0
  245. package/src/video/timeline/canvas/viewport.ts +457 -0
  246. package/src/video/timeline/canvas/waveforms.ts +664 -0
  247. package/src/video/timeline/makeCaptionEdit.ts +5 -1
  248. package/src/video/timeline/multiSelectOps.ts +214 -55
  249. package/src/video/timeline/placement.ts +282 -0
  250. package/src/video/timeline/timeline-model.ts +919 -0
  251. package/src/video/timeline/useItemDragDrop.ts +151 -178
  252. package/src/video/timeline/useTimelineZoom.ts +25 -60
  253. package/src/video/timeline/utils.ts +0 -12
  254. package/src/video/use-report-context.ts +75 -0
  255. package/src/video/preview/OverlayPropsModal.tsx +0 -292
  256. package/src/video/preview/__tests__/OverlayPropsModal.test.tsx +0 -32
  257. package/src/video/timeline/AudioTrackRow.tsx +0 -404
  258. package/src/video/timeline/AudioWaveformLayer.tsx +0 -117
  259. package/src/video/timeline/CaptionTrackRow.tsx +0 -235
  260. package/src/video/timeline/PlayheadLine.tsx +0 -18
  261. package/src/video/timeline/TranscriptModal.tsx +0 -70
  262. package/src/video/timeline/TranscriptPanel.tsx +0 -273
  263. package/src/video/timeline/VisualTrackRow.tsx +0 -300
  264. package/src/video/timeline/__tests__/CaptionTrackRow.test.tsx +0 -241
  265. package/src/video/timeline/__tests__/PlayheadLine.test.tsx +0 -60
  266. package/src/video/timeline/__tests__/TranscriptModal.test.tsx +0 -41
  267. package/src/video/timeline/__tests__/TranscriptPanel.test.tsx +0 -184
  268. package/src/video/timeline/__tests__/useItemDragDrop.test.ts +0 -72
@@ -0,0 +1,644 @@
1
+ /**
2
+ * SP4 T3 — the frame server: main-side decode-ahead orchestration.
3
+ *
4
+ * Owns the decode worker's whole lifecycle (spawn from the inlined source
5
+ * string via a Blob URL, the postMessage protocol below, and — critically —
6
+ * `VideoFrame` ownership on every path) and drives it with the pure decisions
7
+ * `batch-planner.ts` makes. One frame server serves exactly ONE demuxed source:
8
+ * a clip-boundary swap terminates it and spawns another, which is the spike's
9
+ * "never reconfigure a live decoder" rule (`player.ts`'s `load`).
10
+ *
11
+ * ─────────────────────────────────────────────────────────────────────────
12
+ * WHERE THE DEMUX HAPPENS — and why
13
+ * ─────────────────────────────────────────────────────────────────────────
14
+ * **Main side**, exactly as the spike did it (`player.ts` calls `demux(url)`
15
+ * and posts sample slices to the worker). It is also the only option here: the
16
+ * worker ships as a source *string*, so it cannot `import` `demux.ts` or
17
+ * `mp4box` — a worker-side demux would mean either bundling mp4box into the
18
+ * string (which would blow up the "small enough for line review" constraint
19
+ * decision 6 rests on) or a second, separately-loaded worker. T2 made
20
+ * `demuxBytes` synchronous and worker-callable so that option stays open for a
21
+ * later ranged-loading design; it is deliberately not exercised in v1.
22
+ *
23
+ * Consequence: sample bytes cross the boundary once per batch, by **structured
24
+ * clone, never transfer**. mp4box allocates a dedicated `Uint8Array` per
25
+ * sample (`ISOFile.getSample`: `sample.data = new Uint8Array(sample.size)`,
26
+ * then a memcpy into it), so a clone copies exactly that sample's bytes — not
27
+ * the whole file — while transferring would detach the buffer out of the
28
+ * demuxed sample table and corrupt every later request that overlaps the same
29
+ * GOP. Frames come back the other way *with* a transfer list, because a
30
+ * `VideoFrame` must not be cloned.
31
+ *
32
+ * ─────────────────────────────────────────────────────────────────────────
33
+ * FRAME OWNERSHIP (the leak contract)
34
+ * ─────────────────────────────────────────────────────────────────────────
35
+ * Every `VideoFrame` that reaches main is closed exactly once, by this module,
36
+ * with one exception: the frame handed out by `seek()` or `nextFrameFor()`
37
+ * becomes the CALLER's to close (T5 paints it via `drawImage` and closes it —
38
+ * holding a reference past that point is what exhausts the decoder's surface
39
+ * pool). Everything else — stale frames from a superseded request, frames
40
+ * skipped because a newer one is already due, whatever is still buffered at
41
+ * `stopStream()`/`dispose()` — is closed here.
42
+ */
43
+ import {
44
+ DEFAULT_DECODE_AHEAD_FRAMES,
45
+ MAX_IN_FLIGHT_BATCHES,
46
+ PRE_ROLL_EPSILON_US,
47
+ isSuperseded,
48
+ onFrameBuffered,
49
+ onFramesConsumed,
50
+ planNextBatches,
51
+ planSeek,
52
+ planStreamStart,
53
+ reconcileOnBatchDone,
54
+ type PipelineConfig,
55
+ type PipelineState,
56
+ } from './batch-planner'
57
+ import { decodeWorkerSource } from './decode-worker-source'
58
+ import type { ChunkSource, DemuxedSource, SampleRef } from './demux'
59
+
60
+ /**
61
+ * WebCodecs' `hardwareAcceleration` hint. The spike exposed only the two
62
+ * extremes because Task 5 existed to compare them; production defaults to
63
+ * `no-preference` (let the browser decide) — SP1 §6 measured the AV1 proxy
64
+ * holding 30fps even under forced software decode, so there is nothing to win
65
+ * by pinning it.
66
+ */
67
+ export type HardwarePref = 'prefer-hardware' | 'prefer-software' | 'no-preference'
68
+
69
+ /**
70
+ * One sample as the worker sees it. Deliberately narrower than T2's
71
+ * `SampleRef`: `dtsUs` is dropped because `EncodedVideoChunk.timestamp` is a
72
+ * *presentation* time (decode order is carried by the ORDER of the array, not
73
+ * by a field), and shipping it would just be bytes on the wire.
74
+ */
75
+ export interface WorkerSample {
76
+ tsUs: number
77
+ durUs: number
78
+ isKey: boolean
79
+ data: Uint8Array
80
+ }
81
+
82
+ /** main → worker. */
83
+ export type DecodeCmd =
84
+ | {
85
+ t: 'init'
86
+ codec: string
87
+ description?: Uint8Array
88
+ hardwareAcceleration: HardwarePref
89
+ /** SP1 §7.2's epsilon, owned by `batch-planner.ts` and injected so the worker string holds no planning constants. */
90
+ preRollEpsilonUs: number
91
+ }
92
+ | { t: 'decodeRange'; samples: WorkerSample[]; targetTsUs: number; reqId: number }
93
+
94
+ /** worker → main. `frame` carries a transferred `VideoFrame`. */
95
+ export type DecodeEvt =
96
+ | { t: 'frame'; frame: VideoFrame; reqId: number }
97
+ | { t: 'done'; reqId: number; decoded: number; dropped: number }
98
+ | { t: 'error'; message: string; reqId?: number }
99
+
100
+ /**
101
+ * The slice of `Worker` this module uses. Injectable (`spawnWorker`) so tests
102
+ * drive a fake port — jsdom has neither `Worker` nor `URL.createObjectURL`,
103
+ * and there is no WebCodecs to decode with even if it did.
104
+ */
105
+ export interface DecodeWorkerPort {
106
+ postMessage(msg: DecodeCmd): void
107
+ terminate(): void
108
+ onmessage: ((ev: { data: DecodeEvt }) => void) | null
109
+ }
110
+
111
+ export interface FrameServerOptions {
112
+ /** The single source this server decodes. There is no other input and no fallback path — proxy-only playback (SP1 requirement 4) is a structural property, not a runtime check. */
113
+ source: DemuxedSource
114
+ hardwareAcceleration?: HardwarePref
115
+ /** Decode-ahead budget in frames (N). Also settable at runtime via the returned server. */
116
+ decodeAheadFrames?: number
117
+ /** Decoder/worker errors. T5 routes these to the same per-clip "Preparing preview…" state as a missing proxy — the engine never reverts the whole project to the legacy player. */
118
+ onError?: (message: string) => void
119
+ spawnWorker?: () => DecodeWorkerPort
120
+ }
121
+
122
+ export interface FrameServerStats {
123
+ buffered: number
124
+ inFlightFrames: number
125
+ inFlightBatches: number
126
+ /** Frames delivered to main since construction (buffered or handed to a seek). */
127
+ received: number
128
+ /** Frames closed here rather than handed out: superseded, or skipped as already-late. */
129
+ dropped: number
130
+ /** Every sample has been requested — nothing left to ask the worker for. */
131
+ atEndOfSource: boolean
132
+ /** At end of source with nothing buffered and nothing in flight: the stream is finished. */
133
+ drained: boolean
134
+ lastError: string | null
135
+ }
136
+
137
+ export interface FrameServer {
138
+ /** The `src` this server decodes — the proxy path the caller demuxed. */
139
+ readonly src: string
140
+ readonly video: ChunkSource
141
+ /** Runtime-settable N (SP1 §8's sweep knob; T7's HUD reads it back through `stats()`). */
142
+ decodeAheadFrames: number
143
+ /**
144
+ * One-shot frame-accurate seek. `targetTsUs` is RAW CONTAINER time — the
145
+ * caller maps project time through `video.firstPresentationTsUs`.
146
+ *
147
+ * `reqId` comes back synchronously so a caller can key latency
148
+ * instrumentation to this exact seek before any round trip. `frame` always
149
+ * settles (the worker's total `done` contract), with `null` when the seek
150
+ * was superseded, produced nothing, or the source is empty. **The resolved
151
+ * frame belongs to the caller** — close it after painting.
152
+ *
153
+ * Stops any streaming session first: the worker serves one intent at a time,
154
+ * and buffered playback frames are worthless once the playhead jumps.
155
+ */
156
+ seek(targetTsUs: number): { reqId: number; frame: Promise<VideoFrame | null> }
157
+ /**
158
+ * Open a streaming decode-ahead session at `targetTsUs` (raw container
159
+ * time). Returns the session's `reqId`. Frames land in the internal buffer;
160
+ * the scheduler pulls them with `nextFrameFor`.
161
+ */
162
+ startStream(targetTsUs: number): number
163
+ /** Close the session and every frame still buffered. Safe to call when nothing is streaming. */
164
+ stopStream(): void
165
+ /**
166
+ * The frame due at `clockUs` (raw container time): the LATEST buffered frame
167
+ * whose timestamp is at or before the clock, or `null` when nothing is due
168
+ * yet. Any buffered frame older than the one returned became due and was
169
+ * overtaken before this call — those are closed here and reported as
170
+ * `dropped`, never leaked. Tops the pipeline up on every call, so the
171
+ * scheduler's tick needs no separate pump. **The returned frame belongs to
172
+ * the caller.**
173
+ */
174
+ nextFrameFor(clockUs: number): { frame: VideoFrame | null; dropped: number }
175
+ stats(): FrameServerStats
176
+ /** Terminate the worker, settle anything pending, close every frame still held. */
177
+ dispose(): void
178
+ }
179
+
180
+ /** Default spawn: inline source → Blob URL → classic `Worker`, per plan decision 6. */
181
+ function spawnBlobWorker(): DecodeWorkerPort {
182
+ const url = URL.createObjectURL(new Blob([decodeWorkerSource], { type: 'text/javascript' }))
183
+ const worker = new Worker(url)
184
+ // Safe immediately: the worker has already been handed the URL, and holding
185
+ // it would leak the blob for the page's lifetime.
186
+ URL.revokeObjectURL(url)
187
+ // The port is structurally narrower than `Worker` (a typed `onmessage`, no
188
+ // transfer list on `postMessage` — this module never transfers TO the
189
+ // worker, see the module doc). One honest cast rather than a wrapper object.
190
+ return worker as unknown as DecodeWorkerPort
191
+ }
192
+
193
+ /** Insertion point that keeps the buffer ascending by timestamp. */
194
+ function insertByTimestamp(buffer: VideoFrame[], frame: VideoFrame): void {
195
+ let i = buffer.length
196
+ while (i > 0 && buffer[i - 1].timestamp > frame.timestamp) i--
197
+ buffer.splice(i, 0, frame)
198
+ }
199
+
200
+ interface PendingSeek {
201
+ targetTsUs: number
202
+ /** Best candidate so far: the latest frame at-or-before the target. */
203
+ best: VideoFrame | null
204
+ superseded: boolean
205
+ resolve: (frame: VideoFrame | null) => void
206
+ }
207
+
208
+ class FrameServerImpl implements FrameServer {
209
+ readonly src: string
210
+ readonly video: ChunkSource
211
+ decodeAheadFrames: number
212
+
213
+ private readonly worker: DecodeWorkerPort
214
+ private readonly onError?: (message: string) => void
215
+
216
+ private nextReqId = 0
217
+ private latestReqId = -1
218
+
219
+ private readonly pendingSeeks = new Map<number, PendingSeek>()
220
+
221
+ private streaming = false
222
+ private streamReqId = -1
223
+ private pipeline: PipelineState = {
224
+ nextSampleIdx: 0,
225
+ buffered: 0,
226
+ inFlightFrames: 0,
227
+ inFlightBatches: 0,
228
+ firstBatchTargetTsUs: 0,
229
+ }
230
+ /**
231
+ * FIFO of per-batch expected frame counts. Pre-roll frames are dropped in the
232
+ * worker and never arrive, so `inFlightFrames` is reconciled against this on
233
+ * every `done` — without it the count drifts upward and permanently eats
234
+ * decode-ahead capacity (SP1 §6.1).
235
+ */
236
+ private batchExpected: number[] = []
237
+ /** Decoded, undelivered frames, ascending by timestamp. */
238
+ private frameBuffer: VideoFrame[] = []
239
+ /**
240
+ * Serializes posts behind a ranged source's byte fetches. Idle (depth 0)
241
+ * whenever nothing is waiting on bytes, which is when `postBatch` skips it
242
+ * entirely and posts synchronously — see `postBatch`.
243
+ */
244
+ private postQueue: Promise<void> = Promise.resolve()
245
+ private postDepth = 0
246
+ /**
247
+ * Which post chain is live. Bumped by `claimReqId`, i.e. every time
248
+ * everything older is superseded: a scrub landing mid-fetch must not queue
249
+ * behind the fetch it just made pointless, or the new frame would arrive no
250
+ * sooner than the abandoned one would have. Steps from a retired generation
251
+ * still run — they just settle their seek and return without posting.
252
+ */
253
+ private postGen = 0
254
+
255
+ private received = 0
256
+ private dropped = 0
257
+ private lastError: string | null = null
258
+ private disposed = false
259
+
260
+ constructor(options: FrameServerOptions) {
261
+ this.src = options.source.src
262
+ this.video = options.source.video
263
+ this.decodeAheadFrames = options.decodeAheadFrames ?? DEFAULT_DECODE_AHEAD_FRAMES
264
+ this.onError = options.onError
265
+
266
+ this.worker = (options.spawnWorker ?? spawnBlobWorker)()
267
+ this.worker.onmessage = (ev) => this.onMessage(ev.data)
268
+ this.worker.postMessage({
269
+ t: 'init',
270
+ // Verbatim from the container — normalization is not this layer's job,
271
+ // and for video there is nothing to normalize (T2's note about `'Opus'`
272
+ // vs `'opus'` is an AUDIO-side hazard, T4's).
273
+ codec: this.video.codec,
274
+ description: this.video.description,
275
+ hardwareAcceleration: options.hardwareAcceleration ?? 'no-preference',
276
+ preRollEpsilonUs: PRE_ROLL_EPSILON_US,
277
+ })
278
+ }
279
+
280
+ private get config(): PipelineConfig {
281
+ return {
282
+ decodeAheadFrames: this.decodeAheadFrames,
283
+ maxInFlightBatches: MAX_IN_FLIGHT_BATCHES,
284
+ }
285
+ }
286
+
287
+ /**
288
+ * Claim a new request id, superseding everything older. Any pending seek
289
+ * left behind is settled here (with its held frame closed) rather than
290
+ * waiting for its `done` — the caller asked for something else, so its
291
+ * answer can only be `null`.
292
+ */
293
+ private claimReqId(): number {
294
+ const reqId = this.nextReqId++
295
+ this.latestReqId = reqId
296
+ this.retirePostQueue()
297
+ for (const [id, pending] of this.pendingSeeks) {
298
+ if (isSuperseded(id, this.latestReqId) && !pending.superseded) {
299
+ pending.superseded = true
300
+ if (pending.best) {
301
+ pending.best.close()
302
+ pending.best = null
303
+ this.dropped++
304
+ }
305
+ }
306
+ }
307
+ return reqId
308
+ }
309
+
310
+ seek(targetTsUs: number): { reqId: number; frame: Promise<VideoFrame | null> } {
311
+ this.stopStream()
312
+ const reqId = this.claimReqId()
313
+ const batch = planSeek(this.video, targetTsUs)
314
+
315
+ let resolveFrame: (frame: VideoFrame | null) => void = () => {}
316
+ const frame = new Promise<VideoFrame | null>((resolve) => {
317
+ resolveFrame = resolve
318
+ })
319
+
320
+ if (!batch || this.disposed) {
321
+ resolveFrame(null)
322
+ return { reqId, frame }
323
+ }
324
+
325
+ this.pendingSeeks.set(reqId, {
326
+ targetTsUs: batch.targetTsUs,
327
+ best: null,
328
+ superseded: false,
329
+ resolve: resolveFrame,
330
+ })
331
+ this.postBatch(batch.startIdx, batch.endIdx, batch.targetTsUs, reqId)
332
+ return { reqId, frame }
333
+ }
334
+
335
+ startStream(targetTsUs: number): number {
336
+ this.stopStream()
337
+ if (this.disposed) return -1
338
+ this.streamReqId = this.claimReqId()
339
+ this.streaming = true
340
+ this.pipeline = planStreamStart(this.video, targetTsUs)
341
+ this.batchExpected = []
342
+ this.pump()
343
+ return this.streamReqId
344
+ }
345
+
346
+ stopStream(): void {
347
+ for (const frame of this.frameBuffer) frame.close()
348
+ this.dropped += this.frameBuffer.length
349
+ this.frameBuffer = []
350
+ this.batchExpected = []
351
+ this.pipeline = { ...this.pipeline, buffered: 0, inFlightFrames: 0, inFlightBatches: 0 }
352
+ if (!this.streaming) return
353
+ this.streaming = false
354
+ // Burn a request id so straggling frames from the session just retired are
355
+ // recognized as stale by `onMessage` and closed rather than buffered.
356
+ this.claimReqId()
357
+ }
358
+
359
+ nextFrameFor(clockUs: number): { frame: VideoFrame | null; dropped: number } {
360
+ let dueIdx = -1
361
+ for (let i = 0; i < this.frameBuffer.length; i++) {
362
+ if (this.frameBuffer[i].timestamp <= clockUs) dueIdx = i
363
+ else break
364
+ }
365
+
366
+ if (dueIdx < 0) {
367
+ // Nothing due: hold what's on screen, close nothing, but keep the
368
+ // pipeline fed (a starving buffer is exactly when a top-up matters).
369
+ this.pump()
370
+ return { frame: null, dropped: 0 }
371
+ }
372
+
373
+ // Everything before the due frame became due and was overtaken before this
374
+ // call — a drop, counted and closed, never silent loss.
375
+ let droppedNow = 0
376
+ for (let i = 0; i < dueIdx; i++) {
377
+ this.frameBuffer[i].close()
378
+ droppedNow++
379
+ }
380
+ const frame = this.frameBuffer[dueIdx]
381
+ this.frameBuffer.splice(0, dueIdx + 1)
382
+ this.dropped += droppedNow
383
+ this.pipeline = onFramesConsumed(this.pipeline, droppedNow + 1)
384
+ this.pump()
385
+ return { frame, dropped: droppedNow }
386
+ }
387
+
388
+ stats(): FrameServerStats {
389
+ const atEndOfSource = this.pipeline.nextSampleIdx >= this.video.samples.length
390
+ return {
391
+ buffered: this.frameBuffer.length,
392
+ inFlightFrames: this.pipeline.inFlightFrames,
393
+ inFlightBatches: this.pipeline.inFlightBatches,
394
+ received: this.received,
395
+ dropped: this.dropped,
396
+ atEndOfSource,
397
+ drained:
398
+ atEndOfSource && this.frameBuffer.length === 0 && this.pipeline.inFlightBatches === 0,
399
+ lastError: this.lastError,
400
+ }
401
+ }
402
+
403
+ dispose(): void {
404
+ if (this.disposed) return
405
+ this.disposed = true
406
+ this.stopStream()
407
+ for (const [, pending] of this.pendingSeeks) {
408
+ if (pending.best) pending.best.close()
409
+ pending.resolve(null)
410
+ }
411
+ this.pendingSeeks.clear()
412
+ this.worker.onmessage = null
413
+ this.worker.terminate()
414
+ }
415
+
416
+ // ── worker protocol ──────────────────────────────────────────────────────
417
+
418
+ /**
419
+ * Post one decode range. Samples are sliced out of the demuxed table in
420
+ * DECODE order and mapped to the wire shape; no transfer list, on purpose —
421
+ * see the module doc.
422
+ *
423
+ * ── Ranged sources ──
424
+ * A ranged source (`demux.ts`) has the sample INDEX but not necessarily the
425
+ * sample BYTES, so the range has to be `ensure`d first. `ensure` returns
426
+ * `null` when the bytes are already there — which is always, on the
427
+ * whole-file path, and usually, mid-stream — so the common case stays exactly
428
+ * as synchronous as it was before ranged loading existed. That matters: a
429
+ * seek that had to hop a microtask would be a seek that paints a frame late.
430
+ *
431
+ * When a fetch IS needed, every subsequent post queues behind it. Not an
432
+ * optimization detail — `VideoDecoder` takes chunks in DECODE order and
433
+ * throws on the first one out of sequence, so a later batch overtaking an
434
+ * earlier one that is still fetching would kill the decoder for the rest of
435
+ * the session (SP1 §7.1, from the other direction).
436
+ */
437
+ private postBatch(startIdx: number, endIdx: number, targetTsUs: number, reqId: number): void {
438
+ if (this.postDepth === 0) {
439
+ const pending = this.video.ensure?.(startIdx, endIdx)
440
+ if (!pending) {
441
+ this.sendBatch(startIdx, endIdx, targetTsUs, reqId)
442
+ return
443
+ }
444
+ this.queuePost(pending, startIdx, endIdx, targetTsUs, reqId)
445
+ return
446
+ }
447
+ this.queuePost(null, startIdx, endIdx, targetTsUs, reqId)
448
+ }
449
+
450
+ /** Chain a post behind whatever byte fetches are already queued. */
451
+ private queuePost(
452
+ started: Promise<void> | null,
453
+ startIdx: number,
454
+ endIdx: number,
455
+ targetTsUs: number,
456
+ reqId: number,
457
+ ): void {
458
+ const gen = this.postGen
459
+ this.postDepth++
460
+ this.postQueue = this.postQueue
461
+ .then(async () => {
462
+ if (this.disposed) return
463
+ // A batch whose request was superseded while its bytes were in flight
464
+ // has nothing to answer, so it is not sent: decoding it would only
465
+ // produce frames `onFrameEvt` closes as stale. Its seek still has to be
466
+ // settled here — `claimReqId` marks a superseded seek and closes the
467
+ // frame it was holding, but the `null` normally arrives with the
468
+ // worker's `done`, which is never coming for a batch that never went.
469
+ if (this.retired(gen, reqId)) return
470
+ await (started ?? this.video.ensure?.(startIdx, endIdx))
471
+ if (this.disposed || this.retired(gen, reqId)) return
472
+ this.sendBatch(startIdx, endIdx, targetTsUs, reqId)
473
+ })
474
+ .catch((err: unknown) => {
475
+ this.failPost(reqId, err instanceof Error ? err.message : String(err))
476
+ })
477
+ .finally(() => {
478
+ if (gen === this.postGen) this.postDepth--
479
+ })
480
+ }
481
+
482
+ /** Superseded or from a retired chain: settle the seek and post nothing. */
483
+ private retired(gen: number, reqId: number): boolean {
484
+ if (gen === this.postGen && !isSuperseded(reqId, this.latestReqId)) return false
485
+ this.dropSeek(reqId)
486
+ return true
487
+ }
488
+
489
+ /**
490
+ * Retire the current post chain so the next post starts from an empty queue.
491
+ *
492
+ * Called from `claimReqId`, where everything older has just been superseded
493
+ * by definition — so nothing still queued can be wanted, and making the new
494
+ * request wait behind an abandoned fetch would hand back exactly the latency
495
+ * ranged loading exists to remove. No-op when nothing is queued, which is the
496
+ * normal case and keeps `postBatch`'s synchronous path intact.
497
+ */
498
+ private retirePostQueue(): void {
499
+ if (this.postDepth === 0) return
500
+ this.postGen++
501
+ this.postQueue = Promise.resolve()
502
+ this.postDepth = 0
503
+ }
504
+
505
+ /** Settle a seek with `null` and release the frame it was holding. */
506
+ private dropSeek(reqId: number): void {
507
+ const pending = this.pendingSeeks.get(reqId)
508
+ if (!pending) return
509
+ this.pendingSeeks.delete(reqId)
510
+ if (pending.best) {
511
+ pending.best.close()
512
+ this.dropped++
513
+ }
514
+ pending.resolve(null)
515
+ }
516
+
517
+ private sendBatch(startIdx: number, endIdx: number, targetTsUs: number, reqId: number): void {
518
+ const samples: WorkerSample[] = []
519
+ for (let i = startIdx; i < endIdx; i++) {
520
+ const s: SampleRef = this.video.samples[i]
521
+ samples.push({ tsUs: s.tsUs, durUs: s.durUs, isKey: s.isKey, data: s.data })
522
+ }
523
+ this.worker.postMessage({ t: 'decodeRange', samples, targetTsUs, reqId })
524
+ }
525
+
526
+ /**
527
+ * A batch's bytes could not be fetched.
528
+ *
529
+ * Reported through the same `onError` a decode failure uses, so `index.ts`
530
+ * fails the session and the clip lands on the Preparing placeholder with a
531
+ * reason — the same outcome a proxy that never loaded produces. A seek
532
+ * waiting on this batch is settled with `null` first: its frame promise is
533
+ * contractually total, and leaving it pending would hang the scrub rather
534
+ * than showing the failure.
535
+ */
536
+ private failPost(reqId: number, message: string): void {
537
+ if (this.disposed) return
538
+ this.lastError = message
539
+ this.dropSeek(reqId)
540
+ this.onError?.(message)
541
+ }
542
+
543
+ /** Top up decode-ahead from the planner's decisions. */
544
+ private pump(): void {
545
+ if (!this.streaming || this.disposed) return
546
+ const { batches, state } = planNextBatches(this.video, this.pipeline, this.config)
547
+ this.pipeline = state
548
+ for (const batch of batches) {
549
+ this.batchExpected.push(batch.count)
550
+ this.postBatch(batch.startIdx, batch.endIdx, batch.targetTsUs, this.streamReqId)
551
+ }
552
+ }
553
+
554
+ private onMessage(evt: DecodeEvt): void {
555
+ if (this.disposed) {
556
+ if (evt.t === 'frame') evt.frame.close()
557
+ return
558
+ }
559
+ if (evt.t === 'error') {
560
+ this.lastError = evt.message
561
+ this.onError?.(evt.message)
562
+ return
563
+ }
564
+ if (evt.t === 'frame') {
565
+ this.onFrameEvt(evt.frame, evt.reqId)
566
+ return
567
+ }
568
+ this.onDoneEvt(evt.reqId, evt.decoded)
569
+ }
570
+
571
+ private onFrameEvt(frame: VideoFrame, reqId: number): void {
572
+ // Belt-and-suspenders for the worker's own supersession check: a frame can
573
+ // already have been transferred when the newer request went out.
574
+ if (isSuperseded(reqId, this.latestReqId)) {
575
+ frame.close()
576
+ this.dropped++
577
+ return
578
+ }
579
+ this.received++
580
+
581
+ if (this.streaming && reqId === this.streamReqId) {
582
+ // Kept sorted rather than appended: VideoDecoder emits in PRESENTATION
583
+ // order, which equals arrival order only for all-intra sources like
584
+ // SP3's proxies. The spike appended and documented the assumption; one
585
+ // insertion into a ≤N buffer costs nothing and removes it.
586
+ insertByTimestamp(this.frameBuffer, frame)
587
+ this.pipeline = onFrameBuffered(this.pipeline)
588
+ return
589
+ }
590
+
591
+ const pending = this.pendingSeeks.get(reqId)
592
+ if (!pending) {
593
+ frame.close()
594
+ this.dropped++
595
+ return
596
+ }
597
+ // Keep the EARLIEST arriving frame and close the rest. The worker has
598
+ // already dropped everything below `target - epsilon` as pre-roll, so the
599
+ // lowest timestamp that survives is the requested frame itself; anything
600
+ // later is a B-frame that precedes the target in DECODE order and follows
601
+ // it in presentation order. The spike painted whatever arrived last, which
602
+ // on a reordered source is the wrong frame — invisible on SP3's all-intra
603
+ // proxies, so it is fixed here rather than left to be discovered later.
604
+ if (!pending.best || frame.timestamp < pending.best.timestamp) {
605
+ if (pending.best) {
606
+ pending.best.close()
607
+ this.dropped++
608
+ }
609
+ pending.best = frame
610
+ return
611
+ }
612
+ frame.close()
613
+ this.dropped++
614
+ }
615
+
616
+ private onDoneEvt(reqId: number, decoded: number): void {
617
+ if (this.streaming && reqId === this.streamReqId) {
618
+ const expected = this.batchExpected.shift() ?? 0
619
+ this.pipeline = reconcileOnBatchDone(this.pipeline, expected, decoded)
620
+ this.pump()
621
+ return
622
+ }
623
+ const pending = this.pendingSeeks.get(reqId)
624
+ if (!pending) return
625
+ this.pendingSeeks.delete(reqId)
626
+ const frame = pending.superseded ? null : pending.best
627
+ if (pending.superseded && pending.best) {
628
+ pending.best.close()
629
+ this.dropped++
630
+ }
631
+ pending.resolve(frame)
632
+ }
633
+ }
634
+
635
+ /**
636
+ * Create a frame server for one demuxed source.
637
+ *
638
+ * T5 consumes this surface: `seek` for scrubbing, `startStream`/`nextFrameFor`
639
+ * for the playback tick, `stats` for end-of-clip detection and the T7 HUD,
640
+ * `dispose` for the clip-boundary swap.
641
+ */
642
+ export function createFrameServer(options: FrameServerOptions): FrameServer {
643
+ return new FrameServerImpl(options)
644
+ }