@bycrux/editor 0.11.2 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -1
- package/src/ControlsInfoModal.tsx +251 -61
- package/src/__tests__/ControlsInfoModal.test.tsx +43 -0
- package/src/__tests__/adapter.test.ts +59 -1
- package/src/__tests__/schema-assignability.test.ts +93 -0
- package/src/__tests__/schema.test.ts +15 -0
- package/src/__tests__/video-adapter-contract.test.ts +128 -5
- package/src/carousel/AddElementMenu.tsx +10 -4
- package/src/carousel/CarouselEditor.tsx +46 -8
- package/src/carousel/CarouselRenderModal.tsx +15 -10
- package/src/carousel/OverlayPicker.tsx +10 -3
- package/src/carousel/SlidePropertyPanel.tsx +43 -21
- package/src/components/FilmstripScrubber.tsx +277 -0
- package/src/components/__tests__/FilmstripScrubber.test.tsx +216 -0
- package/src/engine/__tests__/audio-clock.test.ts +655 -0
- package/src/engine/__tests__/audio-worklet-source.test.ts +316 -0
- package/src/engine/__tests__/batch-planner.test.ts +298 -0
- package/src/engine/__tests__/decode-worker-source.test.ts +249 -0
- package/src/engine/__tests__/demux-ranged.test.ts +495 -0
- package/src/engine/__tests__/demux-truncated.test.ts +136 -0
- package/src/engine/__tests__/demux.test.ts +305 -0
- package/src/engine/__tests__/eligibility.test.ts +146 -0
- package/src/engine/__tests__/engine-recovery.test.ts +263 -0
- package/src/engine/__tests__/engine.test.ts +326 -0
- package/src/engine/__tests__/frame-server-ranged.test.ts +261 -0
- package/src/engine/__tests__/frame-server.test.ts +326 -0
- package/src/engine/__tests__/media-loader-ranged.test.ts +289 -0
- package/src/engine/__tests__/media-loader.test.ts +100 -0
- package/src/engine/__tests__/scheduler-crop.test.ts +353 -0
- package/src/engine/__tests__/scheduler.test.ts +1310 -0
- package/src/engine/__tests__/scrub-resolve.test.ts +96 -0
- package/src/engine/__tests__/scrub-source.test.ts +195 -0
- package/src/engine/__tests__/source-host.test.ts +455 -0
- package/src/engine/__tests__/time-stretch.test.ts +182 -0
- package/src/engine/audio-clock.ts +1179 -0
- package/src/engine/audio-worklet-source.ts +160 -0
- package/src/engine/batch-planner.ts +308 -0
- package/src/engine/debug-hud.tsx +54 -0
- package/src/engine/decode-worker-source.ts +142 -0
- package/src/engine/demux.ts +900 -0
- package/src/engine/eligibility.ts +140 -0
- package/src/engine/frame-server.ts +644 -0
- package/src/engine/index.ts +950 -0
- package/src/engine/media-loader.ts +380 -0
- package/src/engine/mp4box.d.ts +214 -0
- package/src/engine/scheduler.ts +1324 -0
- package/src/engine/scrub-resolve.ts +66 -0
- package/src/engine/scrub-source.ts +496 -0
- package/src/engine/time-stretch.ts +224 -0
- package/src/index.ts +83 -1
- package/src/preview/OverlayPreview.tsx +2 -24
- package/src/schema.ts +146 -3
- package/src/state/__tests__/use-project-sync.test.tsx +56 -0
- package/src/state/use-project-sync.ts +17 -0
- package/src/test-setup.ts +32 -0
- package/src/text/FontPicker.tsx +85 -51
- package/src/text/TextFormattingToolbar.tsx +6 -1
- package/src/text/__tests__/FontPicker.options.test.ts +43 -0
- package/src/text/__tests__/TextFormattingToolbar.test.tsx +4 -4
- package/src/theme.ts +108 -3
- package/src/types.ts +601 -22
- package/src/ui/Loader.tsx +64 -0
- package/src/ui/NumberField.tsx +254 -0
- package/src/ui/Slider.tsx +128 -0
- package/src/ui/Tooltip.tsx +118 -0
- package/src/ui/__tests__/NumberField.test.tsx +298 -0
- package/src/ui/__tests__/Slider.test.tsx +150 -0
- package/src/ui/__tests__/Tooltip.test.tsx +105 -0
- package/src/ui/__tests__/usePersistentState.test.tsx +112 -0
- package/src/ui/badge.tsx +12 -4
- package/src/ui/index.ts +6 -0
- package/src/ui/input.tsx +1 -1
- package/src/ui/select.tsx +1 -1
- package/src/ui/switch.tsx +12 -2
- package/src/ui/textarea.tsx +1 -1
- package/src/ui/usePersistentState.ts +63 -0
- package/src/video/AudioPolishModal.tsx +983 -0
- package/src/video/CaptionListPanel.test.tsx +944 -0
- package/src/video/CaptionListPanel.tsx +1106 -0
- package/src/video/CaptionRegenModal.tsx +37 -9
- package/src/video/CaptionSpecimen.tsx +160 -0
- package/src/video/CaptionStyleGallery.tsx +466 -0
- package/src/video/CommandPalette.tsx +165 -0
- package/src/video/ImageToneMenu.tsx +137 -0
- package/src/video/OverlayInspector.tsx +977 -0
- package/src/video/RenderModal.tsx +1035 -62
- package/src/video/VersionCompare.tsx +258 -0
- package/src/video/VersionPanel.tsx +117 -42
- package/src/video/VideoEditor.tsx +2229 -243
- package/src/video/__tests__/AudioPolishModal.test.tsx +825 -0
- package/src/video/__tests__/CaptionListPanel.font.test.tsx +397 -0
- package/src/video/__tests__/CaptionListPanel.generate.test.tsx +153 -0
- package/src/video/__tests__/CaptionRegenModal.test.tsx +29 -0
- package/src/video/__tests__/CaptionSpecimen.test.tsx +213 -0
- package/src/video/__tests__/CaptionStyleGallery.test.tsx +362 -0
- package/src/video/__tests__/CommandPalette.test.tsx +119 -0
- package/src/video/__tests__/OverlayInspector.test.tsx +1367 -0
- package/src/video/__tests__/RenderModal.exportControls.test.tsx +319 -0
- package/src/video/__tests__/RenderModal.options.test.tsx +562 -0
- package/src/video/__tests__/RenderModal.progress.test.ts +65 -0
- package/src/video/__tests__/VersionCompare.test.tsx +182 -0
- package/src/video/__tests__/VersionPanel.test.tsx +279 -0
- package/src/video/__tests__/VideoEditor.audioPolish.test.tsx +241 -0
- package/src/video/__tests__/VideoEditor.captionDelete.test.tsx +147 -0
- package/src/video/__tests__/VideoEditor.captionGesture.test.tsx +170 -0
- package/src/video/__tests__/VideoEditor.captionSeam.test.tsx +134 -0
- package/src/video/__tests__/VideoEditor.clipKeyframes.test.tsx +59 -0
- package/src/video/__tests__/VideoEditor.context.test.tsx +44 -0
- package/src/video/__tests__/VideoEditor.editFocus.test.tsx +55 -0
- package/src/video/__tests__/VideoEditor.keymap.test.tsx +725 -0
- package/src/video/__tests__/VideoEditor.layout.test.tsx +338 -0
- package/src/video/__tests__/VideoEditor.propertiesPanel.test.tsx +765 -0
- package/src/video/__tests__/VideoEditor.rippleDeleteCaptions.test.tsx +159 -0
- package/src/video/__tests__/VideoEditor.sourcePreview.test.tsx +122 -0
- package/src/video/__tests__/VideoEditor.test.tsx +403 -50
- package/src/video/__tests__/audioMagnet.test.ts +158 -0
- package/src/video/__tests__/audioPolish.test.ts +1202 -0
- package/src/video/__tests__/captionActiveWord.test.ts +107 -0
- package/src/video/__tests__/captionLanes.test.ts +262 -0
- package/src/video/__tests__/captionPositioning.test.tsx +8 -3
- package/src/video/__tests__/captionWordFloor.test.ts +228 -0
- package/src/video/__tests__/clipboard-ops.test.ts +430 -0
- package/src/video/__tests__/cuts.insert.test.ts +244 -0
- package/src/video/__tests__/cuts.test.ts +1213 -34
- package/src/video/__tests__/export-limits.test.ts +195 -0
- package/src/video/__tests__/hover-scrub.test.ts +163 -0
- package/src/video/__tests__/keyframeOps.canKeyframeProp.test.ts +61 -0
- package/src/video/__tests__/keyframeOps.test.ts +643 -0
- package/src/video/__tests__/keymap.test.tsx +171 -0
- package/src/video/__tests__/render-progress.test.tsx +20 -4
- package/src/video/__tests__/shuttle.test.ts +210 -0
- package/src/video/__tests__/source-preview.test.ts +60 -0
- package/src/video/__tests__/timecode.test.ts +77 -0
- package/src/video/__tests__/use-report-context.test.tsx +101 -0
- package/src/video/audioMagnet.ts +72 -0
- package/src/video/audioPolish.ts +774 -0
- package/src/video/captionActiveWord.ts +74 -0
- package/src/video/captionLanes.ts +202 -0
- package/src/video/captionRepair.ts +9 -5
- package/src/video/captionStyleDefaults.ts +100 -0
- package/src/video/captionWordFloor.ts +83 -0
- package/src/video/clipboard-ops.ts +377 -0
- package/src/video/cuts.ts +713 -37
- package/src/video/export-limits.ts +102 -0
- package/src/video/hover-scrub.ts +102 -0
- package/src/video/imageTone.ts +58 -0
- package/src/video/imageToneExamples.ts +10 -0
- package/src/video/keyframeOps.ts +384 -0
- package/src/video/keymap.ts +146 -0
- package/src/video/panels/ClipPropertiesPanel.tsx +706 -0
- package/src/video/panels/LeftPanelTabs.tsx +187 -0
- package/src/video/panels/OverlayContentPanel.tsx +351 -0
- package/src/video/panels/TabNav.tsx +60 -0
- package/src/video/panels/__tests__/ClipPropertiesPanel.test.tsx +645 -0
- package/src/video/panels/__tests__/LeftPanelTabs.test.tsx +189 -0
- package/src/video/panels/__tests__/OverlayContentPanel.test.tsx +222 -0
- package/src/video/panels/__tests__/TabNav.test.tsx +71 -0
- package/src/video/preview/CaptionPreview.tsx +57 -69
- package/src/video/preview/EngineSurface.tsx +103 -0
- package/src/video/preview/OverlayItemsLayer.tsx +315 -103
- package/src/video/preview/PreviewPlayer.tsx +337 -56
- package/src/video/preview/SocialPreviewMenu.tsx +214 -0
- package/src/video/preview/SocialSafeZoneOverlay.tsx +478 -0
- package/src/video/preview/__tests__/CaptionPreview.fonts.test.tsx +97 -0
- package/src/video/preview/__tests__/EngineSurface.test.tsx +114 -0
- package/src/video/preview/__tests__/OverlayItemsLayer.edit.test.tsx +4 -2
- package/src/video/preview/__tests__/OverlayItemsLayer.keyframes.test.tsx +363 -0
- package/src/video/preview/__tests__/OverlayItemsLayer.selection.test.tsx +329 -0
- package/src/video/preview/__tests__/PreviewPlayer.engine.test.tsx +139 -0
- package/src/video/preview/__tests__/SocialPreviewMenu.test.tsx +121 -0
- package/src/video/preview/__tests__/SocialSafeZoneOverlay.test.tsx +165 -0
- package/src/video/preview/__tests__/captionDragState.test.ts +120 -1
- package/src/video/preview/__tests__/latencyCompensation.test.tsx +451 -0
- package/src/video/preview/__tests__/proxySupport.test.ts +75 -0
- package/src/video/preview/__tests__/transformStyle.test.ts +29 -1
- package/src/video/preview/__tests__/useDragOverlay.perAxis.test.ts +197 -0
- package/src/video/preview/__tests__/useEnginePlayback.test.tsx +530 -0
- package/src/video/preview/__tests__/useVideoPlayback.corpus.test.ts +313 -0
- package/src/video/preview/__tests__/useVideoPlayback.test.ts +38 -5
- package/src/video/preview/__tests__/useVideoPlayback.trackAudio.test.ts +278 -0
- package/src/video/preview/audio-context.ts +111 -0
- package/src/video/preview/captionDragState.ts +91 -1
- package/src/video/preview/proxySupport.ts +86 -0
- package/src/video/preview/transformStyle.ts +22 -12
- package/src/video/preview/useDragOverlay.ts +92 -18
- package/src/video/preview/useEnginePlayback.ts +625 -0
- package/src/video/preview/useVideoPlayback.ts +211 -167
- package/src/video/sdrCurves.ts +56 -0
- package/src/video/shuttle.ts +159 -0
- package/src/video/source-preview.ts +66 -0
- package/src/video/timecode.ts +59 -0
- package/src/video/timeline/EditableSegment.tsx +1 -1
- package/src/video/timeline/Scrubber.tsx +46 -174
- package/src/video/timeline/SpeedControl.tsx +95 -0
- package/src/video/timeline/Timeline.tsx +1061 -328
- package/src/video/timeline/TimelineContext.ts +17 -10
- package/src/video/timeline/TrackGutter.tsx +560 -0
- package/src/video/timeline/TrackSettingsPopover.tsx +228 -0
- package/src/video/timeline/VolumeControl.tsx +113 -0
- package/src/video/timeline/__tests__/Timeline.backgroundClick.test.tsx +110 -0
- package/src/video/timeline/__tests__/Timeline.crossfade.test.tsx +104 -0
- package/src/video/timeline/__tests__/Timeline.fadeCurveMenu.test.tsx +174 -0
- package/src/video/timeline/__tests__/Timeline.keyframeDelete.test.tsx +273 -0
- package/src/video/timeline/__tests__/Timeline.keyframeFollow.test.tsx +215 -0
- package/src/video/timeline/__tests__/Timeline.keyframeMenu.test.tsx +253 -0
- package/src/video/timeline/__tests__/Timeline.keymap.test.tsx +372 -0
- package/src/video/timeline/__tests__/Timeline.subcutRegen.test.tsx +351 -0
- package/src/video/timeline/__tests__/TrackGutter.test.tsx +372 -0
- package/src/video/timeline/__tests__/_canvasSelect.test.tsx +273 -0
- package/src/video/timeline/__tests__/_canvasSelect.ts +414 -0
- package/src/video/timeline/__tests__/dragdrop-math.test.ts +135 -0
- package/src/video/timeline/__tests__/effectiveItemAudio.test.ts +50 -0
- package/src/video/timeline/__tests__/enabledTrackItems.test.ts +164 -0
- package/src/video/timeline/__tests__/moveItemAcrossTracks.test.ts +363 -0
- package/src/video/timeline/__tests__/multiSelectOps.test.ts +447 -0
- package/src/video/timeline/__tests__/placement.test.ts +278 -0
- package/src/video/timeline/__tests__/resizeWindowedItem.test.ts +140 -0
- package/src/video/timeline/__tests__/timeline-model.test.ts +576 -0
- package/src/video/timeline/__tests__/visualItemLabel.test.ts +69 -0
- package/src/video/timeline/canvas/TimelineCanvas.tsx +1447 -0
- package/src/video/timeline/canvas/__tests__/TimelineCanvas.drop.test.tsx +439 -0
- package/src/video/timeline/canvas/__tests__/TimelineCanvas.edgeScroll.test.tsx +346 -0
- package/src/video/timeline/canvas/__tests__/TimelineCanvas.panefill.test.tsx +122 -0
- package/src/video/timeline/canvas/__tests__/TimelineCanvas.pendingDrops.test.tsx +316 -0
- package/src/video/timeline/canvas/__tests__/TimelineCanvas.pointer.test.tsx +606 -0
- package/src/video/timeline/canvas/__tests__/TimelineCanvas.test.tsx +407 -0
- package/src/video/timeline/canvas/__tests__/clip-bands.test.ts +77 -0
- package/src/video/timeline/canvas/__tests__/draw.test.ts +2198 -0
- package/src/video/timeline/canvas/__tests__/fade-curve.test.ts +187 -0
- package/src/video/timeline/canvas/__tests__/filmstrips.test.ts +561 -0
- package/src/video/timeline/canvas/__tests__/hit-test.test.ts +818 -0
- package/src/video/timeline/canvas/__tests__/pending-drop.test.ts +210 -0
- package/src/video/timeline/canvas/__tests__/pointer-machine.test.ts +3358 -0
- package/src/video/timeline/canvas/__tests__/snap.test.ts +257 -0
- package/src/video/timeline/canvas/__tests__/viewport.test.ts +399 -0
- package/src/video/timeline/canvas/__tests__/waveforms.test.ts +946 -0
- package/src/video/timeline/canvas/clip-bands.ts +56 -0
- package/src/video/timeline/canvas/draw.ts +2187 -0
- package/src/video/timeline/canvas/fade-curve.ts +111 -0
- package/src/video/timeline/canvas/filmstrips.ts +418 -0
- package/src/video/timeline/canvas/hit-test.ts +501 -0
- package/src/video/timeline/canvas/keyframe-strip.ts +73 -0
- package/src/video/timeline/canvas/pointer-machine.ts +1828 -0
- package/src/video/timeline/canvas/snap.ts +232 -0
- package/src/video/timeline/canvas/viewport.ts +457 -0
- package/src/video/timeline/canvas/waveforms.ts +664 -0
- package/src/video/timeline/makeCaptionEdit.ts +5 -1
- package/src/video/timeline/multiSelectOps.ts +214 -55
- package/src/video/timeline/placement.ts +282 -0
- package/src/video/timeline/timeline-model.ts +919 -0
- package/src/video/timeline/useItemDragDrop.ts +151 -178
- package/src/video/timeline/useTimelineZoom.ts +25 -60
- package/src/video/timeline/utils.ts +0 -12
- package/src/video/use-report-context.ts +75 -0
- package/src/video/preview/OverlayPropsModal.tsx +0 -292
- package/src/video/preview/__tests__/OverlayPropsModal.test.tsx +0 -32
- package/src/video/timeline/AudioTrackRow.tsx +0 -404
- package/src/video/timeline/AudioWaveformLayer.tsx +0 -117
- package/src/video/timeline/CaptionTrackRow.tsx +0 -235
- package/src/video/timeline/PlayheadLine.tsx +0 -18
- package/src/video/timeline/TranscriptModal.tsx +0 -70
- package/src/video/timeline/TranscriptPanel.tsx +0 -273
- package/src/video/timeline/VisualTrackRow.tsx +0 -300
- package/src/video/timeline/__tests__/CaptionTrackRow.test.tsx +0 -241
- package/src/video/timeline/__tests__/PlayheadLine.test.tsx +0 -60
- package/src/video/timeline/__tests__/TranscriptModal.test.tsx +0 -41
- package/src/video/timeline/__tests__/TranscriptPanel.test.tsx +0 -184
- package/src/video/timeline/__tests__/useItemDragDrop.test.ts +0 -72
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SP4 T4 — the master clock's `AudioWorkletProcessor`, as an inlined source
|
|
3
|
+
* string.
|
|
4
|
+
*
|
|
5
|
+
* Loaded via `URL.createObjectURL(new Blob([audioWorkletSource], { type:
|
|
6
|
+
* 'text/javascript' }))` and `ctx.audioWorklet.addModule(url)` — plan decision
|
|
7
|
+
* 6, the same rule `decode-worker-source.ts` follows and for the same reason:
|
|
8
|
+
* the Blob-URL form is the only one that needs nothing from a consumer's
|
|
9
|
+
* bundler (`?raw` is Vite-only, and montaj's ui is not the only host).
|
|
10
|
+
*
|
|
11
|
+
* Because it is a string it cannot be type-checked, linted or unit-tested, so
|
|
12
|
+
* — exactly like the decode worker — it is **thin by architecture**. Every
|
|
13
|
+
* decision lives main-side in `audio-clock.ts` under test: what to decode,
|
|
14
|
+
* how much to keep queued, the resample ratio, the project↔media time mapping,
|
|
15
|
+
* the clock extrapolation (volume rides a main-side output GainNode, downstream
|
|
16
|
+
* of this node — not the ring). What is left here is
|
|
17
|
+
* the irreducible audio-thread part:
|
|
18
|
+
*
|
|
19
|
+
* 1. a FIFO of interleaved Float32 PCM chunks, drained one render quantum at
|
|
20
|
+
* a time ("ring buffer" is the spike's name for it; the transport is
|
|
21
|
+
* postMessage chunks, NOT a `SharedArrayBuffer` — Hub cannot serve the
|
|
22
|
+
* COOP/COEP headers cross-origin isolation requires, plan decision 4);
|
|
23
|
+
* 2. `samplesConsumed`: how many output frames have actually been rendered
|
|
24
|
+
* since the last reset. This — not decode progress, not wall time — is
|
|
25
|
+
* the master clock's ground truth, reported back at ~10Hz;
|
|
26
|
+
* 3. **silence through underruns**: an empty queue renders silence and the
|
|
27
|
+
* frame counter still advances. A real audio device does not stall when
|
|
28
|
+
* it is starved, and neither may the clock: freezing it here would freeze
|
|
29
|
+
* the canvas painter that follows it (SP1 §6 measured playback against
|
|
30
|
+
* exactly this clock);
|
|
31
|
+
* 4. a `playing` gate, so pause stops output *and* stops the clock without
|
|
32
|
+
* tearing down the node;
|
|
33
|
+
* 5. an `epoch` echoed on every report, so main can discard readings that
|
|
34
|
+
* were in flight when it reset the ring (a seek). Same shape as the decode
|
|
35
|
+
* worker's `reqId` supersession — SP1 §7.3/§7.4 are what stale-request
|
|
36
|
+
* bookkeeping costs when it is missing.
|
|
37
|
+
*
|
|
38
|
+
* Constants are INJECTED via `processorOptions` rather than duplicated in the
|
|
39
|
+
* string (the string cannot import), matching how `decode-worker-source.ts`
|
|
40
|
+
* receives `preRollEpsilonUs` at `init`.
|
|
41
|
+
*
|
|
42
|
+
* `spikes/playback-engine/src/audio-worklet.js` is the precedent. The
|
|
43
|
+
* deliberate additions over it are the `playing` gate, the `epoch`, and the
|
|
44
|
+
* duplicate-registration guard — the spike had no transport (it played once,
|
|
45
|
+
* from a context it owned and threw away), whereas this runs on the page's
|
|
46
|
+
* ONE shared `window.__montajSharedCtx`, which outlives every clock built on
|
|
47
|
+
* it and would otherwise throw `NotSupportedError` on the second
|
|
48
|
+
* `registerProcessor` of the same name (dev HMR and every clip-boundary
|
|
49
|
+
* clock swap both hit that path).
|
|
50
|
+
*/
|
|
51
|
+
|
|
52
|
+
/** Processor name registered by `audioWorkletSource`. Keep in sync with the string. */
|
|
53
|
+
export const RING_PROCESSOR_NAME = 'montaj-engine-pcm-ring'
|
|
54
|
+
|
|
55
|
+
export const audioWorkletSource = `'use strict';
|
|
56
|
+
|
|
57
|
+
class MontajRingProcessor extends AudioWorkletProcessor {
|
|
58
|
+
constructor(options) {
|
|
59
|
+
super();
|
|
60
|
+
var opts = (options && options.processorOptions) || {};
|
|
61
|
+
// ~10Hz. Injected, not hardcoded: audio-clock.ts owns the number because
|
|
62
|
+
// its wall-clock extrapolation window is derived from the same constant.
|
|
63
|
+
this.reportIntervalS = typeof opts.reportIntervalS === 'number' ? opts.reportIntervalS : 0.1;
|
|
64
|
+
|
|
65
|
+
// FIFO of { pcm: Float32Array (interleaved), channels: number, offsetFrames: number }.
|
|
66
|
+
this.queue = [];
|
|
67
|
+
this.queuedFrames = 0; // frames sitting in the FIFO, not yet rendered
|
|
68
|
+
this.samplesConsumed = 0; // output frames rendered since the last reset — the clock's ground truth
|
|
69
|
+
this.underrunFrames = 0; // of those, how many were silence because the FIFO was empty
|
|
70
|
+
this.epoch = 0; // bumped by main on every reset; echoed on every report
|
|
71
|
+
this.playing = false; // pause gate: no output, no drain, no clock advance
|
|
72
|
+
this.lastReportAt = 0; // AudioWorkletGlobalScope currentTime (s) at the last report
|
|
73
|
+
|
|
74
|
+
this.port.onmessage = (ev) => {
|
|
75
|
+
var msg = ev.data;
|
|
76
|
+
if (msg.t === 'chunk') {
|
|
77
|
+
this.queue.push({ pcm: msg.pcm, channels: msg.channels, offsetFrames: 0 });
|
|
78
|
+
this.queuedFrames += msg.frames;
|
|
79
|
+
} else if (msg.t === 'play') {
|
|
80
|
+
this.playing = true;
|
|
81
|
+
} else if (msg.t === 'pause') {
|
|
82
|
+
this.playing = false;
|
|
83
|
+
// Report immediately: main freezes the playhead at pause and wants the
|
|
84
|
+
// truest sample count it can get rather than waiting up to a full
|
|
85
|
+
// report interval for one.
|
|
86
|
+
this.report();
|
|
87
|
+
} else if (msg.t === 'reset') {
|
|
88
|
+
this.queue = [];
|
|
89
|
+
this.queuedFrames = 0;
|
|
90
|
+
this.samplesConsumed = 0;
|
|
91
|
+
this.underrunFrames = 0;
|
|
92
|
+
this.epoch = msg.epoch;
|
|
93
|
+
this.lastReportAt = currentTime;
|
|
94
|
+
this.report();
|
|
95
|
+
}
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
report() {
|
|
100
|
+
this.port.postMessage({
|
|
101
|
+
t: 'consumed',
|
|
102
|
+
epoch: this.epoch,
|
|
103
|
+
samplesConsumed: this.samplesConsumed,
|
|
104
|
+
underrunFrames: this.underrunFrames,
|
|
105
|
+
queuedFrames: this.queuedFrames,
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
process(_inputs, outputs) {
|
|
110
|
+
var output = outputs[0];
|
|
111
|
+
if (!output || output.length === 0 || !output[0]) return true;
|
|
112
|
+
var frameCount = output[0].length;
|
|
113
|
+
var outChannels = output.length;
|
|
114
|
+
|
|
115
|
+
// Paused: leave \`output\` untouched (it arrives zero-filled, i.e. silence)
|
|
116
|
+
// and do NOT advance the clock or drain the FIFO. Resuming continues from
|
|
117
|
+
// exactly where the audio stopped.
|
|
118
|
+
if (this.playing) {
|
|
119
|
+
for (var i = 0; i < frameCount; i++) {
|
|
120
|
+
var entry = this.queue[0];
|
|
121
|
+
if (!entry) {
|
|
122
|
+
// Underrun: nothing queued. This frame is silence and the clock
|
|
123
|
+
// still advances through it — see the module header, point 3.
|
|
124
|
+
this.underrunFrames++;
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
var frameBase = entry.offsetFrames * entry.channels;
|
|
128
|
+
for (var ch = 0; ch < outChannels; ch++) {
|
|
129
|
+
// Fewer source channels than outputs (mono source, stereo out):
|
|
130
|
+
// duplicate the last source channel rather than emitting silence.
|
|
131
|
+
var srcCh = ch < entry.channels ? ch : entry.channels - 1;
|
|
132
|
+
output[ch][i] = entry.pcm[frameBase + srcCh];
|
|
133
|
+
}
|
|
134
|
+
entry.offsetFrames++;
|
|
135
|
+
this.queuedFrames--;
|
|
136
|
+
if (entry.offsetFrames * entry.channels >= entry.pcm.length) this.queue.shift();
|
|
137
|
+
}
|
|
138
|
+
this.samplesConsumed += frameCount;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
if (currentTime - this.lastReportAt >= this.reportIntervalS) {
|
|
142
|
+
this.lastReportAt = currentTime;
|
|
143
|
+
this.report();
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
return true; // keep the processor alive for the life of the node
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// The AudioWorkletGlobalScope belongs to the CONTEXT, and the context here is
|
|
151
|
+
// the page-lifetime shared one — so this module can be added more than once
|
|
152
|
+
// (a second clock, a dev-server hot reload). A duplicate registerProcessor
|
|
153
|
+
// throws NotSupportedError and rejects the addModule promise, which would drop
|
|
154
|
+
// an otherwise-healthy engine onto the wall-clock fallback. The name stays
|
|
155
|
+
// registered from the first load either way.
|
|
156
|
+
if (!globalThis.__montajEngineRingRegistered) {
|
|
157
|
+
globalThis.__montajEngineRingRegistered = true;
|
|
158
|
+
registerProcessor('${RING_PROCESSOR_NAME}', MontajRingProcessor);
|
|
159
|
+
}
|
|
160
|
+
`
|
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SP4 T3 — the decode planner.
|
|
3
|
+
*
|
|
4
|
+
* Every decision about *what to decode next* lives here, as pure functions
|
|
5
|
+
* over a `SampleTable` (T2) and a small state record. Nothing in this file
|
|
6
|
+
* touches the DOM, WebCodecs, workers or `postMessage`, which is the point:
|
|
7
|
+
* the decode worker ships as an inlined source *string* (plan decision 6) and
|
|
8
|
+
* a string cannot be unit tested the way a module can, so the architecture
|
|
9
|
+
* pushes all planning main-side and leaves the string with nothing but decode
|
|
10
|
+
* calls and supersession bookkeeping. `frame-server.ts` is the imperative
|
|
11
|
+
* shell over these functions; `__tests__/batch-planner.test.ts` is where the
|
|
12
|
+
* SP1 requirements are actually proven.
|
|
13
|
+
*
|
|
14
|
+
* Three of SP1's five engine requirements are decided here — each one is a
|
|
15
|
+
* *fix* for a measured defect in `spikes/playback-engine/RESULTS.md`, not a
|
|
16
|
+
* stylistic preference:
|
|
17
|
+
*
|
|
18
|
+
* 1. **Decode-order feed** (§7.1). Batches are contiguous slices of
|
|
19
|
+
* `samples`, which T2 guarantees is container/decode order. Time→index
|
|
20
|
+
* questions bisect `presIndex` (via T2's `sampleAtOrBefore` /
|
|
21
|
+
* `keyframeAtOrBefore`); the *slice* is never reordered. Feeding
|
|
22
|
+
* WebCodecs in presentation order kills the decoder on the first
|
|
23
|
+
* non-intra frame, and the bug is invisible on all-intra proxies — which
|
|
24
|
+
* is exactly how it survived to SP1's measurement stage.
|
|
25
|
+
* 2. **Pipelined decode-ahead** (§6.1, the 4.5fps bug). Up to
|
|
26
|
+
* `MAX_IN_FLIGHT_BATCHES` requests may sit on the worker at once, and
|
|
27
|
+
* batches are floored at `minBatchFrames(N)`. The pre-fix planner allowed
|
|
28
|
+
* exactly one in flight and sized each batch to the current shortfall, so
|
|
29
|
+
* after the initial fill it asked for **one frame per batch** — and every
|
|
30
|
+
* batch ends in `decoder.flush()`, which drains the whole decoder
|
|
31
|
+
* pipeline. That configuration is still reachable through `PipelineConfig`
|
|
32
|
+
* (`maxInFlightBatches: 1, minBatchFrames: 1`) purely so the test can
|
|
33
|
+
* reproduce the degeneration against this same code.
|
|
34
|
+
* 3. **Min-cts batch targets** (§6.1's second, latent bug). The worker drops
|
|
35
|
+
* frames below `targetTsUs` as pre-roll; in *decode* order `batch[0]` is
|
|
36
|
+
* not the earliest presentation time in the batch, so `batch[0].tsUs` as
|
|
37
|
+
* the target silently discards valid frames. The target is the MINIMUM
|
|
38
|
+
* cts in the batch, floored at the session's real first-wanted frame
|
|
39
|
+
* (the only legitimate pre-roll, i.e. keyframe → start).
|
|
40
|
+
*
|
|
41
|
+
* The fourth (`PRE_ROLL_EPSILON_US`, §7.2) is decided here and *applied*
|
|
42
|
+
* inside the worker — it is handed over in the `init` command rather than
|
|
43
|
+
* hardcoded in the worker string, so the constant has exactly one home.
|
|
44
|
+
*/
|
|
45
|
+
import { keyframeAtOrBefore, sampleAtOrBefore, type SampleRef, type SampleTable } from './demux'
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Tolerance for the worker's pre-roll comparison, in microseconds (SP1 §7.2).
|
|
49
|
+
*
|
|
50
|
+
* `EncodedVideoChunk.timestamp` is a `long long` — INTEGER microseconds —
|
|
51
|
+
* while T2's `SampleRef.tsUs` is deliberately an un-rounded float
|
|
52
|
+
* (`cts / timescale * 1e6`, e.g. `33333.333…`). A chunk fed in at `33333.333`
|
|
53
|
+
* comes back off the decoder stamped `33333`, so a naive
|
|
54
|
+
* `frame.timestamp < targetTsUs` test rejects **the very frame that was
|
|
55
|
+
* requested** as pre-roll: SP1 measured ~2 of 3 seeks painting nothing, and
|
|
56
|
+
* those seeks were silently excluded from the latency distribution rather
|
|
57
|
+
* than failing loudly. Frames are ≥16000µs apart at any sane frame rate, so
|
|
58
|
+
* 1µs of slack cannot admit a genuinely earlier frame.
|
|
59
|
+
*/
|
|
60
|
+
export const PRE_ROLL_EPSILON_US = 1
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* How many `decodeRange` requests may sit on the worker at once. Two is
|
|
64
|
+
* enough to hide the main-thread round trip: the worker starts the next batch
|
|
65
|
+
* the instant it finishes flushing the previous one, instead of idling until
|
|
66
|
+
* main notices and asks again (SP1 §6.1).
|
|
67
|
+
*/
|
|
68
|
+
export const MAX_IN_FLIGHT_BATCHES = 2
|
|
69
|
+
|
|
70
|
+
/** Default decode-ahead budget in frames — the N the spike measured at 30fps. */
|
|
71
|
+
export const DEFAULT_DECODE_AHEAD_FRAMES = 30
|
|
72
|
+
|
|
73
|
+
/** One contiguous run of DECODE indices to hand the worker. */
|
|
74
|
+
export interface DecodeBatch {
|
|
75
|
+
/** First decode index, inclusive. */
|
|
76
|
+
startIdx: number
|
|
77
|
+
/** Last decode index, exclusive. */
|
|
78
|
+
endIdx: number
|
|
79
|
+
/** `endIdx - startIdx`; carried so callers don't recompute it for `batchExpected`. */
|
|
80
|
+
count: number
|
|
81
|
+
/**
|
|
82
|
+
* Pre-roll floor for this batch: the worker emits frames with
|
|
83
|
+
* `cts >= targetTsUs - PRE_ROLL_EPSILON_US` and closes everything below it.
|
|
84
|
+
* Always the minimum cts in the batch (floored at the session's first
|
|
85
|
+
* wanted frame) — never `samples[startIdx].tsUs`.
|
|
86
|
+
*/
|
|
87
|
+
targetTsUs: number
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Decode-ahead bookkeeping for one streaming session. Immutable; every transition returns a new record. */
|
|
91
|
+
export interface PipelineState {
|
|
92
|
+
/** Next decode index to request. Advances monotonically through `samples`. */
|
|
93
|
+
nextSampleIdx: number
|
|
94
|
+
/** Frames decoded, delivered to main, and not yet consumed by the painter. */
|
|
95
|
+
buffered: number
|
|
96
|
+
/** Frames requested but not yet delivered (or accounted for by a batch's `done`). */
|
|
97
|
+
inFlightFrames: number
|
|
98
|
+
/** `decodeRange` requests posted but not yet `done`. */
|
|
99
|
+
inFlightBatches: number
|
|
100
|
+
/**
|
|
101
|
+
* The session's genuine first wanted frame. Only the first batch of a
|
|
102
|
+
* session has real pre-roll (keyframe → start sample); every later batch
|
|
103
|
+
* floors above this anyway, so applying it unconditionally is a no-op after
|
|
104
|
+
* the first batch and keeps the rule in one place.
|
|
105
|
+
*/
|
|
106
|
+
firstBatchTargetTsUs: number
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export interface PipelineConfig {
|
|
110
|
+
/**
|
|
111
|
+
* Decode-ahead budget in frames. `buffered + inFlightFrames` never exceeds
|
|
112
|
+
* it, so N keeps meaning "at most N frames alive at once" — the property
|
|
113
|
+
* SP1 §8's N sweep is measured against.
|
|
114
|
+
*/
|
|
115
|
+
decodeAheadFrames: number
|
|
116
|
+
/** See `MAX_IN_FLIGHT_BATCHES`. */
|
|
117
|
+
maxInFlightBatches: number
|
|
118
|
+
/**
|
|
119
|
+
* Batch floor. Defaults to `minBatchFrames(decodeAheadFrames)`; overridable
|
|
120
|
+
* ONLY so the test can rebuild SP1's pre-fix (4.5fps) configuration against
|
|
121
|
+
* this same planner. Production never sets it.
|
|
122
|
+
*/
|
|
123
|
+
minBatchFrames?: number
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Don't pay a worker round trip plus a full `decoder.flush()` for a handful of
|
|
128
|
+
* frames: floor each batch at a quarter of the decode-ahead budget (minimum
|
|
129
|
+
* 4), so flush cost amortizes over many frames instead of one (SP1 §6.1).
|
|
130
|
+
*/
|
|
131
|
+
export function minBatchFrames(decodeAheadFrames: number): number {
|
|
132
|
+
return Math.max(4, Math.floor(decodeAheadFrames / 4))
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* The pre-roll predicate, shared by the worker (which receives
|
|
137
|
+
* `PRE_ROLL_EPSILON_US` via its `init` command) and by anything main-side that
|
|
138
|
+
* needs to reason about the same boundary. See `PRE_ROLL_EPSILON_US`.
|
|
139
|
+
*/
|
|
140
|
+
export function shouldEmitFrame(
|
|
141
|
+
frameTsUs: number,
|
|
142
|
+
targetTsUs: number,
|
|
143
|
+
epsilonUs: number = PRE_ROLL_EPSILON_US,
|
|
144
|
+
): boolean {
|
|
145
|
+
return frameTsUs >= targetTsUs - epsilonUs
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* The supersession rule, stated once. A request is superseded the moment a
|
|
150
|
+
* strictly newer request id exists; the worker applies it to decide whether to
|
|
151
|
+
* transfer or close a frame, and `frame-server.ts` applies it again main-side
|
|
152
|
+
* to stragglers that were already in flight when the newer request went out.
|
|
153
|
+
*/
|
|
154
|
+
export function isSuperseded(reqId: number, latestReqId: number): boolean {
|
|
155
|
+
return reqId < latestReqId
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Minimum presentation time across a decode-order slice.
|
|
160
|
+
*
|
|
161
|
+
* The whole reason this function exists: in decode order the timestamps
|
|
162
|
+
* zig-zag whenever B-frames are present, so the first sample of a slice is not
|
|
163
|
+
* its earliest presentation time (SP1 §6.1). Latent on SP3's all-intra
|
|
164
|
+
* proxies, live the moment a reordered source shows up.
|
|
165
|
+
*/
|
|
166
|
+
export function minCtsUs(samples: readonly SampleRef[], startIdx: number, endIdx: number): number {
|
|
167
|
+
let min = Infinity
|
|
168
|
+
for (let i = startIdx; i < endIdx; i++) {
|
|
169
|
+
if (samples[i].tsUs < min) min = samples[i].tsUs
|
|
170
|
+
}
|
|
171
|
+
return min
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Plan a one-shot seek: the decode run needed to produce the frame displayed
|
|
176
|
+
* at `targetTsUs` (raw container microseconds — the caller maps project time
|
|
177
|
+
* through `firstPresentationTsUs`).
|
|
178
|
+
*
|
|
179
|
+
* The run is `samples.slice(keyframeAtOrBefore, sampleAtOrBefore + 1)`, both
|
|
180
|
+
* of which are DECODE indices from T2, and `targetTsUs` on the returned batch
|
|
181
|
+
* is the exact cts of the requested frame — so the worker emits that frame and
|
|
182
|
+
* anything at-or-after it in the run, and closes the keyframe→target pre-roll.
|
|
183
|
+
* Returns `null` for an empty table.
|
|
184
|
+
*/
|
|
185
|
+
export function planSeek(table: SampleTable, targetTsUs: number): DecodeBatch | null {
|
|
186
|
+
if (table.samples.length === 0) return null
|
|
187
|
+
const startIdx = keyframeAtOrBefore(table, targetTsUs)
|
|
188
|
+
const targetIdx = sampleAtOrBefore(table, targetTsUs)
|
|
189
|
+
const endIdx = targetIdx + 1
|
|
190
|
+
return {
|
|
191
|
+
startIdx,
|
|
192
|
+
endIdx,
|
|
193
|
+
count: endIdx - startIdx,
|
|
194
|
+
targetTsUs: table.samples[targetIdx].tsUs,
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/**
|
|
199
|
+
* Open a streaming session at `fromTsUs`: decoding starts at the enclosing
|
|
200
|
+
* keyframe, and everything between that keyframe and the first wanted frame is
|
|
201
|
+
* this session's only genuine pre-roll.
|
|
202
|
+
*/
|
|
203
|
+
export function planStreamStart(table: SampleTable, fromTsUs: number): PipelineState {
|
|
204
|
+
const samples = table.samples
|
|
205
|
+
const keyIdx = samples.length === 0 ? 0 : keyframeAtOrBefore(table, fromTsUs)
|
|
206
|
+
const startIdx = samples.length === 0 ? 0 : sampleAtOrBefore(table, fromTsUs)
|
|
207
|
+
return {
|
|
208
|
+
nextSampleIdx: keyIdx,
|
|
209
|
+
buffered: 0,
|
|
210
|
+
inFlightFrames: 0,
|
|
211
|
+
inFlightBatches: 0,
|
|
212
|
+
firstBatchTargetTsUs: samples.length === 0 ? 0 : samples[startIdx].tsUs,
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* Top up the decode-ahead pipeline: the batches to post right now, plus the
|
|
218
|
+
* state that results from posting them (the caller posts them and keeps the
|
|
219
|
+
* state; nothing is mutated in place).
|
|
220
|
+
*
|
|
221
|
+
* Loop conditions, in order, and why each one is there:
|
|
222
|
+
* - `inFlightBatches < maxInFlightBatches` — keep the worker fed so it never
|
|
223
|
+
* idles waiting for a main-thread round trip (SP1 §6.1).
|
|
224
|
+
* - end of source — nothing left to ask for.
|
|
225
|
+
* - `capacity = N - buffered - inFlightFrames` — the frame budget. Holding
|
|
226
|
+
* `buffered + inFlight ≤ N` is what keeps N meaning "at most N frames alive
|
|
227
|
+
* at once", which SP1 §8's memory sweep depends on.
|
|
228
|
+
* - `capacity < minBatch` while not starving — amortize the flush. "Starving"
|
|
229
|
+
* (nothing buffered, nothing coming) overrides it: any batch beats a stall.
|
|
230
|
+
*/
|
|
231
|
+
export function planNextBatches(
|
|
232
|
+
table: SampleTable,
|
|
233
|
+
state: PipelineState,
|
|
234
|
+
config: PipelineConfig,
|
|
235
|
+
): { batches: DecodeBatch[]; state: PipelineState } {
|
|
236
|
+
const samples = table.samples
|
|
237
|
+
const minBatch = config.minBatchFrames ?? minBatchFrames(config.decodeAheadFrames)
|
|
238
|
+
const batches: DecodeBatch[] = []
|
|
239
|
+
let next = { ...state }
|
|
240
|
+
|
|
241
|
+
while (next.inFlightBatches < config.maxInFlightBatches) {
|
|
242
|
+
if (next.nextSampleIdx >= samples.length) break
|
|
243
|
+
|
|
244
|
+
const capacity = config.decodeAheadFrames - next.buffered - next.inFlightFrames
|
|
245
|
+
if (capacity <= 0) break
|
|
246
|
+
const starving = next.buffered === 0 && next.inFlightFrames === 0
|
|
247
|
+
if (!starving && capacity < minBatch) break
|
|
248
|
+
|
|
249
|
+
const startIdx = next.nextSampleIdx
|
|
250
|
+
const endIdx = Math.min(startIdx + capacity, samples.length)
|
|
251
|
+
const count = endIdx - startIdx
|
|
252
|
+
if (count <= 0) break
|
|
253
|
+
|
|
254
|
+
batches.push({
|
|
255
|
+
startIdx,
|
|
256
|
+
endIdx,
|
|
257
|
+
count,
|
|
258
|
+
// Min cts, never samples[startIdx].tsUs — see `minCtsUs`. The floor is
|
|
259
|
+
// the session's real first wanted frame, which is what legitimately
|
|
260
|
+
// discards the keyframe→start pre-roll on the first batch only.
|
|
261
|
+
targetTsUs: Math.max(next.firstBatchTargetTsUs, minCtsUs(samples, startIdx, endIdx)),
|
|
262
|
+
})
|
|
263
|
+
|
|
264
|
+
next = {
|
|
265
|
+
...next,
|
|
266
|
+
nextSampleIdx: endIdx,
|
|
267
|
+
inFlightFrames: next.inFlightFrames + count,
|
|
268
|
+
inFlightBatches: next.inFlightBatches + 1,
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
return { batches, state: next }
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
/** A frame arrived from the worker and joined the main-side buffer. */
|
|
276
|
+
export function onFrameBuffered(state: PipelineState): PipelineState {
|
|
277
|
+
return {
|
|
278
|
+
...state,
|
|
279
|
+
buffered: state.buffered + 1,
|
|
280
|
+
inFlightFrames: Math.max(0, state.inFlightFrames - 1),
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/** `n` buffered frames left the buffer (painted, dropped as late, or closed on teardown). */
|
|
285
|
+
export function onFramesConsumed(state: PipelineState, n = 1): PipelineState {
|
|
286
|
+
return { ...state, buffered: Math.max(0, state.buffered - n) }
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/**
|
|
290
|
+
* A batch reported `done`. Reconciliation, not just a decrement: frames the
|
|
291
|
+
* worker dropped as pre-roll never arrive, so the per-frame decrement in
|
|
292
|
+
* `onFrameBuffered` under-counts and `inFlightFrames` would drift upward,
|
|
293
|
+
* permanently eating decode-ahead capacity until the pipeline wedged.
|
|
294
|
+
* `expected` is the batch's `count`; `decoded` is what the worker actually
|
|
295
|
+
* emitted.
|
|
296
|
+
*/
|
|
297
|
+
export function reconcileOnBatchDone(
|
|
298
|
+
state: PipelineState,
|
|
299
|
+
expected: number,
|
|
300
|
+
decoded: number,
|
|
301
|
+
): PipelineState {
|
|
302
|
+
const missing = Math.max(0, expected - decoded)
|
|
303
|
+
return {
|
|
304
|
+
...state,
|
|
305
|
+
inFlightBatches: Math.max(0, state.inFlightBatches - 1),
|
|
306
|
+
inFlightFrames: Math.max(0, state.inFlightFrames - missing),
|
|
307
|
+
}
|
|
308
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SP4 T7 — the debug HUD: fps / dropped / buffered / clock-kind, live,
|
|
3
|
+
* rendered only behind `engine.debugHud`.
|
|
4
|
+
*
|
|
5
|
+
* Precedent: `spikes/playback-engine/src/hud.ts`'s `mountPlaybackHud` — the
|
|
6
|
+
* same four numbers (painted fps over a rolling window, drops, decode-ahead
|
|
7
|
+
* buffer depth, which clock is driving), read the same way (a rolling
|
|
8
|
+
* window, never a single instantaneous sample). Deliberately NOT that file:
|
|
9
|
+
* no per-(source, hardwareAcceleration) bucket matrix, no JSON export
|
|
10
|
+
* button, no memory probe, no `window.__playbackHud` console handle — this
|
|
11
|
+
* is a production affordance for "is the engine keeping up right now", not
|
|
12
|
+
* the spike's benchmarking instrument. One engine, one bucket.
|
|
13
|
+
*
|
|
14
|
+
* Polls `Engine.stats()` (via the stable `getStats` the facade/hook exposes)
|
|
15
|
+
* on its own timer rather than subscribing to engine status: `EngineStatus`
|
|
16
|
+
* only republishes on a genuine transport/picture/clip/clock CHANGE
|
|
17
|
+
* (`scheduler.ts`'s `publish`), so fps/buffer — which move on every painted
|
|
18
|
+
* frame while playing — would sit stale between those events if this
|
|
19
|
+
* component keyed off status instead.
|
|
20
|
+
*/
|
|
21
|
+
import { useEffect, useState } from 'react'
|
|
22
|
+
import type { EngineStats } from './index'
|
|
23
|
+
|
|
24
|
+
/** 2Hz — fast enough to read as "live" without re-rendering on every paint. */
|
|
25
|
+
const POLL_MS = 500
|
|
26
|
+
|
|
27
|
+
interface DebugHudProps {
|
|
28
|
+
getStats: () => EngineStats | null
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function fmtNum(v: number | undefined, digits = 0): string {
|
|
32
|
+
return v === undefined ? 'n/a' : v.toFixed(digits)
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export default function DebugHud({ getStats }: DebugHudProps) {
|
|
36
|
+
const [stats, setStats] = useState<EngineStats | null>(() => getStats())
|
|
37
|
+
|
|
38
|
+
useEffect(() => {
|
|
39
|
+
const id = setInterval(() => setStats(getStats()), POLL_MS)
|
|
40
|
+
return () => clearInterval(id)
|
|
41
|
+
}, [getStats])
|
|
42
|
+
|
|
43
|
+
return (
|
|
44
|
+
<div
|
|
45
|
+
className="montaj-engine-debug-hud absolute bottom-2 left-2 select-none rounded bg-black/70 px-2 py-1 font-mono text-[10px] leading-tight text-white/80 pointer-events-none"
|
|
46
|
+
style={{ zIndex: 3 }}
|
|
47
|
+
>
|
|
48
|
+
<div>fps {fmtNum(stats?.fps, 1)}</div>
|
|
49
|
+
<div>dropped {fmtNum(stats?.dropped)}</div>
|
|
50
|
+
<div>buffered {fmtNum(stats?.buffered)}</div>
|
|
51
|
+
<div>clock {stats?.clock ?? 'n/a'}</div>
|
|
52
|
+
</div>
|
|
53
|
+
)
|
|
54
|
+
}
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SP4 T3 — the decode worker, as an inlined source string.
|
|
3
|
+
*
|
|
4
|
+
* Loaded via `URL.createObjectURL(new Blob([decodeWorkerSource], { type:
|
|
5
|
+
* 'text/javascript' }))` and `new Worker(url)` — a **classic** worker, so this
|
|
6
|
+
* string must be plain JS with no imports (plan decision 6: the Blob-URL form
|
|
7
|
+
* is the only one that needs nothing from a consumer's bundler; `?raw` is
|
|
8
|
+
* Vite-only and `new Worker(new URL(...))` can only be verified against Hub's
|
|
9
|
+
* webpack config, which does not live in this repo).
|
|
10
|
+
*
|
|
11
|
+
* Because it is a string, it cannot be type-checked, linted or unit-tested as
|
|
12
|
+
* a module — so it is kept **thin by architecture**. Everything that decides
|
|
13
|
+
* *what* to decode (batch boundaries, keyframe alignment, batch sizing, the
|
|
14
|
+
* min-cts pre-roll target, the epsilon value itself) lives main-side in
|
|
15
|
+
* `batch-planner.ts` under test. What is left here is the irreducible
|
|
16
|
+
* off-main-thread part:
|
|
17
|
+
*
|
|
18
|
+
* 1. a serial queue over exactly one `VideoDecoder` (every request awaits the
|
|
19
|
+
* previous one's `flush()`, so the `output` callback always belongs to the
|
|
20
|
+
* request marked `current`);
|
|
21
|
+
* 2. supersession by request id — **no `decoder.reset()` anywhere**. `reset()`
|
|
22
|
+
* drops the decoder to `unconfigured` and the next `decode()` throws
|
|
23
|
+
* `InvalidStateError`, so a superseded request is handled purely by never
|
|
24
|
+
* transferring its frames (they are closed here instead, immediately);
|
|
25
|
+
* 3. the pre-roll drop, using the epsilon handed over at `init` (SP1 §7.2 —
|
|
26
|
+
* see `PRE_ROLL_EPSILON_US` in `batch-planner.ts` for why 1µs);
|
|
27
|
+
* 4. the **total `done` contract**: every `decodeRange` posts exactly one
|
|
28
|
+
* `done`, whether it decoded, was skipped as stale before reaching the
|
|
29
|
+
* decoder, or threw. SP1 §7.4 is what happens without it — `frame-server`'s
|
|
30
|
+
* pending map orphans an entry and a seek promise never settles.
|
|
31
|
+
*
|
|
32
|
+
* Samples arrive by structured clone (never transfer): mp4box allocates a
|
|
33
|
+
* dedicated `Uint8Array` per sample, so a clone copies exactly that sample's
|
|
34
|
+
* bytes, whereas transferring would detach the buffer from the demuxed sample
|
|
35
|
+
* table and corrupt every later request that overlaps the same GOP. See
|
|
36
|
+
* `frame-server.ts`'s `postBatch`.
|
|
37
|
+
*
|
|
38
|
+
* `spikes/playback-engine/src/decode-worker.ts` is the precedent; the
|
|
39
|
+
* deliberate simplification is that `done` is posted from ONE place after the
|
|
40
|
+
* try/catch instead of once per exit path.
|
|
41
|
+
*/
|
|
42
|
+
export const decodeWorkerSource = `'use strict';
|
|
43
|
+
|
|
44
|
+
var decoder = null;
|
|
45
|
+
var queue = [];
|
|
46
|
+
var latestReqId = -1;
|
|
47
|
+
var processing = false;
|
|
48
|
+
var preRollEpsilonUs = 1;
|
|
49
|
+
|
|
50
|
+
// The request being fed to the decoder right now. At most one exists: drain()
|
|
51
|
+
// awaits each request's flush() before starting the next, so every output
|
|
52
|
+
// callback belongs to whichever request is current when it fires.
|
|
53
|
+
var current = null;
|
|
54
|
+
|
|
55
|
+
function post(msg, transfer) {
|
|
56
|
+
self.postMessage(msg, transfer || []);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function describe(err) {
|
|
60
|
+
return err && err.message ? err.message : String(err);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function onFrame(frame) {
|
|
64
|
+
if (!current || current.reqId < latestReqId) {
|
|
65
|
+
// Superseded while this request was still flushing. Close, never transfer.
|
|
66
|
+
frame.close();
|
|
67
|
+
if (current) current.dropped++;
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
if (frame.timestamp < current.targetTsUs - preRollEpsilonUs) {
|
|
71
|
+
// Pre-roll: decoded only to prime prediction from the keyframe.
|
|
72
|
+
frame.close();
|
|
73
|
+
current.dropped++;
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
current.decoded++;
|
|
77
|
+
post({ t: 'frame', frame: frame, reqId: current.reqId }, [frame]);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
async function runRange(req) {
|
|
81
|
+
current = { reqId: req.reqId, targetTsUs: req.targetTsUs, decoded: 0, dropped: 0 };
|
|
82
|
+
try {
|
|
83
|
+
if (!decoder) throw new Error('decodeRange before init');
|
|
84
|
+
for (var i = 0; i < req.samples.length; i++) {
|
|
85
|
+
var s = req.samples[i];
|
|
86
|
+
decoder.decode(new EncodedVideoChunk({
|
|
87
|
+
type: s.isKey ? 'key' : 'delta',
|
|
88
|
+
timestamp: s.tsUs,
|
|
89
|
+
duration: s.durUs,
|
|
90
|
+
data: s.data,
|
|
91
|
+
}));
|
|
92
|
+
}
|
|
93
|
+
await decoder.flush();
|
|
94
|
+
} catch (err) {
|
|
95
|
+
post({ t: 'error', message: 'decode: ' + describe(err), reqId: req.reqId });
|
|
96
|
+
}
|
|
97
|
+
// Unconditional: the total done contract has exactly one site.
|
|
98
|
+
post({ t: 'done', reqId: req.reqId, decoded: current.decoded, dropped: current.dropped });
|
|
99
|
+
current = null;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
async function drain() {
|
|
103
|
+
if (processing) return;
|
|
104
|
+
processing = true;
|
|
105
|
+
while (queue.length > 0) {
|
|
106
|
+
var req = queue.shift();
|
|
107
|
+
if (req.reqId < latestReqId) {
|
|
108
|
+
// Stale before decoding a byte - still owes its done.
|
|
109
|
+
post({ t: 'done', reqId: req.reqId, decoded: 0, dropped: 0 });
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
await runRange(req);
|
|
113
|
+
}
|
|
114
|
+
processing = false;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
self.onmessage = function (ev) {
|
|
118
|
+
var cmd = ev.data;
|
|
119
|
+
if (cmd.t === 'init') {
|
|
120
|
+
if (typeof cmd.preRollEpsilonUs === 'number') preRollEpsilonUs = cmd.preRollEpsilonUs;
|
|
121
|
+
try {
|
|
122
|
+
decoder = new VideoDecoder({
|
|
123
|
+
output: onFrame,
|
|
124
|
+
error: function (e) { post({ t: 'error', message: 'decoder: ' + describe(e) }); },
|
|
125
|
+
});
|
|
126
|
+
decoder.configure({
|
|
127
|
+
codec: cmd.codec,
|
|
128
|
+
description: cmd.description,
|
|
129
|
+
hardwareAcceleration: cmd.hardwareAcceleration,
|
|
130
|
+
});
|
|
131
|
+
} catch (err) {
|
|
132
|
+
post({ t: 'error', message: 'init: ' + describe(err) });
|
|
133
|
+
}
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
136
|
+
if (cmd.t === 'decodeRange') {
|
|
137
|
+
if (cmd.reqId > latestReqId) latestReqId = cmd.reqId;
|
|
138
|
+
queue.push(cmd);
|
|
139
|
+
drain();
|
|
140
|
+
}
|
|
141
|
+
};
|
|
142
|
+
`
|