@bycrux/editor 0.12.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -1
- package/src/ControlsInfoModal.tsx +251 -61
- package/src/__tests__/ControlsInfoModal.test.tsx +43 -0
- package/src/__tests__/adapter.test.ts +59 -1
- package/src/__tests__/schema-assignability.test.ts +93 -0
- package/src/__tests__/schema.test.ts +15 -0
- package/src/__tests__/video-adapter-contract.test.ts +128 -5
- package/src/carousel/AddElementMenu.tsx +10 -4
- package/src/carousel/CarouselEditor.tsx +46 -8
- package/src/carousel/CarouselRenderModal.tsx +15 -10
- package/src/carousel/OverlayPicker.tsx +10 -3
- package/src/carousel/SlidePropertyPanel.tsx +43 -21
- package/src/components/FilmstripScrubber.tsx +277 -0
- package/src/components/__tests__/FilmstripScrubber.test.tsx +216 -0
- package/src/engine/__tests__/audio-clock.test.ts +655 -0
- package/src/engine/__tests__/audio-worklet-source.test.ts +316 -0
- package/src/engine/__tests__/batch-planner.test.ts +298 -0
- package/src/engine/__tests__/decode-worker-source.test.ts +249 -0
- package/src/engine/__tests__/demux-ranged.test.ts +495 -0
- package/src/engine/__tests__/demux-truncated.test.ts +136 -0
- package/src/engine/__tests__/demux.test.ts +305 -0
- package/src/engine/__tests__/eligibility.test.ts +146 -0
- package/src/engine/__tests__/engine-recovery.test.ts +263 -0
- package/src/engine/__tests__/engine.test.ts +326 -0
- package/src/engine/__tests__/frame-server-ranged.test.ts +261 -0
- package/src/engine/__tests__/frame-server.test.ts +326 -0
- package/src/engine/__tests__/media-loader-ranged.test.ts +289 -0
- package/src/engine/__tests__/media-loader.test.ts +100 -0
- package/src/engine/__tests__/scheduler-crop.test.ts +353 -0
- package/src/engine/__tests__/scheduler.test.ts +1310 -0
- package/src/engine/__tests__/scrub-resolve.test.ts +96 -0
- package/src/engine/__tests__/scrub-source.test.ts +195 -0
- package/src/engine/__tests__/source-host.test.ts +455 -0
- package/src/engine/__tests__/time-stretch.test.ts +182 -0
- package/src/engine/audio-clock.ts +1179 -0
- package/src/engine/audio-worklet-source.ts +160 -0
- package/src/engine/batch-planner.ts +308 -0
- package/src/engine/debug-hud.tsx +54 -0
- package/src/engine/decode-worker-source.ts +142 -0
- package/src/engine/demux.ts +900 -0
- package/src/engine/eligibility.ts +140 -0
- package/src/engine/frame-server.ts +644 -0
- package/src/engine/index.ts +950 -0
- package/src/engine/media-loader.ts +380 -0
- package/src/engine/mp4box.d.ts +214 -0
- package/src/engine/scheduler.ts +1324 -0
- package/src/engine/scrub-resolve.ts +66 -0
- package/src/engine/scrub-source.ts +496 -0
- package/src/engine/time-stretch.ts +224 -0
- package/src/index.ts +75 -1
- package/src/preview/OverlayPreview.tsx +2 -24
- package/src/schema.ts +134 -2
- package/src/state/__tests__/use-project-sync.test.tsx +56 -0
- package/src/state/use-project-sync.ts +17 -0
- package/src/test-setup.ts +32 -0
- package/src/text/FontPicker.tsx +85 -51
- package/src/text/TextFormattingToolbar.tsx +6 -1
- package/src/text/__tests__/FontPicker.options.test.ts +43 -0
- package/src/text/__tests__/TextFormattingToolbar.test.tsx +4 -4
- package/src/theme.ts +108 -3
- package/src/types.ts +586 -22
- package/src/ui/Loader.tsx +64 -0
- package/src/ui/NumberField.tsx +254 -0
- package/src/ui/Slider.tsx +128 -0
- package/src/ui/Tooltip.tsx +118 -0
- package/src/ui/__tests__/NumberField.test.tsx +298 -0
- package/src/ui/__tests__/Slider.test.tsx +150 -0
- package/src/ui/__tests__/Tooltip.test.tsx +105 -0
- package/src/ui/__tests__/usePersistentState.test.tsx +112 -0
- package/src/ui/badge.tsx +12 -4
- package/src/ui/index.ts +6 -0
- package/src/ui/input.tsx +1 -1
- package/src/ui/select.tsx +1 -1
- package/src/ui/switch.tsx +12 -2
- package/src/ui/textarea.tsx +1 -1
- package/src/ui/usePersistentState.ts +63 -0
- package/src/video/AudioPolishModal.tsx +983 -0
- package/src/video/CaptionListPanel.test.tsx +944 -0
- package/src/video/CaptionListPanel.tsx +1106 -0
- package/src/video/CaptionRegenModal.tsx +37 -9
- package/src/video/CaptionSpecimen.tsx +160 -0
- package/src/video/CaptionStyleGallery.tsx +466 -0
- package/src/video/CommandPalette.tsx +165 -0
- package/src/video/ImageToneMenu.tsx +6 -2
- package/src/video/OverlayInspector.tsx +977 -0
- package/src/video/RenderModal.tsx +1035 -62
- package/src/video/VersionCompare.tsx +258 -0
- package/src/video/VersionPanel.tsx +117 -42
- package/src/video/VideoEditor.tsx +2202 -251
- package/src/video/__tests__/AudioPolishModal.test.tsx +825 -0
- package/src/video/__tests__/CaptionListPanel.font.test.tsx +397 -0
- package/src/video/__tests__/CaptionListPanel.generate.test.tsx +153 -0
- package/src/video/__tests__/CaptionRegenModal.test.tsx +29 -0
- package/src/video/__tests__/CaptionSpecimen.test.tsx +213 -0
- package/src/video/__tests__/CaptionStyleGallery.test.tsx +362 -0
- package/src/video/__tests__/CommandPalette.test.tsx +119 -0
- package/src/video/__tests__/OverlayInspector.test.tsx +1367 -0
- package/src/video/__tests__/RenderModal.exportControls.test.tsx +319 -0
- package/src/video/__tests__/RenderModal.options.test.tsx +562 -0
- package/src/video/__tests__/RenderModal.progress.test.ts +65 -0
- package/src/video/__tests__/VersionCompare.test.tsx +182 -0
- package/src/video/__tests__/VersionPanel.test.tsx +279 -0
- package/src/video/__tests__/VideoEditor.audioPolish.test.tsx +241 -0
- package/src/video/__tests__/VideoEditor.captionDelete.test.tsx +147 -0
- package/src/video/__tests__/VideoEditor.captionGesture.test.tsx +170 -0
- package/src/video/__tests__/VideoEditor.captionSeam.test.tsx +134 -0
- package/src/video/__tests__/VideoEditor.clipKeyframes.test.tsx +59 -0
- package/src/video/__tests__/VideoEditor.context.test.tsx +44 -0
- package/src/video/__tests__/VideoEditor.editFocus.test.tsx +55 -0
- package/src/video/__tests__/VideoEditor.keymap.test.tsx +725 -0
- package/src/video/__tests__/VideoEditor.layout.test.tsx +338 -0
- package/src/video/__tests__/VideoEditor.propertiesPanel.test.tsx +765 -0
- package/src/video/__tests__/VideoEditor.rippleDeleteCaptions.test.tsx +159 -0
- package/src/video/__tests__/VideoEditor.sourcePreview.test.tsx +122 -0
- package/src/video/__tests__/VideoEditor.test.tsx +403 -50
- package/src/video/__tests__/audioMagnet.test.ts +158 -0
- package/src/video/__tests__/audioPolish.test.ts +1202 -0
- package/src/video/__tests__/captionActiveWord.test.ts +107 -0
- package/src/video/__tests__/captionLanes.test.ts +262 -0
- package/src/video/__tests__/captionPositioning.test.tsx +8 -3
- package/src/video/__tests__/captionWordFloor.test.ts +228 -0
- package/src/video/__tests__/clipboard-ops.test.ts +430 -0
- package/src/video/__tests__/cuts.insert.test.ts +244 -0
- package/src/video/__tests__/cuts.test.ts +1213 -34
- package/src/video/__tests__/export-limits.test.ts +195 -0
- package/src/video/__tests__/hover-scrub.test.ts +163 -0
- package/src/video/__tests__/keyframeOps.canKeyframeProp.test.ts +61 -0
- package/src/video/__tests__/keyframeOps.test.ts +643 -0
- package/src/video/__tests__/keymap.test.tsx +171 -0
- package/src/video/__tests__/render-progress.test.tsx +20 -4
- package/src/video/__tests__/shuttle.test.ts +210 -0
- package/src/video/__tests__/source-preview.test.ts +60 -0
- package/src/video/__tests__/timecode.test.ts +77 -0
- package/src/video/__tests__/use-report-context.test.tsx +101 -0
- package/src/video/audioMagnet.ts +72 -0
- package/src/video/audioPolish.ts +774 -0
- package/src/video/captionActiveWord.ts +74 -0
- package/src/video/captionLanes.ts +202 -0
- package/src/video/captionRepair.ts +9 -5
- package/src/video/captionStyleDefaults.ts +100 -0
- package/src/video/captionWordFloor.ts +83 -0
- package/src/video/clipboard-ops.ts +377 -0
- package/src/video/cuts.ts +713 -37
- package/src/video/export-limits.ts +102 -0
- package/src/video/hover-scrub.ts +102 -0
- package/src/video/keyframeOps.ts +384 -0
- package/src/video/keymap.ts +146 -0
- package/src/video/panels/ClipPropertiesPanel.tsx +706 -0
- package/src/video/panels/LeftPanelTabs.tsx +187 -0
- package/src/video/panels/OverlayContentPanel.tsx +351 -0
- package/src/video/panels/TabNav.tsx +60 -0
- package/src/video/panels/__tests__/ClipPropertiesPanel.test.tsx +645 -0
- package/src/video/panels/__tests__/LeftPanelTabs.test.tsx +189 -0
- package/src/video/panels/__tests__/OverlayContentPanel.test.tsx +222 -0
- package/src/video/panels/__tests__/TabNav.test.tsx +71 -0
- package/src/video/preview/CaptionPreview.tsx +57 -69
- package/src/video/preview/EngineSurface.tsx +103 -0
- package/src/video/preview/OverlayItemsLayer.tsx +315 -103
- package/src/video/preview/PreviewPlayer.tsx +337 -56
- package/src/video/preview/SocialPreviewMenu.tsx +214 -0
- package/src/video/preview/SocialSafeZoneOverlay.tsx +478 -0
- package/src/video/preview/__tests__/CaptionPreview.fonts.test.tsx +97 -0
- package/src/video/preview/__tests__/EngineSurface.test.tsx +114 -0
- package/src/video/preview/__tests__/OverlayItemsLayer.edit.test.tsx +4 -2
- package/src/video/preview/__tests__/OverlayItemsLayer.keyframes.test.tsx +363 -0
- package/src/video/preview/__tests__/OverlayItemsLayer.selection.test.tsx +329 -0
- package/src/video/preview/__tests__/PreviewPlayer.engine.test.tsx +139 -0
- package/src/video/preview/__tests__/SocialPreviewMenu.test.tsx +121 -0
- package/src/video/preview/__tests__/SocialSafeZoneOverlay.test.tsx +165 -0
- package/src/video/preview/__tests__/captionDragState.test.ts +120 -1
- package/src/video/preview/__tests__/latencyCompensation.test.tsx +451 -0
- package/src/video/preview/__tests__/proxySupport.test.ts +75 -0
- package/src/video/preview/__tests__/transformStyle.test.ts +29 -1
- package/src/video/preview/__tests__/useDragOverlay.perAxis.test.ts +197 -0
- package/src/video/preview/__tests__/useEnginePlayback.test.tsx +530 -0
- package/src/video/preview/__tests__/useVideoPlayback.corpus.test.ts +313 -0
- package/src/video/preview/__tests__/useVideoPlayback.test.ts +38 -5
- package/src/video/preview/__tests__/useVideoPlayback.trackAudio.test.ts +278 -0
- package/src/video/preview/audio-context.ts +111 -0
- package/src/video/preview/captionDragState.ts +91 -1
- package/src/video/preview/proxySupport.ts +86 -0
- package/src/video/preview/transformStyle.ts +22 -12
- package/src/video/preview/useDragOverlay.ts +92 -18
- package/src/video/preview/useEnginePlayback.ts +625 -0
- package/src/video/preview/useVideoPlayback.ts +211 -167
- package/src/video/sdrCurves.ts +56 -0
- package/src/video/shuttle.ts +159 -0
- package/src/video/source-preview.ts +66 -0
- package/src/video/timecode.ts +59 -0
- package/src/video/timeline/EditableSegment.tsx +1 -1
- package/src/video/timeline/Scrubber.tsx +46 -174
- package/src/video/timeline/SpeedControl.tsx +95 -0
- package/src/video/timeline/Timeline.tsx +1061 -328
- package/src/video/timeline/TimelineContext.ts +17 -10
- package/src/video/timeline/TrackGutter.tsx +560 -0
- package/src/video/timeline/TrackSettingsPopover.tsx +228 -0
- package/src/video/timeline/VolumeControl.tsx +113 -0
- package/src/video/timeline/__tests__/Timeline.backgroundClick.test.tsx +110 -0
- package/src/video/timeline/__tests__/Timeline.crossfade.test.tsx +104 -0
- package/src/video/timeline/__tests__/Timeline.fadeCurveMenu.test.tsx +174 -0
- package/src/video/timeline/__tests__/Timeline.keyframeDelete.test.tsx +273 -0
- package/src/video/timeline/__tests__/Timeline.keyframeFollow.test.tsx +215 -0
- package/src/video/timeline/__tests__/Timeline.keyframeMenu.test.tsx +253 -0
- package/src/video/timeline/__tests__/Timeline.keymap.test.tsx +372 -0
- package/src/video/timeline/__tests__/Timeline.subcutRegen.test.tsx +351 -0
- package/src/video/timeline/__tests__/TrackGutter.test.tsx +372 -0
- package/src/video/timeline/__tests__/_canvasSelect.test.tsx +273 -0
- package/src/video/timeline/__tests__/_canvasSelect.ts +414 -0
- package/src/video/timeline/__tests__/dragdrop-math.test.ts +135 -0
- package/src/video/timeline/__tests__/effectiveItemAudio.test.ts +50 -0
- package/src/video/timeline/__tests__/enabledTrackItems.test.ts +164 -0
- package/src/video/timeline/__tests__/moveItemAcrossTracks.test.ts +363 -0
- package/src/video/timeline/__tests__/multiSelectOps.test.ts +447 -0
- package/src/video/timeline/__tests__/placement.test.ts +278 -0
- package/src/video/timeline/__tests__/resizeWindowedItem.test.ts +140 -0
- package/src/video/timeline/__tests__/timeline-model.test.ts +576 -0
- package/src/video/timeline/__tests__/visualItemLabel.test.ts +69 -0
- package/src/video/timeline/canvas/TimelineCanvas.tsx +1447 -0
- package/src/video/timeline/canvas/__tests__/TimelineCanvas.drop.test.tsx +439 -0
- package/src/video/timeline/canvas/__tests__/TimelineCanvas.edgeScroll.test.tsx +346 -0
- package/src/video/timeline/canvas/__tests__/TimelineCanvas.panefill.test.tsx +122 -0
- package/src/video/timeline/canvas/__tests__/TimelineCanvas.pendingDrops.test.tsx +316 -0
- package/src/video/timeline/canvas/__tests__/TimelineCanvas.pointer.test.tsx +606 -0
- package/src/video/timeline/canvas/__tests__/TimelineCanvas.test.tsx +407 -0
- package/src/video/timeline/canvas/__tests__/clip-bands.test.ts +77 -0
- package/src/video/timeline/canvas/__tests__/draw.test.ts +2198 -0
- package/src/video/timeline/canvas/__tests__/fade-curve.test.ts +187 -0
- package/src/video/timeline/canvas/__tests__/filmstrips.test.ts +561 -0
- package/src/video/timeline/canvas/__tests__/hit-test.test.ts +818 -0
- package/src/video/timeline/canvas/__tests__/pending-drop.test.ts +210 -0
- package/src/video/timeline/canvas/__tests__/pointer-machine.test.ts +3358 -0
- package/src/video/timeline/canvas/__tests__/snap.test.ts +257 -0
- package/src/video/timeline/canvas/__tests__/viewport.test.ts +399 -0
- package/src/video/timeline/canvas/__tests__/waveforms.test.ts +946 -0
- package/src/video/timeline/canvas/clip-bands.ts +56 -0
- package/src/video/timeline/canvas/draw.ts +2187 -0
- package/src/video/timeline/canvas/fade-curve.ts +111 -0
- package/src/video/timeline/canvas/filmstrips.ts +418 -0
- package/src/video/timeline/canvas/hit-test.ts +501 -0
- package/src/video/timeline/canvas/keyframe-strip.ts +73 -0
- package/src/video/timeline/canvas/pointer-machine.ts +1828 -0
- package/src/video/timeline/canvas/snap.ts +232 -0
- package/src/video/timeline/canvas/viewport.ts +457 -0
- package/src/video/timeline/canvas/waveforms.ts +664 -0
- package/src/video/timeline/makeCaptionEdit.ts +5 -1
- package/src/video/timeline/multiSelectOps.ts +214 -55
- package/src/video/timeline/placement.ts +282 -0
- package/src/video/timeline/timeline-model.ts +919 -0
- package/src/video/timeline/useItemDragDrop.ts +151 -178
- package/src/video/timeline/useTimelineZoom.ts +25 -60
- package/src/video/timeline/utils.ts +0 -12
- package/src/video/use-report-context.ts +75 -0
- package/src/video/preview/OverlayPropsModal.tsx +0 -292
- package/src/video/preview/__tests__/OverlayPropsModal.test.tsx +0 -32
- package/src/video/timeline/AudioTrackRow.tsx +0 -404
- package/src/video/timeline/AudioWaveformLayer.tsx +0 -117
- package/src/video/timeline/CaptionTrackRow.tsx +0 -235
- package/src/video/timeline/PlayheadLine.tsx +0 -18
- package/src/video/timeline/TranscriptModal.tsx +0 -70
- package/src/video/timeline/TranscriptPanel.tsx +0 -273
- package/src/video/timeline/VisualTrackRow.tsx +0 -300
- package/src/video/timeline/__tests__/CaptionTrackRow.test.tsx +0 -241
- package/src/video/timeline/__tests__/PlayheadLine.test.tsx +0 -60
- package/src/video/timeline/__tests__/TranscriptModal.test.tsx +0 -41
- package/src/video/timeline/__tests__/TranscriptPanel.test.tsx +0 -184
- package/src/video/timeline/__tests__/useItemDragDrop.test.ts +0 -72
|
@@ -0,0 +1,1179 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SP4 T4 — the master clock.
|
|
3
|
+
*
|
|
4
|
+
* Everything the engine paints is timed by this module. The clock is derived
|
|
5
|
+
* from how many audio frames an `AudioWorklet` has ACTUALLY rendered
|
|
6
|
+
* (`samplesConsumed / sampleRate + anchor`), not from decode progress and not
|
|
7
|
+
* from wall time — so the canvas painter (T5) tracks what is really coming out
|
|
8
|
+
* of the speakers. SP1 §6 measured that design at ±4ms A/V error (p50 −4.3ms,
|
|
9
|
+
* p95 −0.4ms) over 71-second windows, which is what earns it the job.
|
|
10
|
+
*
|
|
11
|
+
* `spikes/playback-engine/src/audio.ts` is the precedent. Ported patterns and
|
|
12
|
+
* their reasons:
|
|
13
|
+
*
|
|
14
|
+
* - **postMessage chunks, not `SharedArrayBuffer`.** SAB needs cross-origin
|
|
15
|
+
* isolation (COOP/COEP). The spike could not get it under Vite dev or a
|
|
16
|
+
* `file://` Electron load; the product cannot get it either, because Hub
|
|
17
|
+
* serves the editor and cannot satisfy isolation (plan decision 4). Not the
|
|
18
|
+
* simpler choice — the only viable one.
|
|
19
|
+
* - **`normalizeAudioCodec`.** mp4box reports Opus capitalized (`'Opus'`);
|
|
20
|
+
* WebCodecs requires `'opus'`, and `'Opus'.startsWith('opus')` is false.
|
|
21
|
+
* SP1 §7.5: without this, EVERY proxy silently falls back to the wall clock
|
|
22
|
+
* and the audio master clock never runs once.
|
|
23
|
+
* - **Silence through underruns.** An empty ring renders silence and the frame
|
|
24
|
+
* counter keeps advancing (see `audio-worklet-source.ts`). A clock that
|
|
25
|
+
* stalled on starvation would stall the painter with it.
|
|
26
|
+
* - **Wall-clock extrapolation between ~10Hz reports**, so a 60Hz rAF paint
|
|
27
|
+
* loop does not read the same clock value five or six times before it steps.
|
|
28
|
+
* - **`createMasterClock` never throws.** Canvas projects, muted clips,
|
|
29
|
+
* tracks with no decodable audio, browsers with no `AudioDecoder`, a worklet
|
|
30
|
+
* that fails to load, a decoder that rejects its config — every one of them
|
|
31
|
+
* resolves to a wall-clock `MasterClock` with a `reason`, never a rejected
|
|
32
|
+
* promise. The caller awaits this function; it does not guard it.
|
|
33
|
+
*
|
|
34
|
+
* Deliberate deviations from the spike, each with its reason at the code:
|
|
35
|
+
* 1. the clock reads out in PROJECT SECONDS, not raw container µs (the spike
|
|
36
|
+
* had one file and no timeline);
|
|
37
|
+
* 2. it has a transport (`play`/`pause`/`seek`) — the spike started once and
|
|
38
|
+
* ran to the end;
|
|
39
|
+
* 3. it runs on the page's SHARED `AudioContext` and never closes it;
|
|
40
|
+
* 4. PCM is volume-scaled at ring-enqueue time;
|
|
41
|
+
* 5. PCM is resampled when the shared context's rate is not the decoded rate;
|
|
42
|
+
* 6. extrapolation is bounded.
|
|
43
|
+
*/
|
|
44
|
+
import { sampleAtOrBefore, type ChunkSource } from './demux'
|
|
45
|
+
import { audioWorkletSource, RING_PROCESSOR_NAME } from './audio-worklet-source'
|
|
46
|
+
import { getSharedAudioContext } from '../video/preview/audio-context'
|
|
47
|
+
import { timeStretch } from './time-stretch'
|
|
48
|
+
|
|
49
|
+
// ── Tuning constants ────────────────────────────────────────────────────────
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* How much decoded-but-unrendered audio (seconds) the feeder keeps queued in
|
|
53
|
+
* the worklet. PCM is cheap — there is no GPU surface pool to exhaust the way
|
|
54
|
+
* `VideoFrame` has — so this needs none of the careful N-frame budgeting
|
|
55
|
+
* `batch-planner.ts` does for video; it is just enough slack to ride out a slow
|
|
56
|
+
* decode tick without underrunning. Spike value, unchanged.
|
|
57
|
+
*/
|
|
58
|
+
export const RING_SECONDS = 2
|
|
59
|
+
|
|
60
|
+
/** Encoded audio packets fed to the decoder per `decode()`+`flush()` batch. Spike value. */
|
|
61
|
+
export const AUDIO_BATCH_PACKETS = 50
|
|
62
|
+
|
|
63
|
+
/** Worklet report cadence (~10Hz). Injected into the worklet via `processorOptions`. */
|
|
64
|
+
export const REPORT_INTERVAL_S = 0.1
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Ceiling on wall-clock extrapolation between reports.
|
|
68
|
+
*
|
|
69
|
+
* Extrapolation assumes nominal playback continues in the gap since the last
|
|
70
|
+
* report — true within one ~100ms interval. It stops being true when the
|
|
71
|
+
* AudioContext is SUSPENDED: `process()` is never called, no report ever
|
|
72
|
+
* arrives, and an unbounded extrapolation would run the playhead away for as
|
|
73
|
+
* long as the page stays suspended and then snap it violently backwards when
|
|
74
|
+
* audio finally resumes. Bounding it makes a suspended context present as the
|
|
75
|
+
* thing it actually is — a stalled clock, hence a stalled painter — which is
|
|
76
|
+
* precisely the hazard `resumeAudioContextFromGesture` exists to cure (T6
|
|
77
|
+
* owns the gesture anchors). 3× the report interval: generous for a late
|
|
78
|
+
* report, decisive for a dead one.
|
|
79
|
+
*/
|
|
80
|
+
export const MAX_EXTRAPOLATION_MS = REPORT_INTERVAL_S * 1000 * 3
|
|
81
|
+
|
|
82
|
+
/** Fallback assumptions when a container omits its audio parameters. Opus is 48kHz by codec definition. */
|
|
83
|
+
const DEFAULT_SAMPLE_RATE = 48000
|
|
84
|
+
const DEFAULT_CHANNELS = 2
|
|
85
|
+
|
|
86
|
+
// ── Public surface ──────────────────────────────────────────────────────────
|
|
87
|
+
|
|
88
|
+
export type ClockKind = 'audio' | 'fallback'
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* How one clip's audio maps onto the project timeline.
|
|
92
|
+
*
|
|
93
|
+
* `inPoint` is the in-point **in the loaded file's own timeline**, i.e.
|
|
94
|
+
* `sourceWindow(clip, 'preview').inPoint` — NOT the raw `clip.inPoint`. For the
|
|
95
|
+
* SP3 proxy (and for `nobg_preview_src`) the two are equal because those cover
|
|
96
|
+
* the full source; for a `normalizedSrc` window cache they are not, and using
|
|
97
|
+
* the raw field would land every seek at the wrong place. This is the same
|
|
98
|
+
* value `useVideoPlayback.ts` feeds its `<video>` elements through
|
|
99
|
+
* `effectiveInPoint`, so both paths answer "where in this file does the clip
|
|
100
|
+
* start" identically.
|
|
101
|
+
*
|
|
102
|
+
* `firstPresentationTsUs` is the AUDIO track's origin (`ChunkSource
|
|
103
|
+
* .firstPresentationTsUs`), not the video track's. Media timestamps need not
|
|
104
|
+
* start at 0 and the two tracks need not share an origin; each track maps
|
|
105
|
+
* through its own, which is the same per-track convention the spike used and
|
|
106
|
+
* the same one `frame-server.ts` applies to video.
|
|
107
|
+
*/
|
|
108
|
+
export interface ClipTimebase {
|
|
109
|
+
/** Project-time seconds at which the clip starts on the timeline (`item.start`). */
|
|
110
|
+
start: number
|
|
111
|
+
/** In-point in the LOADED file's timeline, seconds (`sourceWindow(item, 'preview').inPoint`). */
|
|
112
|
+
inPoint: number
|
|
113
|
+
/** The audio track's t=0 origin, container µs (`ChunkSource.firstPresentationTsUs`). */
|
|
114
|
+
firstPresentationTsUs: number
|
|
115
|
+
/**
|
|
116
|
+
* Per-clip playback speed S (`item.speed`), default 1. Scales the source↔
|
|
117
|
+
* timeline mapping ONLY — the clip shows S× its source content per project-
|
|
118
|
+
* second — and does NOT change how fast project time runs (that is the
|
|
119
|
+
* transport rate, `setTransportRate`). Absent ⇒ 1, i.e. bit-identical to the
|
|
120
|
+
* pre-speed mapping.
|
|
121
|
+
*/
|
|
122
|
+
speed?: number
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
export interface MasterClockStats {
|
|
126
|
+
kind: ClockKind
|
|
127
|
+
/** Set only when `kind === 'fallback'` — why audio isn't driving, for the T7 HUD. */
|
|
128
|
+
reason?: string
|
|
129
|
+
playing: boolean
|
|
130
|
+
/** Output frames rendered since the last reset (audio kind only; 0 on the fallback). */
|
|
131
|
+
samplesConsumed: number
|
|
132
|
+
/** Of those, how many were silence because the ring was empty. A rising number means decode is losing. */
|
|
133
|
+
underrunFrames: number
|
|
134
|
+
/** Decoded frames sitting in the ring, un-rendered. */
|
|
135
|
+
queuedFrames: number
|
|
136
|
+
/** `queuedFrames` in seconds — the real decode-ahead headroom. */
|
|
137
|
+
queuedSeconds: number
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* The engine's one clock. T5's scheduler reads `now()` on every rAF tick and
|
|
142
|
+
* drives the transport through `play`/`pause`/`seek`; T6 bridges `now()` to the
|
|
143
|
+
* editor's playhead. Both kinds ('audio' and 'fallback') implement the whole
|
|
144
|
+
* surface identically — a caller never branches on `kind` for behavior, only
|
|
145
|
+
* for display.
|
|
146
|
+
*
|
|
147
|
+
* The clock is bound to ONE clip's audio for its whole life. At a clip
|
|
148
|
+
* boundary the scheduler `dispose()`s it and awaits a new one anchored to the
|
|
149
|
+
* next clip — the same "never reconfigure a live decoder" rule
|
|
150
|
+
* `frame-server.ts` follows for video. Because the readout is PROJECT time and
|
|
151
|
+
* the new clock is seeked to the boundary's project time before it plays, the
|
|
152
|
+
* swap is continuous in the only units anyone reads.
|
|
153
|
+
*/
|
|
154
|
+
export interface MasterClock {
|
|
155
|
+
readonly kind: ClockKind
|
|
156
|
+
/** Set only when `kind === 'fallback'`. */
|
|
157
|
+
readonly reason?: string
|
|
158
|
+
readonly playing: boolean
|
|
159
|
+
/** Current playhead, in PROJECT SECONDS. */
|
|
160
|
+
now(): number
|
|
161
|
+
/** Start (or resume from wherever `now()` currently reads). Idempotent. */
|
|
162
|
+
play(): void
|
|
163
|
+
/** Freeze the playhead where it reads right now and silence the ring. Idempotent. */
|
|
164
|
+
pause(): void
|
|
165
|
+
/**
|
|
166
|
+
* Jump to a project time. Coherent whether playing or paused; playing
|
|
167
|
+
* survives the jump.
|
|
168
|
+
*
|
|
169
|
+
* `mediaS` is an OPTIONAL override for where in the loaded file's own
|
|
170
|
+
* timeline to restart decoding from, in seconds — the loop-wrap case. A
|
|
171
|
+
* project-time seek alone always maps `projectS` through this clip's
|
|
172
|
+
* timebase (`mediaTsUsForProjectTime`), which is correct for a boundary
|
|
173
|
+
* swap or a scrub but wrong for a wrap: the picture's media position resets
|
|
174
|
+
* to the loop window's start while `projectS` keeps climbing, and those two
|
|
175
|
+
* disagree by exactly one loop's worth of time. Passing the frame server's
|
|
176
|
+
* own wrapped `mediaS` keeps the two tracks anchored to the same instant.
|
|
177
|
+
* Omitted, the target is derived from `projectS` as before.
|
|
178
|
+
*/
|
|
179
|
+
seek(projectS: number, mediaS?: number): void
|
|
180
|
+
/**
|
|
181
|
+
* Update the clip's audio level. Takes effect IMMEDIATELY: the level rides a
|
|
182
|
+
* per-session output gain node, so it re-levels everything already in the ring
|
|
183
|
+
* (up to `RING_SECONDS` of decode-ahead) the instant it is set, not only audio
|
|
184
|
+
* enqueued afterward. Applied via a short (~15ms) ramp to avoid a zipper click
|
|
185
|
+
* on a fast drag. No-op on the fallback clock.
|
|
186
|
+
*/
|
|
187
|
+
setVolume(volume: number): void
|
|
188
|
+
/**
|
|
189
|
+
* Set the live transport rate R (default 1): project time thereafter advances
|
|
190
|
+
* R× wall-time. Continuous — the playhead does not jump across the change (a
|
|
191
|
+
* soft re-anchor freezes `now()` and re-bases the ramp). Independent of
|
|
192
|
+
* per-clip `speed`, which lives in the timebase and never changes how fast
|
|
193
|
+
* project time runs. Persists across `seek`/`play`/`pause`; a re-anchor
|
|
194
|
+
* (`restartAt`) re-bases its accounting without disturbing R.
|
|
195
|
+
*/
|
|
196
|
+
setTransportRate(rate: number): void
|
|
197
|
+
stats(): MasterClockStats
|
|
198
|
+
/** Stop the decoder and detach the worklet node. NEVER closes the shared AudioContext. */
|
|
199
|
+
dispose(): void
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
export interface MasterClockOptions {
|
|
203
|
+
/** The active clip's demuxed audio track, or `null` (canvas project, video with no audio track). */
|
|
204
|
+
audio: ChunkSource | null
|
|
205
|
+
/** How this clip's audio maps onto the project timeline. */
|
|
206
|
+
timebase: ClipTimebase
|
|
207
|
+
/** Project-time seconds the clock starts at (paused; call `play()` to run). */
|
|
208
|
+
startProjectS: number
|
|
209
|
+
/**
|
|
210
|
+
* The clip's `volume` (0–2; >1 amplifies). Applied by scaling PCM at
|
|
211
|
+
* ring-enqueue time — see `scaleAndInterleavePlanes`.
|
|
212
|
+
*/
|
|
213
|
+
volume?: number
|
|
214
|
+
/**
|
|
215
|
+
* The clip's `muted`. Muted clips run on the wall clock: there is nothing to
|
|
216
|
+
* sync to, and building an audio graph to render silence would be waste.
|
|
217
|
+
* Consequence for the caller: `muted` is a CONSTRUCTION-time decision, not a
|
|
218
|
+
* live one — unmuting mid-session means disposing this clock and awaiting a
|
|
219
|
+
* new one (unlike `volume`, which `setVolume` handles in place).
|
|
220
|
+
*/
|
|
221
|
+
muted?: boolean
|
|
222
|
+
/** Decoder/worklet failures, for the caller's log. Failures never throw and never stop the clock. */
|
|
223
|
+
onError?: (message: string) => void
|
|
224
|
+
/** Test seam (`frame-server.ts`'s `spawnWorker` is the precedent). Defaults to `performance.now`. */
|
|
225
|
+
nowMs?: () => number
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
// ── Pure logic (unit-tested; everything below the fold builds on it) ─────────
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* mp4box.js does not normalize its codec strings. Notably Opus comes back
|
|
232
|
+
* capitalized (`'Opus'`), and `AudioDecoder.configure()` rejects that outright
|
|
233
|
+
* — as does any straight comparison against `'opus'`. Trust mp4box for the
|
|
234
|
+
* codec family, never for its casing. `'mp4a'` (a bare AAC family string with
|
|
235
|
+
* no profile) is expanded to AAC-LC, the only form `configure()` accepts.
|
|
236
|
+
*
|
|
237
|
+
* SP1 §7.5 is the bug this function is: without it every H.264+Opus proxy fell
|
|
238
|
+
* silently onto the wall clock and the audio master clock never ran.
|
|
239
|
+
*/
|
|
240
|
+
export function normalizeAudioCodec(codec: string): string {
|
|
241
|
+
const lower = codec.toLowerCase()
|
|
242
|
+
return lower === 'mp4a' ? 'mp4a.40.2' : lower
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* Media timestamp (container µs, this track's own axis) → project seconds.
|
|
247
|
+
*
|
|
248
|
+
* `(mediaTsUs - firstPresentationTsUs) / 1e6` is the time in the LOADED FILE's
|
|
249
|
+
* timeline — exactly what a `<video>`'s `currentTime` reads on the legacy path.
|
|
250
|
+
* The rest reproduces `useVideoPlayback.ts`'s mapping verbatim:
|
|
251
|
+
* `projectT = clip.start + (video.currentTime - effectiveInPoint(clip))`.
|
|
252
|
+
*
|
|
253
|
+
* (The legacy site also adds `loopOffsetRef.current`. Loop is transport, not
|
|
254
|
+
* timebase: a looping clip's wrap is a `seek()` on this clock, owned by T5.)
|
|
255
|
+
*
|
|
256
|
+
* At per-clip speed S≠1 the clip's own time runs S× faster than project time,
|
|
257
|
+
* so the in-clip source offset is DIVIDED by S (`(mediaS − inPoint) / speed`).
|
|
258
|
+
* S=1 (the default, and an absent `timebase.speed`) is the verbatim legacy
|
|
259
|
+
* mapping.
|
|
260
|
+
*/
|
|
261
|
+
export function projectTimeForMediaTsUs(timebase: ClipTimebase, mediaTsUs: number): number {
|
|
262
|
+
const speed = timebase.speed ?? 1
|
|
263
|
+
const mediaS = (mediaTsUs - timebase.firstPresentationTsUs) / 1_000_000
|
|
264
|
+
return timebase.start + (mediaS - timebase.inPoint) / speed
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/**
|
|
268
|
+
* Project seconds → media timestamp (container µs). Exact inverse of
|
|
269
|
+
* `projectTimeForMediaTsUs`: the in-clip project offset is MULTIPLIED by S on
|
|
270
|
+
* the way back to source time (`(projectS − start)·speed + inPoint`).
|
|
271
|
+
*/
|
|
272
|
+
export function mediaTsUsForProjectTime(timebase: ClipTimebase, projectS: number): number {
|
|
273
|
+
const speed = timebase.speed ?? 1
|
|
274
|
+
const mediaS = (projectS - timebase.start) * speed + timebase.inPoint
|
|
275
|
+
return timebase.firstPresentationTsUs + mediaS * 1_000_000
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* Smooth the worklet's ~10Hz sample count into a continuous reading.
|
|
280
|
+
*
|
|
281
|
+
* Assumes nominal playback continued since the last report — true inside one
|
|
282
|
+
* report interval, false when the context is suspended, which is why
|
|
283
|
+
* `elapsedMs` is clamped to `maxElapsedMs` (see `MAX_EXTRAPOLATION_MS`).
|
|
284
|
+
* Negative elapsed (a clock that went backwards) clamps to 0.
|
|
285
|
+
*/
|
|
286
|
+
export function extrapolateSamples(
|
|
287
|
+
lastSamplesConsumed: number,
|
|
288
|
+
elapsedMs: number,
|
|
289
|
+
sampleRate: number,
|
|
290
|
+
maxElapsedMs: number = MAX_EXTRAPOLATION_MS,
|
|
291
|
+
): number {
|
|
292
|
+
const bounded = Math.min(Math.max(elapsedMs, 0), maxElapsedMs)
|
|
293
|
+
return lastSamplesConsumed + (bounded / 1000) * sampleRate
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/**
|
|
297
|
+
* Rendered output frames → project seconds. `anchorProjectS` is the project
|
|
298
|
+
* time the ring's frame 0 was authored to play at, set on every reset.
|
|
299
|
+
*
|
|
300
|
+
* `rate` is the transport rate R (default 1): project time advances R× per
|
|
301
|
+
* rendered second. The caller passes `samples` already RE-BASED to the last
|
|
302
|
+
* rate change (`projected − rateAnchorSamples`), so a mid-playback rate change
|
|
303
|
+
* ramps from the frozen anchor rather than re-scaling all history. R=1 is the
|
|
304
|
+
* pre-rate reading.
|
|
305
|
+
*/
|
|
306
|
+
export function projectTimeForSamples(
|
|
307
|
+
anchorProjectS: number,
|
|
308
|
+
samples: number,
|
|
309
|
+
sampleRate: number,
|
|
310
|
+
rate: number = 1,
|
|
311
|
+
): number {
|
|
312
|
+
return anchorProjectS + (rate * samples) / sampleRate
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* Interleave per-channel PCM planes into the single `[L0,R0,L1,R1,…]` buffer
|
|
317
|
+
* the ring speaks, scaling by `volume` on the way through.
|
|
318
|
+
*
|
|
319
|
+
* **Volume no longer rides here.** `createAudioClock` calls this with
|
|
320
|
+
* `volume === 1` and applies the clip's real level on a per-session output
|
|
321
|
+
* `GainNode` instead (see the node/gain/destination chain there). The original
|
|
322
|
+
* reason to scale at enqueue — that a single ring shared across clip boundaries
|
|
323
|
+
* would let a node-level gain re-level the previous clip's tail — does not hold:
|
|
324
|
+
* each clip session owns its own ring and its own gain node, so a live gain
|
|
325
|
+
* change only ever touches that clip. Moving volume to the node is what makes a
|
|
326
|
+
* `setVolume` audible immediately (it re-levels the buffered decode-ahead)
|
|
327
|
+
* rather than lagging by up to `RING_SECONDS`, and it matches the legacy player,
|
|
328
|
+
* which already routes each clip through its own GainNode. The `volume`
|
|
329
|
+
* parameter is retained for the pure-function tests and any caller that wants
|
|
330
|
+
* pre-scaled PCM.
|
|
331
|
+
*
|
|
332
|
+
* **No clamping, deliberately.** `volume` above 1.0 produces samples outside
|
|
333
|
+
* [-1, 1] and they are left that way — identical to what a GainNode emits for
|
|
334
|
+
* the same clip. `AudioDestinationNode` does the clipping, so amplified clips
|
|
335
|
+
* stay as loud as the user asked; clamping here would quiet them instead.
|
|
336
|
+
*/
|
|
337
|
+
export function scaleAndInterleavePlanes(
|
|
338
|
+
planes: readonly Float32Array[],
|
|
339
|
+
frameCount: number,
|
|
340
|
+
volume: number,
|
|
341
|
+
): Float32Array {
|
|
342
|
+
const channels = planes.length
|
|
343
|
+
const out = new Float32Array(frameCount * channels)
|
|
344
|
+
for (let ch = 0; ch < channels; ch++) {
|
|
345
|
+
const plane = planes[ch]
|
|
346
|
+
for (let i = 0; i < frameCount; i++) {
|
|
347
|
+
out[i * channels + ch] = plane[i] * volume
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
return out
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
/**
|
|
354
|
+
* Linear-interpolating resample of an interleaved buffer, carrying the
|
|
355
|
+
* fractional read position across chunk boundaries via `phase`.
|
|
356
|
+
*
|
|
357
|
+
* **Why this exists at all** — a deviation from the spike, forced by a product
|
|
358
|
+
* constraint the spike did not have. The spike built its own
|
|
359
|
+
* `new AudioContext({ sampleRate: 48000 })` and so could assume decoded rate ==
|
|
360
|
+
* render rate; it only warned when they disagreed. The engine is required to
|
|
361
|
+
* run on the page's SHARED context (plan decision 4), whose rate is whatever
|
|
362
|
+
* the output device reports — 44100 on plenty of Macs. Opus always decodes at
|
|
363
|
+
* 48000 by codec definition, so on such a device an unresampled feed plays
|
|
364
|
+
* 8.8% fast/sharp AND drifts ~0.9s of A/V error per 10 seconds, since the clock
|
|
365
|
+
* counts render frames while the content advances at the decoded rate. Warning
|
|
366
|
+
* and continuing (the spike's choice) would ship that; falling back to the wall
|
|
367
|
+
* clock would silence clip audio outright for those users. Resampling keeps
|
|
368
|
+
* both the pitch and the clock honest.
|
|
369
|
+
*
|
|
370
|
+
* `ratio` is `srcRate / dstRate` — input frames consumed per output frame. The
|
|
371
|
+
* identity case (`ratio === 1`, no carried phase) returns the input untouched,
|
|
372
|
+
* so the overwhelmingly common 48kHz-context path costs nothing.
|
|
373
|
+
*
|
|
374
|
+
* The final output frame of a chunk interpolates against the chunk's own last
|
|
375
|
+
* frame rather than the next chunk's first (which has not been decoded yet):
|
|
376
|
+
* a sub-sample error on one frame per ~10ms packet, inaudible, and the carried
|
|
377
|
+
* `phase` keeps the *timing* exact regardless.
|
|
378
|
+
*/
|
|
379
|
+
export function resampleInterleaved(
|
|
380
|
+
input: Float32Array,
|
|
381
|
+
channels: number,
|
|
382
|
+
ratio: number,
|
|
383
|
+
phase: number,
|
|
384
|
+
): { pcm: Float32Array; phase: number } {
|
|
385
|
+
if (ratio === 1 && phase === 0) return { pcm: input, phase: 0 }
|
|
386
|
+
const inFrames = Math.floor(input.length / channels)
|
|
387
|
+
if (inFrames === 0) return { pcm: new Float32Array(0), phase }
|
|
388
|
+
|
|
389
|
+
const outFrames = Math.max(0, Math.ceil((inFrames - phase) / ratio))
|
|
390
|
+
const out = new Float32Array(outFrames * channels)
|
|
391
|
+
for (let i = 0; i < outFrames; i++) {
|
|
392
|
+
const pos = phase + i * ratio
|
|
393
|
+
const i0 = Math.min(Math.floor(pos), inFrames - 1)
|
|
394
|
+
const i1 = Math.min(i0 + 1, inFrames - 1)
|
|
395
|
+
const frac = pos - i0
|
|
396
|
+
for (let ch = 0; ch < channels; ch++) {
|
|
397
|
+
const a = input[i0 * channels + ch]
|
|
398
|
+
const b = input[i1 * channels + ch]
|
|
399
|
+
out[i * channels + ch] = a + (b - a) * frac
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
return { pcm: out, phase: phase + outFrames * ratio - inFrames }
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
// ── Streaming time-stretch (wraps the batch WSOLA primitive) ─────────────────
|
|
406
|
+
|
|
407
|
+
/**
|
|
408
|
+
* New input frames (per channel) the streamer consumes per WSOLA batch. ~43ms at
|
|
409
|
+
* 48kHz — many decoded Opus packets (~960 frames each), so the batch primitive
|
|
410
|
+
* (`timeStretch`, whose analysis frame is 1024) sees far more than one frame and
|
|
411
|
+
* its similarity search has room to lock. Larger = fewer block joins but more
|
|
412
|
+
* latency before the first stretched audio appears.
|
|
413
|
+
*/
|
|
414
|
+
export const STRETCH_STEP_FRAMES = 2048
|
|
415
|
+
|
|
416
|
+
/**
|
|
417
|
+
* Input frames (per channel) carried from the tail of one block to the head of
|
|
418
|
+
* the next. The next block re-renders this shared content, and the two
|
|
419
|
+
* independent WSOLA renderings of it are crossfaded — that overlap-add is what
|
|
420
|
+
* hides the phase discontinuity between otherwise-independent batches. One full
|
|
421
|
+
* analysis frame (1024) of lead-in.
|
|
422
|
+
*/
|
|
423
|
+
export const STRETCH_CONTEXT_FRAMES = 1024
|
|
424
|
+
|
|
425
|
+
/**
|
|
426
|
+
* A streaming wrapper over the batch `timeStretch`.
|
|
427
|
+
*
|
|
428
|
+
* `timeStretch` is a whole-buffer WSOLA and the decoder hands us ~20ms packets —
|
|
429
|
+
* smaller than one 1024-sample analysis frame — so it cannot be called
|
|
430
|
+
* per-packet. This accumulates volume-scaled PCM across packets and stretches in
|
|
431
|
+
* `STRETCH_STEP_FRAMES` blocks, carrying `STRETCH_CONTEXT_FRAMES` of INPUT
|
|
432
|
+
* overlap between blocks and crossfading the two renderings of that overlap on
|
|
433
|
+
* the OUTPUT side (the "synthesis overlap"). The seam it hides is the price of
|
|
434
|
+
* wrapping a batch primitive rather than a natively-streaming one; the join is
|
|
435
|
+
* click-free at preview quality, not sample-exact — see `time-stretch.ts`.
|
|
436
|
+
*/
|
|
437
|
+
export interface StreamStretch {
|
|
438
|
+
/**
|
|
439
|
+
* Stretch a chunk of interleaved PCM, returning whatever whole-block output is
|
|
440
|
+
* ready now (interleaved, same channel count) — possibly EMPTY while the first
|
|
441
|
+
* block is still filling. At factor 1 it is a zero-copy identity (the input
|
|
442
|
+
* buffer itself is returned, no buffering, no added latency), so it agrees with
|
|
443
|
+
* the caller's own strict bypass.
|
|
444
|
+
*/
|
|
445
|
+
push(input: Float32Array): Float32Array
|
|
446
|
+
/** Adopt a new factor. Drops the cross-block carry (a one-block seam); keeps pending input. */
|
|
447
|
+
setFactor(factor: number): void
|
|
448
|
+
/** Clear all buffered input and cross-block carry — the ring-reset twin. */
|
|
449
|
+
reset(): void
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
/** Concatenate two interleaved buffers (returns the non-empty one directly when possible). */
|
|
453
|
+
function concatPcm(a: Float32Array, b: Float32Array): Float32Array {
|
|
454
|
+
if (a.length === 0) return b
|
|
455
|
+
if (b.length === 0) return a
|
|
456
|
+
const out = new Float32Array(a.length + b.length)
|
|
457
|
+
out.set(a, 0)
|
|
458
|
+
out.set(b, a.length)
|
|
459
|
+
return out
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
/**
|
|
463
|
+
* Linear crossfade of two equal-length interleaved buffers: `a` fades out while
|
|
464
|
+
* `b` fades in. Both are WSOLA renderings of the SAME content, so a linear blend
|
|
465
|
+
* (not equal-power) is correct — the two are already near-identical and in phase.
|
|
466
|
+
*/
|
|
467
|
+
function crossfadePcm(a: Float32Array, b: Float32Array, channels: number): Float32Array {
|
|
468
|
+
const frames = Math.floor(Math.min(a.length, b.length) / channels)
|
|
469
|
+
const out = new Float32Array(frames * channels)
|
|
470
|
+
for (let i = 0; i < frames; i++) {
|
|
471
|
+
const t = frames > 1 ? i / (frames - 1) : 1
|
|
472
|
+
for (let ch = 0; ch < channels; ch++) {
|
|
473
|
+
const idx = i * channels + ch
|
|
474
|
+
out[idx] = a[idx] * (1 - t) + b[idx] * t
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
return out
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
export function createStreamStretch(channels: number, factor: number): StreamStretch {
|
|
481
|
+
const step = STRETCH_STEP_FRAMES * channels
|
|
482
|
+
const context = STRETCH_CONTEXT_FRAMES * channels
|
|
483
|
+
let f = factor
|
|
484
|
+
/** Accumulated, not-yet-consumed input (interleaved). */
|
|
485
|
+
let pending: Float32Array = new Float32Array(0)
|
|
486
|
+
/** Last CONTEXT frames of the previous block's input — the next block's lead-in. */
|
|
487
|
+
let tailIn: Float32Array = new Float32Array(0)
|
|
488
|
+
/** Held output tail (the synthesis overlap) awaiting crossfade with the next block's head. */
|
|
489
|
+
let heldOut: Float32Array | null = null
|
|
490
|
+
|
|
491
|
+
const push = (input: Float32Array): Float32Array => {
|
|
492
|
+
if (f === 1) return input // strict identity — no buffering, no copy
|
|
493
|
+
if (input.length > 0) pending = concatPcm(pending, input)
|
|
494
|
+
// Held-frame count: the output span the carried input context maps to. Same
|
|
495
|
+
// value for what we HOLD and what we crossfade, so the two always align.
|
|
496
|
+
const hold = Math.round(STRETCH_CONTEXT_FRAMES * f) * channels
|
|
497
|
+
let emitted: Float32Array = new Float32Array(0)
|
|
498
|
+
|
|
499
|
+
while (pending.length >= step) {
|
|
500
|
+
const newInput = pending.subarray(0, step)
|
|
501
|
+
const feed = concatPcm(tailIn, newInput)
|
|
502
|
+
const out = timeStretch(feed, channels, f)
|
|
503
|
+
// Keep hold within half the block so `mid` below is never negative.
|
|
504
|
+
const h = Math.min(hold, Math.floor(out.length / channels / 2) * channels)
|
|
505
|
+
|
|
506
|
+
let blockOut: Float32Array
|
|
507
|
+
if (heldOut && h > 0) {
|
|
508
|
+
// out[0..h) and heldOut are two renderings of the carried context; blend
|
|
509
|
+
// them, then take the interior, holding a fresh tail for the next block.
|
|
510
|
+
const head = crossfadePcm(heldOut, out.subarray(0, h), channels)
|
|
511
|
+
const mid = out.subarray(h, out.length - h)
|
|
512
|
+
blockOut = concatPcm(head, mid)
|
|
513
|
+
} else {
|
|
514
|
+
// First block after a reset/factor change: nothing to blend with yet.
|
|
515
|
+
blockOut = out.subarray(0, Math.max(0, out.length - h))
|
|
516
|
+
}
|
|
517
|
+
heldOut = h > 0 ? out.slice(out.length - h) : null
|
|
518
|
+
// Carry the last CONTEXT frames of THIS input as the next block's lead-in.
|
|
519
|
+
tailIn = newInput.slice(newInput.length - context)
|
|
520
|
+
pending = pending.slice(step)
|
|
521
|
+
emitted = concatPcm(emitted, blockOut)
|
|
522
|
+
}
|
|
523
|
+
return emitted
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
return {
|
|
527
|
+
push,
|
|
528
|
+
setFactor(next: number) {
|
|
529
|
+
if (next === f) return
|
|
530
|
+
f = next
|
|
531
|
+
// Drop the cross-block carry so the next block starts clean at the new
|
|
532
|
+
// factor; pending input is kept (a smaller seam than dropping it).
|
|
533
|
+
tailIn = new Float32Array(0)
|
|
534
|
+
heldOut = null
|
|
535
|
+
},
|
|
536
|
+
reset() {
|
|
537
|
+
pending = new Float32Array(0)
|
|
538
|
+
tailIn = new Float32Array(0)
|
|
539
|
+
heldOut = null
|
|
540
|
+
},
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
/**
|
|
545
|
+
* Can this track's audio be handed to `AudioDecoder` at all?
|
|
546
|
+
*
|
|
547
|
+
* A track with no `description` can only be configured if its codec carries
|
|
548
|
+
* its own in-band configuration — Opus does; AAC does not. SP1 §7.1 hit
|
|
549
|
+
* exactly this: mp4box.js cannot parse a QuickTime-v1 `mp4a` sample entry, so
|
|
550
|
+
* the original camera file's AAC has no obtainable `AudioSpecificConfig` and
|
|
551
|
+
* `configure()` can never succeed for it. Detect it up front and never call
|
|
552
|
+
* `configure()` for a config already known to be broken.
|
|
553
|
+
*
|
|
554
|
+
* Compares the NORMALIZED codec: `'Opus'.startsWith('opus')` is false (SP1
|
|
555
|
+
* §7.5).
|
|
556
|
+
*/
|
|
557
|
+
export function audioTrackIsDecodable(audio: ChunkSource): boolean {
|
|
558
|
+
if (audio.description !== undefined) return true
|
|
559
|
+
return normalizeAudioCodec(audio.codec).startsWith('opus')
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
// ── Wall-clock fallback ─────────────────────────────────────────────────────
|
|
563
|
+
|
|
564
|
+
/**
|
|
565
|
+
* The clock every non-audio path runs on: canvas projects, muted clips, clips
|
|
566
|
+
* whose audio cannot be decoded, browsers without WebCodecs audio, and any
|
|
567
|
+
* failure while building the real thing. Same transport surface, driven by
|
|
568
|
+
* `performance.now()` instead of rendered frames.
|
|
569
|
+
*/
|
|
570
|
+
function createFallbackClock(
|
|
571
|
+
startProjectS: number,
|
|
572
|
+
reason: string,
|
|
573
|
+
nowMs: () => number,
|
|
574
|
+
): MasterClock {
|
|
575
|
+
let base = startProjectS
|
|
576
|
+
let startedAtMs = nowMs()
|
|
577
|
+
let playing = false
|
|
578
|
+
let transportRate = 1
|
|
579
|
+
|
|
580
|
+
const read = () => (playing ? base + (transportRate * (nowMs() - startedAtMs)) / 1000 : base)
|
|
581
|
+
|
|
582
|
+
return {
|
|
583
|
+
kind: 'fallback',
|
|
584
|
+
reason,
|
|
585
|
+
get playing() {
|
|
586
|
+
return playing
|
|
587
|
+
},
|
|
588
|
+
now: read,
|
|
589
|
+
play() {
|
|
590
|
+
if (playing) return
|
|
591
|
+
startedAtMs = nowMs()
|
|
592
|
+
playing = true
|
|
593
|
+
},
|
|
594
|
+
pause() {
|
|
595
|
+
if (!playing) return
|
|
596
|
+
base = read()
|
|
597
|
+
playing = false
|
|
598
|
+
},
|
|
599
|
+
// `mediaS` is unused here: the fallback clock has no media position of its
|
|
600
|
+
// own to reconcile, only project time. Widened to match `MasterClock.seek`
|
|
601
|
+
// so it stays a drop-in implementation of the interface.
|
|
602
|
+
seek(projectS: number, _mediaS?: number) {
|
|
603
|
+
base = projectS
|
|
604
|
+
startedAtMs = nowMs()
|
|
605
|
+
},
|
|
606
|
+
setVolume() {
|
|
607
|
+
/* nothing is being rendered here — no level to set */
|
|
608
|
+
},
|
|
609
|
+
setTransportRate(rate: number) {
|
|
610
|
+
// Soft re-anchor so `now()` is continuous across the change: freeze where
|
|
611
|
+
// it reads, restart the wall-time baseline from here, then apply the new
|
|
612
|
+
// rate. Paused, `read()` returns `base` regardless of rate, so this is a
|
|
613
|
+
// harmless re-stamp; playing, it ramps R× from the current instant.
|
|
614
|
+
base = read()
|
|
615
|
+
startedAtMs = nowMs()
|
|
616
|
+
transportRate = rate
|
|
617
|
+
},
|
|
618
|
+
stats() {
|
|
619
|
+
return {
|
|
620
|
+
kind: 'fallback' as const,
|
|
621
|
+
reason,
|
|
622
|
+
playing,
|
|
623
|
+
samplesConsumed: 0,
|
|
624
|
+
underrunFrames: 0,
|
|
625
|
+
queuedFrames: 0,
|
|
626
|
+
queuedSeconds: 0,
|
|
627
|
+
}
|
|
628
|
+
},
|
|
629
|
+
dispose() {
|
|
630
|
+
// Freeze where it reads rather than just un-setting `playing`, so a
|
|
631
|
+
// disposed clock's last reading stays truthful (a caller that reads once
|
|
632
|
+
// more during teardown must not see the playhead snap back to the anchor).
|
|
633
|
+
base = read()
|
|
634
|
+
playing = false
|
|
635
|
+
},
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
/**
|
|
640
|
+
* The wall-clock `MasterClock`, SYNCHRONOUSLY.
|
|
641
|
+
*
|
|
642
|
+
* `createMasterClock` already returns exactly this object for every no-audio
|
|
643
|
+
* case, but it is a `Promise` by contract. T5's scheduler swaps clocks inside a
|
|
644
|
+
* synchronous tick — into a gap, out of a gap, onto a clip whose proxy has not
|
|
645
|
+
* arrived — and cannot await one there. Exported rather than re-implemented in
|
|
646
|
+
* the scheduler so the engine has exactly ONE wall clock: a second copy is the
|
|
647
|
+
* D9-class duplication the divergence registry exists to prevent.
|
|
648
|
+
*
|
|
649
|
+
* `reason` is surfaced in `stats()` and by T7's HUD, so pass something a human
|
|
650
|
+
* can read ('gap', 'canvas project', 'preparing').
|
|
651
|
+
*/
|
|
652
|
+
export function createWallClock(
|
|
653
|
+
startProjectS: number,
|
|
654
|
+
reason: string,
|
|
655
|
+
nowMs: () => number = () => performance.now(),
|
|
656
|
+
): MasterClock {
|
|
657
|
+
return createFallbackClock(startProjectS, reason, nowMs)
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
// ── Audio clock ─────────────────────────────────────────────────────────────
|
|
661
|
+
|
|
662
|
+
/** Tracks whether this context's AudioWorkletGlobalScope already has the ring module. */
|
|
663
|
+
const workletLoaded = new WeakSet<BaseAudioContext>()
|
|
664
|
+
|
|
665
|
+
/**
|
|
666
|
+
* `addModule` once per AudioContext. The context is page-scoped and outlives
|
|
667
|
+
* every clock built on it, so a naive add-per-clock would re-evaluate the
|
|
668
|
+
* module on every clip boundary; the string's own `registerProcessor` guard
|
|
669
|
+
* makes that survivable, and this cache makes it not happen in the first
|
|
670
|
+
* place. (Both are needed: module state resets on a dev hot reload while
|
|
671
|
+
* `window.__montajSharedCtx` does not.)
|
|
672
|
+
*/
|
|
673
|
+
async function ensureRingModule(ctx: BaseAudioContext): Promise<void> {
|
|
674
|
+
if (workletLoaded.has(ctx)) return
|
|
675
|
+
const url = URL.createObjectURL(new Blob([audioWorkletSource], { type: 'text/javascript' }))
|
|
676
|
+
try {
|
|
677
|
+
await ctx.audioWorklet.addModule(url)
|
|
678
|
+
workletLoaded.add(ctx)
|
|
679
|
+
} finally {
|
|
680
|
+
URL.revokeObjectURL(url)
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
async function createAudioClock(
|
|
685
|
+
audio: ChunkSource,
|
|
686
|
+
options: Required<Pick<MasterClockOptions, 'timebase' | 'startProjectS'>> & {
|
|
687
|
+
volume: number
|
|
688
|
+
onError?: (message: string) => void
|
|
689
|
+
nowMs: () => number
|
|
690
|
+
},
|
|
691
|
+
): Promise<MasterClock> {
|
|
692
|
+
const { timebase, startProjectS, onError, nowMs } = options
|
|
693
|
+
const ctx = getSharedAudioContext()
|
|
694
|
+
await ensureRingModule(ctx)
|
|
695
|
+
|
|
696
|
+
// The container's real parameters (T2's deviation from the spike, which had
|
|
697
|
+
// to hardcode 48000/2). The decoded AudioData's own `sampleRate` is what the
|
|
698
|
+
// resampler trusts per chunk; this one only configures the decoder.
|
|
699
|
+
const channels = audio.audio?.channelCount || DEFAULT_CHANNELS
|
|
700
|
+
const decodedRate = audio.audio?.sampleRate || DEFAULT_SAMPLE_RATE
|
|
701
|
+
const renderRate = ctx.sampleRate
|
|
702
|
+
|
|
703
|
+
const node = new AudioWorkletNode(ctx, RING_PROCESSOR_NAME, {
|
|
704
|
+
numberOfInputs: 0,
|
|
705
|
+
numberOfOutputs: 1,
|
|
706
|
+
outputChannelCount: [channels],
|
|
707
|
+
processorOptions: { reportIntervalS: REPORT_INTERVAL_S },
|
|
708
|
+
})
|
|
709
|
+
// Per-session output gain — the clip's volume rides HERE, not in the PCM.
|
|
710
|
+
// Each clip session owns its own node → gain → destination chain (one ring
|
|
711
|
+
// per clip, never a shared stream across clip boundaries — see
|
|
712
|
+
// `startSession`/`createMasterClock`), so a live gain change only ever touches
|
|
713
|
+
// this clip's audio and can never re-level a neighbour's tail sitting in
|
|
714
|
+
// another session's ring. That is the property the old enqueue-time PCM
|
|
715
|
+
// scaling was protecting; a per-session node has it for free, and unlike PCM
|
|
716
|
+
// scaling a gain change is heard IMMEDIATELY — it applies to whatever is
|
|
717
|
+
// leaving the node now, including the up-to-RING_SECONDS already buffered,
|
|
718
|
+
// instead of only to PCM enqueued after the change. This also matches the
|
|
719
|
+
// legacy player, which routes each <video> through its own GainNode.
|
|
720
|
+
const gainNode = ctx.createGain()
|
|
721
|
+
gainNode.gain.value = options.volume
|
|
722
|
+
node.connect(gainNode)
|
|
723
|
+
gainNode.connect(ctx.destination)
|
|
724
|
+
|
|
725
|
+
if (ctx.state === 'suspended') {
|
|
726
|
+
// Best effort only: a resume outside a user-gesture call stack is not
|
|
727
|
+
// gesture-credited and usually stays suspended. T6 wires the real anchors
|
|
728
|
+
// (`togglePlay`, the Space keydown) through
|
|
729
|
+
// `resumeAudioContextFromGesture`. Until something does, the clock — and
|
|
730
|
+
// therefore the painter — legitimately stalls; see MAX_EXTRAPOLATION_MS.
|
|
731
|
+
void ctx.resume().catch(() => {})
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
let playing = false
|
|
735
|
+
let disposed = false
|
|
736
|
+
|
|
737
|
+
// Ring epoch. Bumped on every reset (seek / play-from-frozen); reports
|
|
738
|
+
// carrying an older epoch describe a ring that no longer exists and are
|
|
739
|
+
// discarded — the audio-side twin of the decode worker's `reqId`
|
|
740
|
+
// supersession.
|
|
741
|
+
let epoch = 0
|
|
742
|
+
/** Epoch the in-flight decode batch belongs to; its AudioData is dropped if it went stale. */
|
|
743
|
+
let feedEpoch = 0
|
|
744
|
+
|
|
745
|
+
let anchorProjectS = startProjectS
|
|
746
|
+
let frozenProjectS = startProjectS
|
|
747
|
+
let samplesConsumed = 0
|
|
748
|
+
/** Transport rate R (default 1). Project time advances R× per rendered second. */
|
|
749
|
+
let transportRate = 1
|
|
750
|
+
/**
|
|
751
|
+
* Cumulative-sample baseline captured on every reset and on every
|
|
752
|
+
* `setTransportRate`. `read()` counts rendered frames from HERE, not from the
|
|
753
|
+
* ring epoch's frame 0, so a mid-playback rate change re-bases the R× ramp
|
|
754
|
+
* without a discontinuity while `samplesConsumed` keeps its cumulative meaning
|
|
755
|
+
* (the worklet reports it monotonically and the feeder budgets off it).
|
|
756
|
+
*/
|
|
757
|
+
let rateAnchorSamples = 0
|
|
758
|
+
/**
|
|
759
|
+
* Pitch-preserving time-stretch factor = 1/(R·S), where R is the live
|
|
760
|
+
* `transportRate` and S the fixed per-clip `timebase.speed`. The clip
|
|
761
|
+
* traverses R·S source-seconds per project-second, and that content must be
|
|
762
|
+
* compressed (R·S>1) or expanded (R·S<1) to fill one project-second of audio.
|
|
763
|
+
* factor 1 (R=S=1) is the strict-bypass path: no stretch, no buffering, the
|
|
764
|
+
* pipeline runs exactly as it did before variable rate.
|
|
765
|
+
*/
|
|
766
|
+
const clipSpeed = timebase.speed ?? 1
|
|
767
|
+
let stretchFactor = 1 / (transportRate * clipSpeed)
|
|
768
|
+
const streamStretch = createStreamStretch(channels, stretchFactor)
|
|
769
|
+
let underrunFrames = 0
|
|
770
|
+
let lastReportMs = nowMs()
|
|
771
|
+
/** Frames posted to the ring since the reset. */
|
|
772
|
+
let postedFrames = 0
|
|
773
|
+
/** Frames the ring has actually drained (rendered MINUS silence) as of the last report. */
|
|
774
|
+
let drainedFrames = 0
|
|
775
|
+
let resamplePhase = 0
|
|
776
|
+
|
|
777
|
+
let nextPacketIdx = 0
|
|
778
|
+
let decoding = false
|
|
779
|
+
/**
|
|
780
|
+
* Set when `decoder.decode()` throws synchronously (a dead/misconfigured
|
|
781
|
+
* decoder). Without this, `feedMore` is re-armed at the end of every
|
|
782
|
+
* `output`/report cycle (see `feedMore`'s own tail call and the port
|
|
783
|
+
* `onmessage` handler) and would retry the same broken decoder roughly
|
|
784
|
+
* every report interval forever — a permanent ~10Hz `onError` spam for a
|
|
785
|
+
* session that can never recover. Cleared on `restartAt` (seek re-anchors
|
|
786
|
+
* the decoder's input position, which is the only thing that can make a
|
|
787
|
+
* retry meaningfully different).
|
|
788
|
+
*/
|
|
789
|
+
let feedFailed = false
|
|
790
|
+
|
|
791
|
+
const queuedFrames = () => Math.max(0, postedFrames - drainedFrames)
|
|
792
|
+
|
|
793
|
+
const read = (): number => {
|
|
794
|
+
if (!playing) return frozenProjectS
|
|
795
|
+
const projected = extrapolateSamples(samplesConsumed, nowMs() - lastReportMs, renderRate)
|
|
796
|
+
// Count frames from the last rate-anchor, not the ring epoch's frame 0, and
|
|
797
|
+
// scale by R. At R=1 with no rate change (`rateAnchorSamples === 0`) this is
|
|
798
|
+
// exactly `projectTimeForSamples(anchorProjectS, projected, renderRate)`.
|
|
799
|
+
return projectTimeForSamples(
|
|
800
|
+
anchorProjectS,
|
|
801
|
+
projected - rateAnchorSamples,
|
|
802
|
+
renderRate,
|
|
803
|
+
transportRate,
|
|
804
|
+
)
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
const decoder = new AudioDecoder({
|
|
808
|
+
output: (frame) => {
|
|
809
|
+
try {
|
|
810
|
+
if (disposed || feedEpoch !== epoch) return
|
|
811
|
+
const planeCount = frame.numberOfChannels
|
|
812
|
+
const frames = frame.numberOfFrames
|
|
813
|
+
const planes: Float32Array[] = []
|
|
814
|
+
for (let ch = 0; ch < planeCount; ch++) {
|
|
815
|
+
const plane = new Float32Array(frames)
|
|
816
|
+
// Ask the platform to convert from whatever the decoder's native
|
|
817
|
+
// output format is (s16/f32/planar/interleaved) so nothing here has
|
|
818
|
+
// to special-case it.
|
|
819
|
+
frame.copyTo(plane, { planeIndex: ch, format: 'f32-planar' })
|
|
820
|
+
planes.push(plane)
|
|
821
|
+
}
|
|
822
|
+
// Interleave only — volume is applied downstream by `gainNode` (see the
|
|
823
|
+
// node/gain/destination chain above), so the ring holds the clip at unit
|
|
824
|
+
// level and a `setVolume` is heard without waiting for the ring to drain.
|
|
825
|
+
const scaled = scaleAndInterleavePlanes(planes, frames, 1)
|
|
826
|
+
// Pitch-preserving time-stretch at the DECODED rate, BEFORE the device
|
|
827
|
+
// resample. Strict bypass at factor 1 — `scaled` passes straight through,
|
|
828
|
+
// exactly as before variable rate. Otherwise the streaming wrapper
|
|
829
|
+
// accumulates across these tiny (~20ms) packets and emits whole WSOLA
|
|
830
|
+
// blocks, so `push` returns nothing until a block is ready.
|
|
831
|
+
const stretched = stretchFactor === 1 ? scaled : streamStretch.push(scaled)
|
|
832
|
+
if (stretched.length === 0) return
|
|
833
|
+
const ratio = frame.sampleRate / renderRate
|
|
834
|
+
const resampled = resampleInterleaved(stretched, planeCount, ratio, resamplePhase)
|
|
835
|
+
resamplePhase = resampled.phase
|
|
836
|
+
const outFrames = Math.floor(resampled.pcm.length / planeCount)
|
|
837
|
+
if (outFrames === 0) return
|
|
838
|
+
postedFrames += outFrames
|
|
839
|
+
node.port.postMessage(
|
|
840
|
+
{ t: 'chunk', pcm: resampled.pcm, channels: planeCount, frames: outFrames },
|
|
841
|
+
[resampled.pcm.buffer],
|
|
842
|
+
)
|
|
843
|
+
} finally {
|
|
844
|
+
frame.close()
|
|
845
|
+
}
|
|
846
|
+
},
|
|
847
|
+
error: (err) => {
|
|
848
|
+
// A dead decoder starves the ring; the worklet renders silence and the
|
|
849
|
+
// clock keeps advancing (that is the whole point of silence-through-
|
|
850
|
+
// underruns). Report it; never throw at the caller.
|
|
851
|
+
onError?.(`audio decoder: ${err instanceof Error ? err.message : String(err)}`)
|
|
852
|
+
},
|
|
853
|
+
})
|
|
854
|
+
try {
|
|
855
|
+
decoder.configure({
|
|
856
|
+
// `normalizeAudioCodec` both fixes mp4box's casing (Opus → opus, which
|
|
857
|
+
// `configure()` rejects otherwise) and expands a bare `mp4a` — though the
|
|
858
|
+
// latter cannot reach here, `audioTrackIsDecodable` having already
|
|
859
|
+
// rejected description-less non-Opus tracks.
|
|
860
|
+
codec: normalizeAudioCodec(audio.codec),
|
|
861
|
+
description: audio.description,
|
|
862
|
+
sampleRate: decodedRate,
|
|
863
|
+
numberOfChannels: channels,
|
|
864
|
+
})
|
|
865
|
+
} catch (err) {
|
|
866
|
+
// Leave nothing connected to the shared context on the way out; the caller
|
|
867
|
+
// turns this into a wall-clock fallback.
|
|
868
|
+
try {
|
|
869
|
+
decoder.close()
|
|
870
|
+
} catch {
|
|
871
|
+
// never configured — nothing to close.
|
|
872
|
+
}
|
|
873
|
+
node.disconnect()
|
|
874
|
+
gainNode.disconnect()
|
|
875
|
+
throw err
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
/**
|
|
879
|
+
* Top the ring up to `RING_SECONDS`. Runs while PAUSED too, so a parked
|
|
880
|
+
* clock is already primed and `play()` starts on real audio instead of an
|
|
881
|
+
* underrun.
|
|
882
|
+
*/
|
|
883
|
+
function feedMore(): void {
|
|
884
|
+
if (disposed || decoding || feedFailed) return
|
|
885
|
+
if (nextPacketIdx >= audio.samples.length) return
|
|
886
|
+
if (queuedFrames() / renderRate >= RING_SECONDS) return
|
|
887
|
+
|
|
888
|
+
decoding = true
|
|
889
|
+
feedEpoch = epoch
|
|
890
|
+
const end = Math.min(nextPacketIdx + AUDIO_BATCH_PACKETS, audio.samples.length)
|
|
891
|
+
|
|
892
|
+
// A ranged source (`demux.ts`) holds the packet INDEX but not necessarily
|
|
893
|
+
// the packet BYTES. `ensure` returns null once they are resident — the
|
|
894
|
+
// whole-file path always does — so the steady state is unchanged and this
|
|
895
|
+
// costs nothing. On a miss, hold `decoding` (the re-entrance gate) across
|
|
896
|
+
// the fetch and then re-run from the top rather than continuing with this
|
|
897
|
+
// batch's indices: a `restartAt` may have moved `nextPacketIdx` while the
|
|
898
|
+
// bytes were in flight, and feeding the pre-seek batch would fill the ring
|
|
899
|
+
// with audio from the position the user just left.
|
|
900
|
+
const bytes = audio.ensure?.(nextPacketIdx, end)
|
|
901
|
+
if (bytes) {
|
|
902
|
+
bytes.then(
|
|
903
|
+
() => {
|
|
904
|
+
decoding = false
|
|
905
|
+
if (!disposed) feedMore()
|
|
906
|
+
},
|
|
907
|
+
(err: unknown) => {
|
|
908
|
+
decoding = false
|
|
909
|
+
feedFailed = true
|
|
910
|
+
onError?.(`audio fetch: ${err instanceof Error ? err.message : String(err)}`)
|
|
911
|
+
},
|
|
912
|
+
)
|
|
913
|
+
return
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
try {
|
|
917
|
+
for (let i = nextPacketIdx; i < end; i++) {
|
|
918
|
+
const s = audio.samples[i]
|
|
919
|
+
// Audio codecs have no delta-frame concept the way video GOPs do —
|
|
920
|
+
// every packet decodes independently, so always 'key' regardless of
|
|
921
|
+
// `SampleRef.isKey` (which demux derives from the container's
|
|
922
|
+
// sync-sample flag, meaningless on an audio track).
|
|
923
|
+
decoder.decode(
|
|
924
|
+
new EncodedAudioChunk({ type: 'key', timestamp: s.tsUs, duration: s.durUs, data: s.data }),
|
|
925
|
+
)
|
|
926
|
+
}
|
|
927
|
+
} catch (err) {
|
|
928
|
+
decoding = false
|
|
929
|
+
feedFailed = true
|
|
930
|
+
onError?.(`audio decode: ${err instanceof Error ? err.message : String(err)}`)
|
|
931
|
+
return
|
|
932
|
+
}
|
|
933
|
+
nextPacketIdx = end
|
|
934
|
+
decoder
|
|
935
|
+
.flush()
|
|
936
|
+
.then(() => {
|
|
937
|
+
decoding = false
|
|
938
|
+
// The ring may have been reset mid-batch; `feedMore` always reads the
|
|
939
|
+
// CURRENT position, so a stale batch costs nothing beyond the frames
|
|
940
|
+
// its epoch check already discarded in `output`.
|
|
941
|
+
feedMore()
|
|
942
|
+
})
|
|
943
|
+
.catch((err) => {
|
|
944
|
+
decoding = false
|
|
945
|
+
onError?.(`audio flush: ${err instanceof Error ? err.message : String(err)}`)
|
|
946
|
+
})
|
|
947
|
+
}
|
|
948
|
+
|
|
949
|
+
/**
|
|
950
|
+
* Reset the ring and re-anchor everything to `projectS`.
|
|
951
|
+
*
|
|
952
|
+
* Decoding starts at the packet at-or-before the target, so the first audio
|
|
953
|
+
* the ring plays can lead the anchor by up to one packet (~20ms for the
|
|
954
|
+
* proxies' libopus encode) while the clock still calls that instant
|
|
955
|
+
* `projectS`. Spike behavior, kept deliberately: SP1 §6 measured the whole
|
|
956
|
+
* pipeline at −4.3ms p50 / −0.4ms p95 A/V error with exactly this anchoring,
|
|
957
|
+
* i.e. well inside the band, and the alternative — anchoring on the fed
|
|
958
|
+
* packet's true media time — would instead make `seek(t)` land the *picture*
|
|
959
|
+
* up to 20ms off the requested frame, which is the more visible error in an
|
|
960
|
+
* editor. Trimming the first packet's leading samples would remove both and
|
|
961
|
+
* is the named follow-up if the checklist ever measures it as audible.
|
|
962
|
+
*/
|
|
963
|
+
function restartAt(projectS: number, mediaS?: number): void {
|
|
964
|
+
epoch += 1
|
|
965
|
+
anchorProjectS = projectS
|
|
966
|
+
frozenProjectS = projectS
|
|
967
|
+
samplesConsumed = 0
|
|
968
|
+
// Re-base the rate ramp with the ring: samples are counted from 0 again, so
|
|
969
|
+
// the baseline must be 0. `transportRate` PERSISTS — a seek/wrap re-anchors
|
|
970
|
+
// media and project time, not the global transport rate.
|
|
971
|
+
rateAnchorSamples = 0
|
|
972
|
+
underrunFrames = 0
|
|
973
|
+
postedFrames = 0
|
|
974
|
+
drainedFrames = 0
|
|
975
|
+
resamplePhase = 0
|
|
976
|
+
// The stretcher's buffered input and cross-block carry describe the old ring
|
|
977
|
+
// epoch; drop them so the new epoch's first block starts clean.
|
|
978
|
+
streamStretch.reset()
|
|
979
|
+
feedFailed = false
|
|
980
|
+
lastReportMs = nowMs()
|
|
981
|
+
// `mediaS`, when given, is the loop-wrapped media position the frame
|
|
982
|
+
// server just restarted its stream at — decode from THAT packet, not from
|
|
983
|
+
// whatever `projectS` maps to through the timebase (which is the pre-wrap
|
|
984
|
+
// mapping and would restart audio a whole loop window ahead of the
|
|
985
|
+
// picture). See `MasterClock.seek`'s doc.
|
|
986
|
+
nextPacketIdx = sampleAtOrBefore(
|
|
987
|
+
audio,
|
|
988
|
+
mediaS === undefined
|
|
989
|
+
? mediaTsUsForProjectTime(timebase, projectS)
|
|
990
|
+
: timebase.firstPresentationTsUs + mediaS * 1_000_000,
|
|
991
|
+
)
|
|
992
|
+
node.port.postMessage({ t: 'reset', epoch })
|
|
993
|
+
feedMore()
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
node.port.onmessage = (ev: MessageEvent) => {
|
|
997
|
+
const msg = ev.data as {
|
|
998
|
+
t?: string
|
|
999
|
+
epoch?: number
|
|
1000
|
+
samplesConsumed?: number
|
|
1001
|
+
underrunFrames?: number
|
|
1002
|
+
}
|
|
1003
|
+
if (msg?.t !== 'consumed' || msg.epoch !== epoch) return
|
|
1004
|
+
samplesConsumed = msg.samplesConsumed ?? 0
|
|
1005
|
+
underrunFrames = msg.underrunFrames ?? 0
|
|
1006
|
+
// Frames the ring actually drained = rendered minus the silence it
|
|
1007
|
+
// invented. Using `samplesConsumed` alone would undercount the queue after
|
|
1008
|
+
// any underrun and make the feeder over-decode.
|
|
1009
|
+
drainedFrames = samplesConsumed - underrunFrames
|
|
1010
|
+
lastReportMs = nowMs()
|
|
1011
|
+
feedMore()
|
|
1012
|
+
}
|
|
1013
|
+
|
|
1014
|
+
// Anchor without playing: the caller gets a clock parked at startProjectS.
|
|
1015
|
+
restartAt(startProjectS)
|
|
1016
|
+
|
|
1017
|
+
return {
|
|
1018
|
+
kind: 'audio',
|
|
1019
|
+
get playing() {
|
|
1020
|
+
return playing
|
|
1021
|
+
},
|
|
1022
|
+
now: read,
|
|
1023
|
+
play() {
|
|
1024
|
+
if (playing || disposed) return
|
|
1025
|
+
// Resume from the FROZEN reading. After a real pause the ring is primed
|
|
1026
|
+
// for the last position the worklet actually rendered, which trails the
|
|
1027
|
+
// frozen (extrapolated) reading by up to one report interval — resuming
|
|
1028
|
+
// it as-is would visibly walk the playhead backwards, so the ring is
|
|
1029
|
+
// re-anchored instead. One Opus decode batch is far cheaper than real
|
|
1030
|
+
// time, so the restart is not felt.
|
|
1031
|
+
//
|
|
1032
|
+
// The exception is a clock that has rendered nothing since its last
|
|
1033
|
+
// anchor (freshly created, or `seek`ed while paused): its ring is already
|
|
1034
|
+
// primed for exactly this position, so un-gate it and keep the pre-buffer.
|
|
1035
|
+
if (frozenProjectS !== anchorProjectS || samplesConsumed > 0) restartAt(frozenProjectS)
|
|
1036
|
+
playing = true
|
|
1037
|
+
// Re-base extrapolation on NOW. Without this the first `now()` after a
|
|
1038
|
+
// long pause extrapolates from a stale report and jumps the playhead
|
|
1039
|
+
// forward by the whole extrapolation ceiling.
|
|
1040
|
+
lastReportMs = nowMs()
|
|
1041
|
+
node.port.postMessage({ t: 'play' })
|
|
1042
|
+
},
|
|
1043
|
+
pause() {
|
|
1044
|
+
if (!playing) return
|
|
1045
|
+
// Freeze at the EXTRAPOLATED reading — the best estimate of what the user
|
|
1046
|
+
// just saw and heard — rather than at the last report, which can be up to
|
|
1047
|
+
// a full report interval stale and would visibly snap the playhead back.
|
|
1048
|
+
frozenProjectS = read()
|
|
1049
|
+
playing = false
|
|
1050
|
+
node.port.postMessage({ t: 'pause' })
|
|
1051
|
+
},
|
|
1052
|
+
seek(projectS: number, mediaS?: number) {
|
|
1053
|
+
if (disposed) return
|
|
1054
|
+
restartAt(projectS, mediaS)
|
|
1055
|
+
if (playing) node.port.postMessage({ t: 'play' })
|
|
1056
|
+
},
|
|
1057
|
+
setVolume(next: number) {
|
|
1058
|
+
if (disposed) return
|
|
1059
|
+
// Ride the output gain node: heard immediately, no ring flush, no
|
|
1060
|
+
// re-decode. A short time-constant ramp (~15ms) instead of a hard jump so
|
|
1061
|
+
// a fast slider drag doesn't zipper-click. `setTargetAtTime` schedules
|
|
1062
|
+
// against the context clock, so it also settles cleanly when the context
|
|
1063
|
+
// is suspended and later resumes.
|
|
1064
|
+
gainNode.gain.setTargetAtTime(next, ctx.currentTime, 0.015)
|
|
1065
|
+
},
|
|
1066
|
+
setTransportRate(rate: number) {
|
|
1067
|
+
if (disposed) return
|
|
1068
|
+
const wasPlaying = playing
|
|
1069
|
+
// The current project time, captured with the OLD rate BEFORE anything
|
|
1070
|
+
// changes — the anchor the restart re-pins to, so the playhead stays
|
|
1071
|
+
// continuous across the rate step (no jump).
|
|
1072
|
+
const nowProjectS = read()
|
|
1073
|
+
transportRate = rate
|
|
1074
|
+
// Re-derive the stretch factor and hand it to the streamer (drops its
|
|
1075
|
+
// cross-block carry, re-blocks at the new factor).
|
|
1076
|
+
stretchFactor = 1 / (rate * clipSpeed)
|
|
1077
|
+
streamStretch.setFactor(stretchFactor)
|
|
1078
|
+
if (wasPlaying) {
|
|
1079
|
+
// Flush the ring. It holds up to RING_SECONDS of audio already stretched
|
|
1080
|
+
// at the OLD factor; leaving it would keep the speaker on the old rate
|
|
1081
|
+
// for up to ~2s after the picture moved to the new one — and audible
|
|
1082
|
+
// fast-forward is the point. `restartAt` re-anchors the clock to
|
|
1083
|
+
// `nowProjectS` (samplesConsumed/rateAnchorSamples reset ⇒ `read()`
|
|
1084
|
+
// returns `nowProjectS`, no jump), re-primes the decoder from the
|
|
1085
|
+
// matching media position through the timebase, and resets the stretcher
|
|
1086
|
+
// — new-factor audio reaches the speaker within one decode batch, the
|
|
1087
|
+
// same brief re-decode gap `seek`/`play` already pay.
|
|
1088
|
+
restartAt(nowProjectS)
|
|
1089
|
+
node.port.postMessage({ t: 'play' })
|
|
1090
|
+
}
|
|
1091
|
+
// Paused: nothing to flush and nothing to re-anchor — `read()` returns
|
|
1092
|
+
// `frozenProjectS` regardless of rate, and `play()` → `restartAt` re-primes
|
|
1093
|
+
// the ring (with the new factor already set) before the new rate is ever
|
|
1094
|
+
// counted against samples. Keeping the primed ring avoids a needless
|
|
1095
|
+
// re-decode. (Do NOT flush when paused.)
|
|
1096
|
+
},
|
|
1097
|
+
stats() {
|
|
1098
|
+
return {
|
|
1099
|
+
kind: 'audio' as const,
|
|
1100
|
+
playing,
|
|
1101
|
+
samplesConsumed,
|
|
1102
|
+
underrunFrames,
|
|
1103
|
+
queuedFrames: queuedFrames(),
|
|
1104
|
+
queuedSeconds: queuedFrames() / renderRate,
|
|
1105
|
+
}
|
|
1106
|
+
},
|
|
1107
|
+
dispose() {
|
|
1108
|
+
if (disposed) return
|
|
1109
|
+
// Freeze before un-setting `playing`, so a read during teardown returns
|
|
1110
|
+
// the last true position rather than snapping back to the ring anchor.
|
|
1111
|
+
frozenProjectS = read()
|
|
1112
|
+
disposed = true
|
|
1113
|
+
playing = false
|
|
1114
|
+
node.port.onmessage = null
|
|
1115
|
+
try {
|
|
1116
|
+
node.port.postMessage({ t: 'pause' })
|
|
1117
|
+
} catch {
|
|
1118
|
+
// port already torn down — nothing to stop.
|
|
1119
|
+
}
|
|
1120
|
+
try {
|
|
1121
|
+
decoder.close()
|
|
1122
|
+
} catch {
|
|
1123
|
+
// already closed or errored — we are tearing down anyway.
|
|
1124
|
+
}
|
|
1125
|
+
try {
|
|
1126
|
+
node.disconnect()
|
|
1127
|
+
gainNode.disconnect()
|
|
1128
|
+
} catch {
|
|
1129
|
+
// already disconnected — fine.
|
|
1130
|
+
}
|
|
1131
|
+
// The AudioContext is deliberately NOT closed: it is
|
|
1132
|
+
// `window.__montajSharedCtx`, shared with the legacy player's video slots
|
|
1133
|
+
// and audio lanes and reused by every later clock. Closing it here would
|
|
1134
|
+
// silence the whole page. (The spike closed its context because it owned
|
|
1135
|
+
// one privately.)
|
|
1136
|
+
},
|
|
1137
|
+
}
|
|
1138
|
+
}
|
|
1139
|
+
|
|
1140
|
+
/**
|
|
1141
|
+
* Build the master clock for one clip. **Never throws and never rejects** —
|
|
1142
|
+
* every failure resolves to a wall-clock `MasterClock` carrying a `reason`.
|
|
1143
|
+
*
|
|
1144
|
+
* The fallback is not an error path so much as the normal mode for a large
|
|
1145
|
+
* share of projects: canvas (image-only) projects have no audio at all, muted
|
|
1146
|
+
* clips have nothing worth syncing to, and a browser without WebCodecs audio
|
|
1147
|
+
* still has to scrub and play. Everything downstream reads the same surface
|
|
1148
|
+
* either way.
|
|
1149
|
+
*/
|
|
1150
|
+
export async function createMasterClock(options: MasterClockOptions): Promise<MasterClock> {
|
|
1151
|
+
const { audio, timebase, startProjectS, muted, onError } = options
|
|
1152
|
+
const nowMs = options.nowMs ?? (() => performance.now())
|
|
1153
|
+
const volume = muted ? 0 : (options.volume ?? 1)
|
|
1154
|
+
|
|
1155
|
+
// Cheap disqualifications first — none of these should create an AudioContext
|
|
1156
|
+
// for a project that will never render a sample through it.
|
|
1157
|
+
if (muted) return createFallbackClock(startProjectS, 'clip is muted', nowMs)
|
|
1158
|
+
if (!audio || audio.samples.length === 0) {
|
|
1159
|
+
return createFallbackClock(startProjectS, 'no audio track', nowMs)
|
|
1160
|
+
}
|
|
1161
|
+
if (typeof AudioDecoder === 'undefined' || typeof AudioWorkletNode === 'undefined') {
|
|
1162
|
+
return createFallbackClock(startProjectS, 'WebCodecs audio / AudioWorklet unavailable', nowMs)
|
|
1163
|
+
}
|
|
1164
|
+
if (!audioTrackIsDecodable(audio)) {
|
|
1165
|
+
return createFallbackClock(
|
|
1166
|
+
startProjectS,
|
|
1167
|
+
`audio codec "${audio.codec}" has no decoder description (mp4box couldn't parse this file's audio sample entry)`,
|
|
1168
|
+
nowMs,
|
|
1169
|
+
)
|
|
1170
|
+
}
|
|
1171
|
+
|
|
1172
|
+
try {
|
|
1173
|
+
return await createAudioClock(audio, { timebase, startProjectS, volume, onError, nowMs })
|
|
1174
|
+
} catch (err) {
|
|
1175
|
+
const message = err instanceof Error ? err.message : String(err)
|
|
1176
|
+
onError?.(`audio clock: ${message}`)
|
|
1177
|
+
return createFallbackClock(startProjectS, message, nowMs)
|
|
1178
|
+
}
|
|
1179
|
+
}
|