@blockcast/mmt-render 0.1.0-main.c0e9e40265a1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +207 -0
- package/dist/.build-stamp +0 -0
- package/dist/audio/audio-buffer-plan.d.ts +27 -0
- package/dist/audio/audio-buffer-plan.d.ts.map +1 -0
- package/dist/audio/audio-render-pipeline.d.ts +83 -0
- package/dist/audio/audio-render-pipeline.d.ts.map +1 -0
- package/dist/audio/audio-render-pipeline.js +372 -0
- package/dist/audio/audio-render-pipeline.js.map +7 -0
- package/dist/audio/audio-ring-buffer.d.ts +55 -0
- package/dist/audio/audio-ring-buffer.d.ts.map +1 -0
- package/dist/audio/audio-ring-buffer.js +255 -0
- package/dist/audio/audio-ring-buffer.js.map +7 -0
- package/dist/audio/index.d.ts +9 -0
- package/dist/audio/index.d.ts.map +1 -0
- package/dist/audio/index.js +640 -0
- package/dist/audio/index.js.map +7 -0
- package/dist/audio/render-messages.d.ts +94 -0
- package/dist/audio/render-messages.d.ts.map +1 -0
- package/dist/audio/render-messages.js +1 -0
- package/dist/audio/render-messages.js.map +7 -0
- package/dist/audio/render-worklet.d.ts +13 -0
- package/dist/audio/render-worklet.d.ts.map +1 -0
- package/dist/audio/worklet-url.d.ts +25 -0
- package/dist/audio/worklet-url.d.ts.map +1 -0
- package/dist/audio/worklet-url.js +11 -0
- package/dist/audio/worklet-url.js.map +7 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +5 -0
- package/dist/index.js.map +7 -0
- package/dist/pacing/au-duration.d.ts +18 -0
- package/dist/pacing/au-duration.d.ts.map +1 -0
- package/dist/pacing/au-duration.js +23 -0
- package/dist/pacing/au-duration.js.map +7 -0
- package/dist/pacing/frame-pacer.d.ts +71 -0
- package/dist/pacing/frame-pacer.d.ts.map +1 -0
- package/dist/pacing/frame-pacer.js +89 -0
- package/dist/pacing/frame-pacer.js.map +7 -0
- package/dist/pacing/index.d.ts +6 -0
- package/dist/pacing/index.d.ts.map +1 -0
- package/dist/pacing/index.js +166 -0
- package/dist/pacing/index.js.map +7 -0
- package/dist/pacing/interleave-timing.d.ts +64 -0
- package/dist/pacing/interleave-timing.d.ts.map +1 -0
- package/dist/pacing/interleave-timing.js +58 -0
- package/dist/pacing/interleave-timing.js.map +7 -0
- package/dist/video/index.d.ts +4 -0
- package/dist/video/index.d.ts.map +1 -0
- package/dist/video/index.js +247 -0
- package/dist/video/index.js.map +7 -0
- package/dist/video/keyframe-gate.d.ts +21 -0
- package/dist/video/keyframe-gate.d.ts.map +1 -0
- package/dist/video/keyframe-gate.js +39 -0
- package/dist/video/keyframe-gate.js.map +7 -0
- package/dist/video/video-render-pipeline.d.ts +60 -0
- package/dist/video/video-render-pipeline.d.ts.map +1 -0
- package/dist/video/video-render-pipeline.js +247 -0
- package/dist/video/video-render-pipeline.js.map +7 -0
- package/dist/worklet/render-worklet.js +403 -0
- package/dist/worklet/render-worklet.js.map +7 -0
- package/package.json +101 -0
- package/src/audio/audio-buffer-plan.ts +62 -0
- package/src/audio/audio-render-pipeline.ts +313 -0
- package/src/audio/audio-ring-buffer.ts +281 -0
- package/src/audio/index.ts +14 -0
- package/src/audio/render-messages.ts +110 -0
- package/src/audio/render-worklet.ts +161 -0
- package/src/audio/worklet-url.ts +30 -0
- package/src/index.ts +13 -0
- package/src/pacing/au-duration.ts +34 -0
- package/src/pacing/frame-pacer.ts +129 -0
- package/src/pacing/index.ts +5 -0
- package/src/pacing/interleave-timing.ts +95 -0
- package/src/video/index.ts +7 -0
- package/src/video/keyframe-gate.ts +35 -0
- package/src/video/video-render-pipeline.ts +179 -0
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Typed contract for AudioWorklet ↔ main-thread messages.
|
|
3
|
+
* Used by both the worklet processor (render-worklet.ts) and by consumers
|
|
4
|
+
* through audio-render-pipeline.ts.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
/** Main → worklet. */
|
|
8
|
+
export type FromMain =
|
|
9
|
+
| InitMessage
|
|
10
|
+
| DataMessage
|
|
11
|
+
| StartMessage
|
|
12
|
+
| LatencyMessage
|
|
13
|
+
| ResetMessage
|
|
14
|
+
| DiagRequestMessage
|
|
15
|
+
|
|
16
|
+
/** Worklet → main. */
|
|
17
|
+
export type ToMain = WriteAckMessage | StateMessage | DiagMessage
|
|
18
|
+
|
|
19
|
+
export interface InitMessage {
|
|
20
|
+
type: 'init'
|
|
21
|
+
/** Main-thread playout generation. State from older generations is stale. */
|
|
22
|
+
generation: number
|
|
23
|
+
/** Channel count. */
|
|
24
|
+
channels: number
|
|
25
|
+
/** Sample rate in Hz. */
|
|
26
|
+
rate: number
|
|
27
|
+
/** Buffer latency in ms. */
|
|
28
|
+
latencyMs: number
|
|
29
|
+
/** Exact PCM startup threshold derived from catalog delivery units. */
|
|
30
|
+
startupFrames?: number
|
|
31
|
+
/** Exact PCM storage ceiling, including one successor delivery burst. */
|
|
32
|
+
capacityFrames?: number
|
|
33
|
+
/** Start in unstalled state (source-switch reconfig). Default false. */
|
|
34
|
+
startUnstalled?: boolean
|
|
35
|
+
/** Buffer and acknowledge PCM without rendering until a matching start message. */
|
|
36
|
+
holdUntilStart?: boolean
|
|
37
|
+
/** Deprecated compatibility input. Clock recovery is not performed. */
|
|
38
|
+
warmupReads?: number
|
|
39
|
+
/** Enable telemetry reporting. Default false. */
|
|
40
|
+
diag?: boolean
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export interface DataMessage {
|
|
44
|
+
type: 'data'
|
|
45
|
+
/** Per-channel PCM (Float32) for the next chunk. */
|
|
46
|
+
data: Float32Array[]
|
|
47
|
+
/** Presentation timestamp of the first sample in µs. */
|
|
48
|
+
timestamp: number
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export interface WriteAckMessage {
|
|
52
|
+
type: 'writeAck'
|
|
53
|
+
/** Playout generation that accepted the PCM. */
|
|
54
|
+
generation: number
|
|
55
|
+
/** Exact unread PCM sample count after accepting the write. */
|
|
56
|
+
bufferedFrames: number
|
|
57
|
+
/** Cumulative PCM frames accepted since the last init/reset. */
|
|
58
|
+
writtenFrames: number
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export interface StartMessage {
|
|
62
|
+
type: 'start'
|
|
63
|
+
/** Playout generation to release. Stale start messages are ignored. */
|
|
64
|
+
generation: number
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export interface LatencyMessage {
|
|
68
|
+
type: 'latency'
|
|
69
|
+
/** New buffer latency in ms. */
|
|
70
|
+
latencyMs: number
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export interface ResetMessage {
|
|
74
|
+
type: 'reset'
|
|
75
|
+
/** Main-thread playout generation after this reset. */
|
|
76
|
+
generation: number
|
|
77
|
+
/** Start immediately instead of waiting for a full configured buffer. */
|
|
78
|
+
startUnstalled?: boolean
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export interface DiagRequestMessage {
|
|
82
|
+
type: 'requestDiag'
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export interface StateMessage {
|
|
86
|
+
type: 'state'
|
|
87
|
+
/** Playout generation that produced this state snapshot. */
|
|
88
|
+
generation: number
|
|
89
|
+
/** Timestamp of the next MMTP media sample after emitted PCM, in µs. */
|
|
90
|
+
timestamp?: number
|
|
91
|
+
/** Is the ring buffer waiting to fill? */
|
|
92
|
+
stalled: boolean
|
|
93
|
+
/** Exact unread PCM sample count. */
|
|
94
|
+
bufferedFrames: number
|
|
95
|
+
/** Cumulative PCM frames accepted since the last init/reset. */
|
|
96
|
+
writtenFrames: number
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export interface DiagMessage {
|
|
100
|
+
type: 'ringBufDiag'
|
|
101
|
+
/** Playout generation that produced this diagnostic snapshot. */
|
|
102
|
+
generation: number
|
|
103
|
+
reads: number
|
|
104
|
+
avgFill: number
|
|
105
|
+
capacity: number
|
|
106
|
+
skipsPerSec: number
|
|
107
|
+
dupsPerSec: number
|
|
108
|
+
overflowSamplesPerSec: number
|
|
109
|
+
underflowSamplesPerSec: number
|
|
110
|
+
}
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared AudioWorklet render processor.
|
|
3
|
+
*
|
|
4
|
+
* This file is bundled into a self-contained IIFE at build time
|
|
5
|
+
* (dist/worklet/render-worklet.js) because AudioWorkletGlobalScope cannot
|
|
6
|
+
* resolve bare specifiers. The AudioRingBuffer dependency is inlined at
|
|
7
|
+
* bundle time; do not import from @blockcast packages at runtime.
|
|
8
|
+
*
|
|
9
|
+
* Consumers load the prebuilt bundle via audioWorklet.addModule(url) where
|
|
10
|
+
* url is obtained from `@blockcast/mmt-render/audio/worklet-url`.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { AudioRingBuffer } from './audio-ring-buffer.js'
|
|
14
|
+
import type { FromMain, ToMain } from './render-messages.js'
|
|
15
|
+
|
|
16
|
+
class Render extends AudioWorkletProcessor {
|
|
17
|
+
#buffer?: AudioRingBuffer
|
|
18
|
+
#underflow = 0
|
|
19
|
+
#diagUnderflow = 0
|
|
20
|
+
#started = false
|
|
21
|
+
#stateCounter = 0
|
|
22
|
+
#diagEnabled = false
|
|
23
|
+
#diagCounter = 0
|
|
24
|
+
#generation = 0
|
|
25
|
+
#writtenFrames = 0
|
|
26
|
+
#heldUntilStart = false
|
|
27
|
+
#resumeAfterReset = false
|
|
28
|
+
|
|
29
|
+
constructor() {
|
|
30
|
+
super()
|
|
31
|
+
this.port.onmessage = (event: MessageEvent<FromMain>) => {
|
|
32
|
+
const msg = event.data
|
|
33
|
+
if (msg.type === 'init') {
|
|
34
|
+
const heldUntilStart = msg.holdUntilStart ?? false
|
|
35
|
+
const latencyFrames = Math.ceil(msg.rate * (msg.latencyMs / 1000))
|
|
36
|
+
const startupFrames = msg.startupFrames ?? latencyFrames
|
|
37
|
+
// Keep one latency window above the release threshold so PCM accepted
|
|
38
|
+
// during the writeAck -> start round trip cannot evict the prefill head.
|
|
39
|
+
const capacityFrames =
|
|
40
|
+
msg.capacityFrames ?? Math.max(latencyFrames, startupFrames) + latencyFrames
|
|
41
|
+
if (!Number.isSafeInteger(startupFrames) || startupFrames <= 0) {
|
|
42
|
+
throw new Error('invalid startup frames')
|
|
43
|
+
}
|
|
44
|
+
if (!Number.isSafeInteger(capacityFrames) || capacityFrames < startupFrames) {
|
|
45
|
+
throw new Error('invalid capacity frames')
|
|
46
|
+
}
|
|
47
|
+
this.#generation = msg.generation
|
|
48
|
+
this.#buffer = new AudioRingBuffer({
|
|
49
|
+
channels: msg.channels,
|
|
50
|
+
rate: msg.rate,
|
|
51
|
+
latencyMs: msg.latencyMs,
|
|
52
|
+
// The explicit start gate owns the one-time startup prefill. Once
|
|
53
|
+
// released, resume from an underrun as soon as new PCM is accepted.
|
|
54
|
+
startupFrames,
|
|
55
|
+
resumeFrames: heldUntilStart ? 1 : undefined,
|
|
56
|
+
capacityFrames,
|
|
57
|
+
startUnstalled: msg.startUnstalled,
|
|
58
|
+
warmupReads: msg.warmupReads,
|
|
59
|
+
diag: msg.diag ?? false,
|
|
60
|
+
})
|
|
61
|
+
this.#diagEnabled = msg.diag ?? false
|
|
62
|
+
this.#underflow = 0
|
|
63
|
+
this.#diagUnderflow = 0
|
|
64
|
+
this.#started = false
|
|
65
|
+
this.#writtenFrames = 0
|
|
66
|
+
this.#heldUntilStart = heldUntilStart
|
|
67
|
+
this.#resumeAfterReset = heldUntilStart
|
|
68
|
+
} else if (msg.type === 'data') {
|
|
69
|
+
if (!this.#buffer) return
|
|
70
|
+
this.#buffer.write(msg.timestamp, msg.data)
|
|
71
|
+
this.#writtenFrames += msg.data[0]?.length ?? 0
|
|
72
|
+
if (this.#heldUntilStart || this.#diagEnabled) {
|
|
73
|
+
this.port.postMessage({
|
|
74
|
+
type: 'writeAck',
|
|
75
|
+
generation: this.#generation,
|
|
76
|
+
bufferedFrames: this.#buffer.length,
|
|
77
|
+
writtenFrames: this.#writtenFrames,
|
|
78
|
+
})
|
|
79
|
+
}
|
|
80
|
+
} else if (msg.type === 'start') {
|
|
81
|
+
if (msg.generation === this.#generation) this.#heldUntilStart = false
|
|
82
|
+
} else if (msg.type === 'latency') {
|
|
83
|
+
if (!this.#buffer) return
|
|
84
|
+
this.#buffer.resize(msg.latencyMs)
|
|
85
|
+
} else if (msg.type === 'reset') {
|
|
86
|
+
this.#generation = msg.generation
|
|
87
|
+
this.#buffer?.reset(msg.startUnstalled ?? false, this.#resumeAfterReset)
|
|
88
|
+
this.#underflow = 0
|
|
89
|
+
this.#diagUnderflow = 0
|
|
90
|
+
this.#started = false
|
|
91
|
+
this.#writtenFrames = 0
|
|
92
|
+
this.#heldUntilStart = false
|
|
93
|
+
} else if (msg.type === 'requestDiag') {
|
|
94
|
+
this.#flushDiag()
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
process(_inputs: Float32Array[][], outputs: Float32Array[][]): boolean {
|
|
100
|
+
const output = outputs[0]
|
|
101
|
+
const samplesRead = this.#heldUntilStart ? 0 : (this.#buffer?.read(output) ?? 0)
|
|
102
|
+
|
|
103
|
+
if (samplesRead > 0) this.#started = true
|
|
104
|
+
if (samplesRead < output[0].length && this.#started) {
|
|
105
|
+
const missing = output[0].length - samplesRead
|
|
106
|
+
this.#underflow += missing
|
|
107
|
+
this.#diagUnderflow += missing
|
|
108
|
+
} else if (this.#underflow > 0 && this.#buffer) {
|
|
109
|
+
// eslint-disable-next-line no-console
|
|
110
|
+
console.warn(`audio underflow: ${Math.round((1000 * this.#underflow) / this.#buffer.rate)}ms`)
|
|
111
|
+
this.#underflow = 0
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
// State update ~60/sec (every 5 processes at 128-sample quantum @ 48kHz ≈ 2.67ms each)
|
|
115
|
+
this.#stateCounter++
|
|
116
|
+
if (this.#buffer && this.#stateCounter >= 5) {
|
|
117
|
+
this.#stateCounter = 0
|
|
118
|
+
const msg: ToMain = {
|
|
119
|
+
type: 'state',
|
|
120
|
+
generation: this.#generation,
|
|
121
|
+
timestamp: this.#heldUntilStart ? undefined : this.#buffer.timestampUs,
|
|
122
|
+
stalled: this.#heldUntilStart || this.#buffer.stalled,
|
|
123
|
+
bufferedFrames: this.#buffer.length,
|
|
124
|
+
writtenFrames: this.#writtenFrames,
|
|
125
|
+
}
|
|
126
|
+
this.port.postMessage(msg)
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// Diag telemetry every ~1s if enabled.
|
|
130
|
+
if (this.#diagEnabled && this.#buffer) {
|
|
131
|
+
this.#diagCounter++
|
|
132
|
+
if (this.#diagCounter >= 375) {
|
|
133
|
+
this.#diagCounter = 0
|
|
134
|
+
this.#flushDiag()
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
return true
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
#flushDiag(): void {
|
|
142
|
+
if (!this.#buffer) return
|
|
143
|
+
const d = this.#buffer.getDiagAndReset()
|
|
144
|
+
if (!d && this.#diagUnderflow === 0) return
|
|
145
|
+
const msg: ToMain = {
|
|
146
|
+
type: 'ringBufDiag',
|
|
147
|
+
generation: this.#generation,
|
|
148
|
+
reads: d?.reads ?? 0,
|
|
149
|
+
avgFill: d?.avgFill ?? 0,
|
|
150
|
+
capacity: d?.capacity ?? this.#buffer.capacity,
|
|
151
|
+
skipsPerSec: d?.skips ?? 0,
|
|
152
|
+
dupsPerSec: d?.dups ?? 0,
|
|
153
|
+
overflowSamplesPerSec: d?.overflowSamples ?? 0,
|
|
154
|
+
underflowSamplesPerSec: this.#diagUnderflow,
|
|
155
|
+
}
|
|
156
|
+
this.#diagUnderflow = 0
|
|
157
|
+
this.port.postMessage(msg)
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
registerProcessor('mmt-render', Render)
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Helper for loading the prebuilt AudioWorklet bundle.
|
|
3
|
+
*
|
|
4
|
+
* Two equivalent loading patterns supported:
|
|
5
|
+
*
|
|
6
|
+
* 1. Vite-style static URL (recommended for bundled apps):
|
|
7
|
+
* import workletUrl from '@blockcast/mmt-render/audio/worklet.js?url'
|
|
8
|
+
* await ctx.audioWorklet.addModule(workletUrl)
|
|
9
|
+
* Vite resolves the `?url` suffix + the package's "./audio/worklet.js"
|
|
10
|
+
* export to the prebuilt IIFE file.
|
|
11
|
+
*
|
|
12
|
+
* 2. Runtime import.meta.url resolution (no bundler):
|
|
13
|
+
* import { workletModuleUrl } from '@blockcast/mmt-render/audio/worklet-url'
|
|
14
|
+
* await ctx.audioWorklet.addModule(workletModuleUrl())
|
|
15
|
+
*
|
|
16
|
+
* Both resolve to dist/worklet/render-worklet.js.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Returns a URL (or URL string) pointing to the prebuilt worklet bundle.
|
|
21
|
+
* Resolves relative to this module's location via import.meta.url.
|
|
22
|
+
*/
|
|
23
|
+
export function workletModuleUrl(): string {
|
|
24
|
+
// dist layout: dist/audio/worklet-url.js → dist/worklet/render-worklet.js
|
|
25
|
+
const base = new URL('../worklet/render-worklet.js', import.meta.url)
|
|
26
|
+
return base.href
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Processor name registered in the worklet bundle. */
|
|
30
|
+
export const WORKLET_PROCESSOR_NAME = 'mmt-render'
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @blockcast/mmt-render — shared MMT render pipeline for MoQ players.
|
|
3
|
+
*
|
|
4
|
+
* Subpath exports:
|
|
5
|
+
* - ./pacing FrameCountPacer, InterleaveTiming
|
|
6
|
+
* - ./audio AudioRenderPipeline, AudioRingBuffer, worklet helpers
|
|
7
|
+
* - ./video VideoRenderPipeline, KeyframeGate
|
|
8
|
+
* - ./audio/worklet.js Prebuilt self-contained AudioWorklet bundle
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
export * from './pacing/index.js'
|
|
12
|
+
export * from './audio/index.js'
|
|
13
|
+
export * from './video/index.js'
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Nominal Access Unit duration helpers for common audio + video codecs.
|
|
3
|
+
*
|
|
4
|
+
* Kept in the pacing subpath (no mmt-container dep) so lean consumers like
|
|
5
|
+
* moqtail's player.ts can import the pacer + AU duration without pulling
|
|
6
|
+
* in the full audio-render-pipeline (which depends on @blockcast/mmt-container).
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Nominal AU duration in ms for common audio codecs at the given sample rate.
|
|
11
|
+
* AAC-LC: 1024 samples. Opus (default 20ms): 960 samples @ 48kHz.
|
|
12
|
+
* Throws for unknown codecs so callers supply correct timing.
|
|
13
|
+
*/
|
|
14
|
+
export function auDurationMsFromCodec(codec: string, sampleRate: number): number {
|
|
15
|
+
if (!(sampleRate > 0)) throw new Error('sampleRate must be > 0')
|
|
16
|
+
if (codec.startsWith('opus') || codec.startsWith('Opus')) {
|
|
17
|
+
return (960 / sampleRate) * 1000
|
|
18
|
+
}
|
|
19
|
+
if (codec.startsWith('mp4a') || codec.startsWith('AAC') || codec.startsWith('aac')) {
|
|
20
|
+
return (1024 / sampleRate) * 1000
|
|
21
|
+
}
|
|
22
|
+
if (codec.startsWith('flac') || codec.startsWith('fLaC')) {
|
|
23
|
+
return (4096 / sampleRate) * 1000
|
|
24
|
+
}
|
|
25
|
+
throw new Error(`unknown audio codec for AU duration derivation: ${codec}`)
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Nominal picture duration in ms from framerate.
|
|
30
|
+
*/
|
|
31
|
+
export function pictureDurationMs(framerate: number): number {
|
|
32
|
+
if (!(framerate > 0)) throw new Error('framerate must be > 0')
|
|
33
|
+
return 1000 / framerate
|
|
34
|
+
}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* FrameCountPacer — media-agnostic wall-clock pacer for MMT A/V decode loops.
|
|
3
|
+
*
|
|
4
|
+
* Why not use MPU timestamps? Under FEC interleaving, encoders pack multiple
|
|
5
|
+
* Access Units (AUs) per MMTP MPU with a single mpu_timestamp. The observed
|
|
6
|
+
* timestamp delta between MPUs is a fraction of the real per-AU duration
|
|
7
|
+
* (e.g. 10.22ms between MPUs for 21.33ms AAC frames at D=2 interleave).
|
|
8
|
+
* Pacing on that delta sees "media ≈ wall" and never sleeps, letting burst
|
|
9
|
+
* relay delivery saturate the decoder / ring buffer.
|
|
10
|
+
*
|
|
11
|
+
* The Sony libatsc3 pattern (atsc3_mmt_context_mfu_depacketizer) solves this
|
|
12
|
+
* by anchoring wall-clock time to the first received AU and counting AUs:
|
|
13
|
+
* media_elapsed_ms = au_count × nominal_au_duration_ms
|
|
14
|
+
* The MPU timestamp is used elsewhere as an absolute-time anchor, never as
|
|
15
|
+
* an incremental clock.
|
|
16
|
+
*
|
|
17
|
+
* The sleep threshold comes from the catalog's FEC interleaveDepthMs (via
|
|
18
|
+
* FecBlockMapper.pacingThresholdMs). When no FEC is configured, a
|
|
19
|
+
* NullBlockMapper yields threshold=0 and this pacer never sleeps.
|
|
20
|
+
*
|
|
21
|
+
* One pacer instance = one track (e.g. audio, video) on one player.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import type { FecBlockMapper } from '@blockcast/mmt-fec'
|
|
25
|
+
|
|
26
|
+
export interface FrameCountPacerOpts {
|
|
27
|
+
/** Required: AU duration (audio) or picture interval (video), in ms. */
|
|
28
|
+
frameDurationMs: number
|
|
29
|
+
/** FEC mapper from @blockcast/mmt-fec. NullBlockMapper disables pacing. */
|
|
30
|
+
mapper: FecBlockMapper
|
|
31
|
+
/**
|
|
32
|
+
* Number of frames to let through before pacing starts. Lets startup
|
|
33
|
+
* burst drain populate the ring buffer / decoder queue. Default 0.
|
|
34
|
+
*/
|
|
35
|
+
warmupFrames?: number
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface PacerSnapshot {
|
|
39
|
+
/** Frames observed (from notifyFrame). */
|
|
40
|
+
frameCount: number
|
|
41
|
+
/** Computed media time: frameCount × frameDurationMs. */
|
|
42
|
+
mediaMs: number
|
|
43
|
+
/** Wall-clock time since anchor. */
|
|
44
|
+
wallMs: number
|
|
45
|
+
/** mediaMs - wallMs. Positive → decoder ahead of realtime. */
|
|
46
|
+
aheadMs: number
|
|
47
|
+
/** Pacing threshold (ms) from the FEC mapper. */
|
|
48
|
+
thresholdMs: number
|
|
49
|
+
/** Has the first frame been observed (wall anchor set)? */
|
|
50
|
+
anchored: boolean
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export class FrameCountPacer {
|
|
54
|
+
readonly frameDurationMs: number
|
|
55
|
+
readonly thresholdMs: number
|
|
56
|
+
readonly warmupFrames: number
|
|
57
|
+
|
|
58
|
+
#frameCount = 0
|
|
59
|
+
#anchorWallMs = 0
|
|
60
|
+
#anchored = false
|
|
61
|
+
|
|
62
|
+
constructor(opts: FrameCountPacerOpts) {
|
|
63
|
+
if (!(opts.frameDurationMs > 0)) {
|
|
64
|
+
throw new Error('frameDurationMs must be > 0')
|
|
65
|
+
}
|
|
66
|
+
this.frameDurationMs = opts.frameDurationMs
|
|
67
|
+
this.thresholdMs = opts.mapper.pacingThresholdMs
|
|
68
|
+
this.warmupFrames = Math.max(0, opts.warmupFrames ?? 0)
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Call on every emitted decoded frame (AU or picture). Anchors on first call. */
|
|
72
|
+
notifyFrame(): void {
|
|
73
|
+
if (!this.#anchored) {
|
|
74
|
+
this.#anchorWallMs = now()
|
|
75
|
+
this.#anchored = true
|
|
76
|
+
}
|
|
77
|
+
this.#frameCount++
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Compute sleep ms before reading the next MoQ object.
|
|
82
|
+
* Returns 0 when:
|
|
83
|
+
* - pacing is disabled (NullBlockMapper → threshold=0)
|
|
84
|
+
* - the anchor is not yet set
|
|
85
|
+
* - still in warmup
|
|
86
|
+
* - decoder is not ahead of wall clock by more than threshold
|
|
87
|
+
* Otherwise returns aheadMs - thresholdMs/2 (leaves half the threshold as headroom).
|
|
88
|
+
*/
|
|
89
|
+
computeSleepMs(): number {
|
|
90
|
+
if (this.thresholdMs <= 0) return 0
|
|
91
|
+
if (!this.#anchored) return 0
|
|
92
|
+
if (this.#frameCount <= this.warmupFrames) return 0
|
|
93
|
+
|
|
94
|
+
const mediaMs = this.#frameCount * this.frameDurationMs
|
|
95
|
+
const wallMs = now() - this.#anchorWallMs
|
|
96
|
+
const aheadMs = mediaMs - wallMs
|
|
97
|
+
if (aheadMs <= this.thresholdMs) return 0
|
|
98
|
+
return aheadMs - this.thresholdMs / 2
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** Reset anchor + counter. Call on source switch (MoQ ↔ multicast). */
|
|
102
|
+
reset(): void {
|
|
103
|
+
this.#frameCount = 0
|
|
104
|
+
this.#anchorWallMs = 0
|
|
105
|
+
this.#anchored = false
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
get snapshot(): PacerSnapshot {
|
|
109
|
+
const mediaMs = this.#frameCount * this.frameDurationMs
|
|
110
|
+
const wallMs = this.#anchored ? now() - this.#anchorWallMs : 0
|
|
111
|
+
return {
|
|
112
|
+
frameCount: this.#frameCount,
|
|
113
|
+
mediaMs,
|
|
114
|
+
wallMs,
|
|
115
|
+
aheadMs: mediaMs - wallMs,
|
|
116
|
+
thresholdMs: this.thresholdMs,
|
|
117
|
+
anchored: this.#anchored,
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function now(): number {
|
|
123
|
+
// performance.now() is available in main thread, workers, and AudioWorklet.
|
|
124
|
+
// Fall back to Date.now() for exotic runtimes (shouldn't happen in browsers).
|
|
125
|
+
if (typeof performance !== 'undefined' && typeof performance.now === 'function') {
|
|
126
|
+
return performance.now()
|
|
127
|
+
}
|
|
128
|
+
return Date.now()
|
|
129
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export { FrameCountPacer } from './frame-pacer.js'
|
|
2
|
+
export type { FrameCountPacerOpts, PacerSnapshot } from './frame-pacer.js'
|
|
3
|
+
export { InterleaveTiming } from './interleave-timing.js'
|
|
4
|
+
export type { InterleaveTimingOpts } from './interleave-timing.js'
|
|
5
|
+
export { auDurationMsFromCodec, pictureDurationMs } from './au-duration.js'
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* InterleaveTiming — per-AU presentation-time reconstruction.
|
|
3
|
+
*
|
|
4
|
+
* When encoders pack multiple AUs per MMTP MPU with a single mpu_timestamp
|
|
5
|
+
* (normal for FEC-interleaved audio, and for video with multi-picture MPUs),
|
|
6
|
+
* the raw mpu_timestamp is ambiguous for per-AU scheduling. The FEC block
|
|
7
|
+
* math (draft-ramadan-moq-fec-00 §8.3) combined with the Sony libatsc3
|
|
8
|
+
* pattern recovers correct per-AU time:
|
|
9
|
+
*
|
|
10
|
+
* au_index_in_block = groupInBlock × symbolsPerGroup + objectId
|
|
11
|
+
* ssId = sbn × K + au_index_in_block
|
|
12
|
+
* presentationUs = anchorMpuUs + ssId × frameDurationUs
|
|
13
|
+
*
|
|
14
|
+
* The anchor (anchorMpuUs) is the mpu_timestamp of the first observed AU.
|
|
15
|
+
* Every downstream per-AU time is then deterministic from FEC coordinates
|
|
16
|
+
* + codec frame duration — identical whether the AU arrives via MoQ unicast
|
|
17
|
+
* (group_id/object_id) or multicast UDP (SS_ID from FEC Payload ID).
|
|
18
|
+
*
|
|
19
|
+
* Consumers: CMAF segment assembler (block boundary → segment cut points),
|
|
20
|
+
* debug overlays, cross-path FEC combining, sync diagnostics.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import type { FecBlockMapper } from '@blockcast/mmt-fec'
|
|
24
|
+
|
|
25
|
+
export interface InterleaveTimingOpts {
|
|
26
|
+
/** Track's FEC mapper (provides K, D, symbolsPerGroup). */
|
|
27
|
+
mapper: FecBlockMapper
|
|
28
|
+
/** Nominal AU duration in microseconds. */
|
|
29
|
+
frameDurationUs: number
|
|
30
|
+
/** Wall-time anchor: the mpu_timestamp of the first observed AU (µs). */
|
|
31
|
+
anchorMpuUs: number
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export class InterleaveTiming {
|
|
35
|
+
readonly mapper: FecBlockMapper
|
|
36
|
+
readonly frameDurationUs: number
|
|
37
|
+
readonly anchorMpuUs: number
|
|
38
|
+
|
|
39
|
+
constructor(opts: InterleaveTimingOpts) {
|
|
40
|
+
if (!(opts.frameDurationUs > 0)) {
|
|
41
|
+
throw new Error('frameDurationUs must be > 0')
|
|
42
|
+
}
|
|
43
|
+
if (!Number.isFinite(opts.anchorMpuUs)) {
|
|
44
|
+
throw new Error('anchorMpuUs must be finite')
|
|
45
|
+
}
|
|
46
|
+
this.mapper = opts.mapper
|
|
47
|
+
this.frameDurationUs = opts.frameDurationUs
|
|
48
|
+
this.anchorMpuUs = opts.anchorMpuUs
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Presentation time (µs) for an AU at MoQ coords (group_id, object_id).
|
|
53
|
+
* Per draft §8.3 + Sony anchor.
|
|
54
|
+
*/
|
|
55
|
+
presentationUs(groupId: number, objectId: number): number {
|
|
56
|
+
const coords = this.mapper.derive(groupId, objectId)
|
|
57
|
+
return this.anchorMpuUs + coords.ssId * this.frameDurationUs
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Presentation time (µs) for an AU identified by its flat SS_ID
|
|
62
|
+
* (multicast FEC Payload ID path).
|
|
63
|
+
*/
|
|
64
|
+
presentationUsFromSsId(ssId: number): number {
|
|
65
|
+
return this.anchorMpuUs + ssId * this.frameDurationUs
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Inverse: which (group_id, object_id) AU is scheduled at the given
|
|
70
|
+
* wall offset (µs) from anchor? Clamps to ssId≥0.
|
|
71
|
+
*/
|
|
72
|
+
frameAtOffset(offsetUs: number): { groupId: number; objectId: number; ssId: number } {
|
|
73
|
+
const rawSsId = Math.max(0, Math.floor(offsetUs / this.frameDurationUs))
|
|
74
|
+
const { sbn, esi } = this.mapper.fromSsId(rawSsId)
|
|
75
|
+
// Invert §8.3: esi = groupInBlock * symbolsPerGroup + objectId
|
|
76
|
+
const spg = this.mapper.symbolsPerGroup
|
|
77
|
+
const groupInBlock = Math.floor(esi / spg)
|
|
78
|
+
const objectId = esi - groupInBlock * spg
|
|
79
|
+
const groupId = sbn * this.mapper.d + groupInBlock
|
|
80
|
+
return { groupId, objectId, ssId: rawSsId }
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Block start/end in microseconds. Source blocks span `D × GOP_duration`
|
|
85
|
+
* of wall time; this returns the anchored wall range [startUs, endUs).
|
|
86
|
+
*/
|
|
87
|
+
blockTimeRangeUs(sbn: number): { startUs: number; endUs: number } {
|
|
88
|
+
const firstSsId = sbn * this.mapper.k
|
|
89
|
+
const lastSsId = (sbn + 1) * this.mapper.k
|
|
90
|
+
return {
|
|
91
|
+
startUs: this.anchorMpuUs + firstSsId * this.frameDurationUs,
|
|
92
|
+
endUs: this.anchorMpuUs + lastSsId * this.frameDurationUs,
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* KeyframeGate — drops delta frames until a keyframe arrives.
|
|
3
|
+
*
|
|
4
|
+
* Required when subscribing mid-stream: the relay may deliver P/B frames
|
|
5
|
+
* from the middle of a GOP before the first IDR. Feeding those into
|
|
6
|
+
* VideoDecoder before an init yields "key frame required" errors.
|
|
7
|
+
*
|
|
8
|
+
* Call reset() to re-arm the gate after a decoder error (requestKeyframe).
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
export class KeyframeGate {
|
|
12
|
+
#open = false
|
|
13
|
+
|
|
14
|
+
get isOpen(): boolean {
|
|
15
|
+
return this.#open
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Decide whether to pass a frame through.
|
|
20
|
+
* Opens permanently once the first keyframe is seen.
|
|
21
|
+
*/
|
|
22
|
+
allow(keyframe: boolean): boolean {
|
|
23
|
+
if (this.#open) return true
|
|
24
|
+
if (keyframe) {
|
|
25
|
+
this.#open = true
|
|
26
|
+
return true
|
|
27
|
+
}
|
|
28
|
+
return false
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Re-close the gate. Next keyframe reopens it. */
|
|
32
|
+
reset(): void {
|
|
33
|
+
this.#open = false
|
|
34
|
+
}
|
|
35
|
+
}
|