@blockcast/mmt-render 0.1.0-main.5dd8358ae902

package/src/audio/audio-render-pipeline.ts

@@ -0,0 +1,313 @@
+/**
+ * AudioRenderPipeline — end-to-end audio render for MoQ/MMT players.
+ *
+ * Pipeline: MoQ object → MmtpDecodePipeline → WebCodecs AudioDecoder
+ *           → AudioWorkletNode (exact-timestamp FIFO)
+ *
+ * The consumer:
+ *   1. constructs with codec/sampleRate/channels/mapper from catalog
+ *   2. awaits ready()
+ *   3. calls feedMoqObject(packet) for each arriving MoQ object
+ *   4. between objects, awaits paceSleepMs() to throttle burst delivery
+ *   5. connects rootNode to an AudioContext destination (or a GainNode)
+ *   6. close() on teardown
+ *
+ * Pacing uses FrameCountPacer (anchor on first decoded AU, frame count × AU
+ * duration) with threshold = mapper.pacingThresholdMs — NOT MPU timestamps
+ * (ambiguous under FEC interleave).
+ */
+
+import { MmtpDecodePipeline, type PipelineFecClient } from '@blockcast/mmt-container/mmtp-decode-pipeline'
+import type { MfuReassemblyPolicy } from '@blockcast/mmt-container/mfu-reassembly-policy'
+import type { FecBlockMapper } from '@blockcast/mmt-fec'
+import { FrameCountPacer } from '../pacing/frame-pacer.js'
+import { auDurationMsFromCodec } from '../pacing/au-duration.js'
+import type {
+  ToMain,
+  DataMessage,
+  InitMessage,
+  ResetMessage,
+  StartMessage,
+} from './render-messages.js'
+import { WORKLET_PROCESSOR_NAME, workletModuleUrl } from './worklet-url.js'
+
+export interface AudioRenderPipelineOpts {
+  context: AudioContext
+  codec: string
+  /** MMTP packet_id for this audio track — from catalog. */
+  packetId?: number
+  sampleRate: number
+  channels: number
+  description?: Uint8Array
+  mapper: FecBlockMapper
+  fecClient?: PipelineFecClient
+  /** Reliable MoQ is clock-free; datagram deadlines come from catalog/in-band timing. */
+  reassemblyPolicy: MfuReassemblyPolicy
+  /** Catalog-derived PCM startup buffer in milliseconds. */
+  latencyMs: number
+  /** Exact PCM startup threshold when catalog grouping is codec-aligned. */
+  startupFrames?: number
+  /** Exact PCM storage ceiling, including one successor delivery burst. */
+  capacityFrames?: number
+  /** Enable ring-buffer telemetry messages. Default false. */
+  diag?: boolean
+  /** Called when worklet reports state (playback timestamp + exact buffer accounting). Optional. */
+  onState?: (state: {
+    timestampUs: number | undefined
+    stalled: boolean
+    bufferedFrames: number
+    writtenFrames: number
+  }) => void
+  /** Called when worklet reports ring-buffer diag (telemetry). Optional. */
+  onDiag?: (diag: {
+    reads: number; avgFill: number; capacity: number
+    skipsPerSec: number; dupsPerSec: number; overflowSamplesPerSec: number
+    underflowSamplesPerSec: number
+  }) => void
+  /** Warmup frames before pacing engages. Default 0 (audio tolerates instant pacing). */
+  warmupFrames?: number
+  /** Audio underflow / config event logger. Optional. */
+  onError?: (err: unknown) => void
+}
+
+export class AudioRenderPipeline {
+  readonly #context: AudioContext
+  readonly #pacer: FrameCountPacer
+  readonly #mapper: FecBlockMapper
+  readonly #codec: string
+  readonly #packetId: number | undefined
+  readonly #sampleRate: number
+  readonly #channels: number
+  readonly #description?: Uint8Array
+  readonly #latencyMs: number
+  readonly #startupFrames?: number
+  readonly #startupThresholdFrames: number
+  readonly #capacityFrames?: number
+  readonly #diag: boolean
+  readonly #warmupFrames: number
+  readonly #onState?: AudioRenderPipelineOpts['onState']
+  readonly #onDiag?: AudioRenderPipelineOpts['onDiag']
+  readonly #onError?: AudioRenderPipelineOpts['onError']
+  readonly #reassemblyPolicy: MfuReassemblyPolicy
+
+  #decoder: AudioDecoder | null = null
+  #pipeline: MmtpDecodePipeline | null = null
+  #worklet: AudioWorkletNode | null = null
+  #fecClient?: PipelineFecClient
+  #closed = false
+  #readyPromise: Promise<void> | null = null
+  #workletGeneration = 0
+  #startupReleased = false
+
+  constructor(opts: AudioRenderPipelineOpts) {
+    this.#context = opts.context
+    this.#mapper = opts.mapper
+    this.#codec = opts.codec
+    this.#packetId = opts.packetId
+    this.#sampleRate = opts.sampleRate
+    this.#channels = opts.channels
+    this.#description = opts.description
+    this.#latencyMs = opts.latencyMs
+    this.#startupFrames = opts.startupFrames
+    this.#startupThresholdFrames = opts.startupFrames
+      ?? Math.ceil(opts.sampleRate * (opts.latencyMs / 1000))
+    this.#capacityFrames = opts.capacityFrames
+    this.#diag = opts.diag ?? false
+    this.#warmupFrames = opts.warmupFrames ?? 0
+    this.#onState = opts.onState
+    this.#onDiag = opts.onDiag
+    this.#onError = opts.onError
+    this.#fecClient = opts.fecClient
+    this.#reassemblyPolicy = opts.reassemblyPolicy
+
+    // AU duration from codec + sampleRate. Keep inside the lib so consumers
+    // don't re-derive it.
+    const auDurationMs = auDurationMsFromCodec(opts.codec, opts.sampleRate)
+    this.#pacer = new FrameCountPacer({
+      frameDurationMs: auDurationMs,
+      mapper: opts.mapper,
+      warmupFrames: this.#warmupFrames,
+    })
+  }
+
+  /** Load worklet module + create decoder + pipeline. Call once. */
+  async ready(): Promise<void> {
+    if (this.#readyPromise) return this.#readyPromise
+    this.#readyPromise = this.#initialize()
+    return this.#readyPromise
+  }
+
+  async #initialize(): Promise<void> {
+    // 1. Load shared worklet module.
+    await this.#context.audioWorklet.addModule(workletModuleUrl())
+
+    // 2. Create worklet node.
+    const worklet = new AudioWorkletNode(this.#context, WORKLET_PROCESSOR_NAME, {
+      numberOfInputs: 0,
+      numberOfOutputs: 1,
+      outputChannelCount: [this.#channels],
+    })
+    const initMsg: InitMessage = {
+      type: 'init',
+      generation: ++this.#workletGeneration,
+      channels: this.#channels,
+      rate: this.#sampleRate,
+      latencyMs: this.#latencyMs,
+      startupFrames: this.#startupFrames,
+      capacityFrames: this.#capacityFrames,
+      holdUntilStart: true,
+      diag: this.#diag,
+    }
+    worklet.port.postMessage(initMsg)
+    worklet.port.onmessage = (ev: MessageEvent<ToMain>) => {
+      const m = ev.data
+      if (m.type === 'state') {
+        if (m.generation !== this.#workletGeneration) return
+        this.#onState?.({
+          timestampUs: m.timestamp,
+          stalled: m.stalled,
+          bufferedFrames: m.bufferedFrames,
+          writtenFrames: m.writtenFrames,
+        })
+      } else if (m.type === 'ringBufDiag') {
+        if (m.generation !== this.#workletGeneration) return
+        this.#onDiag?.({
+          reads: m.reads, avgFill: m.avgFill, capacity: m.capacity,
+          skipsPerSec: m.skipsPerSec, dupsPerSec: m.dupsPerSec,
+          overflowSamplesPerSec: m.overflowSamplesPerSec,
+          underflowSamplesPerSec: m.underflowSamplesPerSec,
+        })
+      } else if (m.type === 'writeAck') {
+        if (
+          m.generation !== this.#workletGeneration ||
+          this.#startupReleased ||
+          m.bufferedFrames < this.#startupThresholdFrames
+        ) return
+        this.#startupReleased = true
+        const start: StartMessage = {
+          type: 'start',
+          generation: this.#workletGeneration,
+        }
+        worklet.port.postMessage(start)
+      }
+    }
+    this.#worklet = worklet
+
+    // 3. Create decoder.
+    const decoder = new AudioDecoder({
+      output: (data) => this.#onDecoded(data),
+      error: (err) => this.#onError?.(err),
+    })
+    const config: AudioDecoderConfig = {
+      codec: this.#codec,
+      sampleRate: this.#sampleRate,
+      numberOfChannels: this.#channels,
+      ...(this.#description ? { description: this.#description } : {}),
+    }
+    decoder.configure(config)
+    this.#decoder = decoder
+
+    // 4. Create decode pipeline.
+    const pipeline = new MmtpDecodePipeline()
+    pipeline.configure({
+      codec: this.#codec,
+      packetId: this.#packetId,
+      description: this.#description,
+      reorderWindow: 0, // audio doesn't need reorder — AAC/Opus frames are self-contained
+      reassemblyPolicy: this.#reassemblyPolicy,
+    })
+    if (this.#fecClient) pipeline.setFecClient(this.#fecClient)
+    pipeline.onFrame((codecData, meta) => {
+      if (!this.#decoder || this.#decoder.state !== 'configured') return
+      if (codecData.byteLength === 0) return
+      this.#pacer.notifyFrame()
+      try {
+        this.#decoder.decode(new EncodedAudioChunk({
+          type: 'key',
+          data: codecData,
+          timestamp: meta.timestamp,
+        }))
+      } catch (err) {
+        this.#onError?.(err)
+      }
+    })
+    this.#pipeline = pipeline
+  }
+
+  /** Feed one MoQ object (raw MMTP packet). */
+  feedMoqObject(packet: Uint8Array): void {
+    this.#pipeline?.feedMoqObject(packet)
+  }
+
+  /** ms to sleep before the next object read. 0 when not anchored / not ahead. */
+  paceSleepMs(): number {
+    return this.#pacer.computeSleepMs()
+  }
+
+  /** Pacer snapshot (for diagnostics / UI). */
+  get pacerSnapshot() {
+    return this.#pacer.snapshot
+  }
+
+  /** Reset pacer on source switch (MoQ ↔ multicast). */
+  resetPacer(): void {
+    this.#pacer.reset()
+  }
+
+  /** Drop queued PCM and its timestamp epoch at an explicit source boundary. */
+  resetPlayout(startUnstalled = false): void {
+    this.#pacer.reset()
+    // The explicit gate owns only the initial prefill. Source-boundary resets
+    // use the worklet's one-frame resume threshold.
+    this.#startupReleased = true
+    const message: ResetMessage = {
+      type: 'reset',
+      generation: ++this.#workletGeneration,
+      startUnstalled,
+    }
+    this.#worklet?.port.postMessage(message)
+  }
+
+  /** Node to connect to destination / gain. */
+  get rootNode(): AudioNode {
+    if (!this.#worklet) throw new Error('AudioRenderPipeline.ready() must resolve first')
+    return this.#worklet
+  }
+
+  close(): void {
+    if (this.#closed) return
+    this.#closed = true
+    try { this.#decoder?.close() } catch { /* already closed */ }
+    try { this.#pipeline?.dispose() } catch { /* noop */ }
+    try { this.#worklet?.disconnect() } catch { /* noop */ }
+    this.#decoder = null
+    this.#pipeline = null
+    this.#worklet = null
+  }
+
+  // ─── Private ─────────────────────────────────────────────────────
+
+  #onDecoded(sample: AudioData): void {
+    if (this.#closed || !this.#worklet) { sample.close(); return }
+
+    const timestamp = sample.timestamp
+    const channelData: Float32Array[] = []
+    for (let ch = 0; ch < sample.numberOfChannels; ch++) {
+      const data = new Float32Array(sample.numberOfFrames)
+      sample.copyTo(data, { format: 'f32-planar', planeIndex: ch })
+      channelData.push(data)
+    }
+    sample.close()
+
+    const msg: DataMessage = {
+      type: 'data',
+      data: channelData,
+      timestamp,
+    }
+    this.#worklet.port.postMessage(msg, channelData.map((d) => d.buffer))
+  }
+}
+
+// auDurationMsFromCodec lives in ../pacing/au-duration.ts (canonical, no
+// mmt-container dep). Import it from '@blockcast/mmt-render/pacing' directly
+// or via the top-level '@blockcast/mmt-render' export.

package/src/audio/audio-ring-buffer.ts

@@ -0,0 +1,281 @@
+/**
+ * Timestamp-anchored FIFO shared by MMT AudioWorklet consumers.
+ *
+ * PCM remains in decoder emission order. Media timestamps are exact anchors
+ * for playout reporting only; they never reposition samples or trigger a
+ * guessed discontinuity. The FIFO starts after its full catalog-derived
+ * latency is buffered and performs no sample skip/dup clock recovery.
+ */
+
+export interface AudioRingBufferConfig {
+  channels: number
+  rate: number
+  latencyMs: number
+  /** Exact PCM startup threshold. Overrides latencyMs when provided. */
+  startupFrames?: number
+  /** PCM refill threshold after the first underrun. Defaults to startupFrames. */
+  resumeFrames?: number
+  /** Exact PCM storage ceiling. Must cover the startup and resume thresholds. */
+  capacityFrames?: number
+  startUnstalled?: boolean
+  /** Deprecated compatibility input. Clock recovery is not performed. */
+  warmupReads?: number
+  diag?: boolean
+}
+
+export interface AudioRingBufferDiag {
+  reads: number
+  avgFill: number
+  capacity: number
+  skips: number
+  dups: number
+  overflowSamples: number
+}
+
+type TimestampAnchor = {
+  sampleIndex: number
+  timestampUs: number
+}
+
+export class AudioRingBuffer {
+  readonly channels: number
+  readonly rate: number
+
+  #buffer: Float32Array[]
+  #writeIndex = 0
+  #readIndex = 0
+  #stalled: boolean
+  #startupFrames: number
+  #resumeFrames: number
+  #resumeFollowsStartup: boolean
+  #hasStarted: boolean
+  #capacityHeadroomFrames: number
+  #timestampAnchors: TimestampAnchor[] = []
+
+  #diagEnabled: boolean
+  #diagReads = 0
+  #diagFillSum = 0
+  #diagOverflowSamples = 0
+
+  constructor(config: AudioRingBufferConfig) {
+    if (!Number.isInteger(config.channels) || config.channels <= 0) {
+      throw new Error('invalid channels')
+    }
+    if (!Number.isFinite(config.rate) || config.rate <= 0) {
+      throw new Error('invalid sample rate')
+    }
+    if (!Number.isFinite(config.latencyMs) || config.latencyMs <= 0) {
+      throw new Error('invalid latency')
+    }
+
+    const latencyFrames = Math.ceil(config.rate * (config.latencyMs / 1000))
+    const startupFrames = config.startupFrames ?? latencyFrames
+    const resumeFrames = config.resumeFrames ?? startupFrames
+    const thresholdFrames = Math.max(startupFrames, resumeFrames)
+    const capacityFrames = config.capacityFrames ?? Math.max(latencyFrames, thresholdFrames)
+    if (!Number.isSafeInteger(startupFrames) || startupFrames <= 0) {
+      throw new Error('invalid startup frames')
+    }
+    if (!Number.isSafeInteger(resumeFrames) || resumeFrames <= 0) {
+      throw new Error('invalid resume frames')
+    }
+    if (!Number.isSafeInteger(capacityFrames) || capacityFrames < thresholdFrames) {
+      throw new Error('invalid capacity frames')
+    }
+
+    this.channels = config.channels
+    this.rate = config.rate
+    this.#startupFrames = startupFrames
+    this.#resumeFrames = resumeFrames
+    this.#resumeFollowsStartup = config.resumeFrames === undefined
+    this.#hasStarted = config.startUnstalled ?? false
+    this.#capacityHeadroomFrames = capacityFrames - thresholdFrames
+    this.#stalled = !(config.startUnstalled ?? false)
+    this.#diagEnabled = config.diag ?? false
+    this.#buffer = Array.from(
+      { length: config.channels },
+      () => new Float32Array(capacityFrames),
+    )
+  }
+
+  get stalled(): boolean { return this.#stalled }
+  get startupFrames(): number { return this.#startupFrames }
+  get capacity(): number { return this.#buffer[0].length }
+  get length(): number { return this.#writeIndex - this.#readIndex }
+
+  /** Timestamp of the next media sample after the PCM already emitted. */
+  get timestampUs(): number | undefined {
+    const anchor = this.#timestampAnchors[0]
+    if (!anchor || anchor.sampleIndex > this.#readIndex) return undefined
+    return anchor.timestampUs + ((this.#readIndex - anchor.sampleIndex) / this.rate) * 1_000_000
+  }
+
+  write(timestampUs: number, data: Float32Array[]): void {
+    if (data.length !== this.channels) throw new Error('wrong channel count')
+    if (!Number.isFinite(timestampUs)) throw new Error('invalid timestamp')
+    const samples = data[0].length
+    if (samples === 0) return
+    if (data.some((channel) => channel.length !== samples)) {
+      throw new Error('mismatched channel lengths')
+    }
+
+    const start = this.#writeIndex
+    const end = start + samples
+    this.#timestampAnchors.push({ sampleIndex: start, timestampUs })
+
+    const overflow = end - this.#readIndex - this.capacity
+    if (overflow > 0) {
+      this.#stalled = false
+      this.#hasStarted = true
+      this.#readIndex += overflow
+      if (this.#diagEnabled) this.#diagOverflowSamples += overflow
+      this.#pruneTimestampAnchors()
+    }
+
+    const refillFrames = this.#hasStarted ? this.#resumeFrames : this.#startupFrames
+    if (this.#stalled && end - this.#readIndex >= refillFrames) {
+      this.#stalled = false
+      this.#hasStarted = true
+    }
+
+    for (let channel = 0; channel < this.channels; channel++) {
+      const src = data[channel]
+      const dst = this.#buffer[channel]
+      for (let i = 0; i < samples; i++) {
+        dst[(start + i) % dst.length] = src[i]
+      }
+    }
+    this.#writeIndex = end
+  }
+
+  read(output: Float32Array[]): number {
+    if (output.length !== this.channels) throw new Error('wrong channel count')
+    if (this.#stalled) return 0
+
+    const available = this.length
+    if (available === 0) {
+      this.#stalled = true
+      return 0
+    }
+
+    const requested = output[0].length
+    const readSamples = Math.min(available, requested)
+
+    for (let channel = 0; channel < this.channels; channel++) {
+      const dst = output[channel]
+      const src = this.#buffer[channel]
+      for (let i = 0; i < readSamples; i++) {
+        dst[i] = src[(this.#readIndex + i) % src.length]
+      }
+      for (let i = readSamples; i < requested; i++) dst[i] = 0
+    }
+
+    this.#readIndex += readSamples
+    this.#pruneTimestampAnchors()
+    if (readSamples < requested) this.#stalled = true
+
+    if (this.#diagEnabled) {
+      this.#diagReads++
+      this.#diagFillSum += available
+    }
+    return readSamples
+  }
+
+  /** Clear queued PCM; optionally treat the next fill as post-start recovery. */
+  reset(startUnstalled = false, resumeAfterReset = false): void {
+    this.#writeIndex = 0
+    this.#readIndex = 0
+    this.#stalled = !startUnstalled
+    this.#hasStarted = startUnstalled || resumeAfterReset
+    this.#timestampAnchors = []
+    this.#diagReads = 0
+    this.#diagFillSum = 0
+    this.#diagOverflowSamples = 0
+  }
+
+  /**
+   * Change the startup latency while preserving the configured storage
+   * headroom above its buffering thresholds. Active playout keeps draining
+   * retained PCM; a stalled buffer waits for its startup or resume threshold.
+   */
+  resize(latencyMs: number): void {
+    if (!Number.isFinite(latencyMs) || latencyMs <= 0) {
+      throw new Error('invalid latency')
+    }
+    const newStartupFrames = Math.ceil(this.rate * (latencyMs / 1000))
+    if (!Number.isSafeInteger(newStartupFrames) || newStartupFrames <= 0) {
+      throw new Error('invalid latency')
+    }
+    const newResumeFrames = this.#resumeFollowsStartup ? newStartupFrames : this.#resumeFrames
+    const newCapacity =
+      Math.max(newStartupFrames, newResumeFrames) + this.#capacityHeadroomFrames
+    if (
+      !Number.isSafeInteger(newCapacity) ||
+      newCapacity < Math.max(newStartupFrames, newResumeFrames)
+    ) {
+      throw new Error('invalid capacity frames')
+    }
+    const wasStalled = this.#stalled
+    this.#startupFrames = newStartupFrames
+    this.#resumeFrames = newResumeFrames
+    if (newCapacity === this.capacity) {
+      if (wasStalled && this.length >= this.#refillFrames()) {
+        this.#stalled = false
+        this.#hasStarted = true
+      }
+      return
+    }
+
+    const next = Array.from(
+      { length: this.channels },
+      () => new Float32Array(newCapacity),
+    )
+    const samplesToKeep = Math.min(this.length, newCapacity)
+    const copyStart = this.#writeIndex - samplesToKeep
+    for (let channel = 0; channel < this.channels; channel++) {
+      const src = this.#buffer[channel]
+      const dst = next[channel]
+      for (let i = 0; i < samplesToKeep; i++) {
+        const sampleIndex = copyStart + i
+        dst[sampleIndex % dst.length] = src[sampleIndex % src.length]
+      }
+    }
+
+    this.#buffer = next
+    this.#readIndex = copyStart
+    this.#pruneTimestampAnchors()
+    this.#stalled = samplesToKeep === 0 || (wasStalled && samplesToKeep < this.#refillFrames())
+    if (!this.#stalled) this.#hasStarted = true
+  }
+
+  getDiagAndReset(): AudioRingBufferDiag | null {
+    if (!this.#diagEnabled || this.#diagReads === 0) return null
+    const result: AudioRingBufferDiag = {
+      reads: this.#diagReads,
+      avgFill: Math.round(this.#diagFillSum / this.#diagReads),
+      capacity: this.capacity,
+      skips: 0,
+      dups: 0,
+      overflowSamples: this.#diagOverflowSamples,
+    }
+    this.#diagReads = 0
+    this.#diagFillSum = 0
+    this.#diagOverflowSamples = 0
+    return result
+  }
+
+  #pruneTimestampAnchors(): void {
+    let keep = 0
+    while (
+      keep + 1 < this.#timestampAnchors.length &&
+      this.#timestampAnchors[keep + 1].sampleIndex <= this.#readIndex
+    ) {
+      keep++
+    }
+    if (keep > 0) this.#timestampAnchors.splice(0, keep)
+  }
+
+  #refillFrames(): number {
+    return this.#hasStarted ? this.#resumeFrames : this.#startupFrames
+  }
+}

package/src/audio/index.ts

@@ -0,0 +1,14 @@
+export { AudioRingBuffer } from './audio-ring-buffer.js'
+export type { AudioRingBufferConfig, AudioRingBufferDiag } from './audio-ring-buffer.js'
+export { deriveAudioBurstBufferPlan } from './audio-buffer-plan.js'
+export type { AudioBurstBufferPlan, AudioBurstBufferPlanInput } from './audio-buffer-plan.js'
+export { AudioRenderPipeline } from './audio-render-pipeline.js'
+export type { AudioRenderPipelineOpts } from './audio-render-pipeline.js'
+// auDurationMsFromCodec is canonical at ../pacing/au-duration.ts; re-exported
+// from both pacing/index.ts and the top-level index. Not duplicated here to
+// avoid export conflicts in index.ts's `export *`.
+export type {
+  FromMain, ToMain, InitMessage, DataMessage, StartMessage, WriteAckMessage, LatencyMessage,
+  ResetMessage, DiagRequestMessage, StateMessage, DiagMessage,
+} from './render-messages.js'
+export { WORKLET_PROCESSOR_NAME, workletModuleUrl } from './worklet-url.js'