@blockcast/mmt-render 0.1.0-main.5dd8358ae902

package/src/video/video-render-pipeline.ts

@@ -0,0 +1,179 @@
+/**
+ * VideoRenderPipeline — end-to-end video render for MoQ/MMT players.
+ *
+ * Pipeline: MoQ object → MmtpDecodePipeline → keyframe gate → WebCodecs
+ *           VideoDecoder → consumer's onFrame(VideoFrame).
+ *
+ * Replaces:
+ *   - moqtail's `decodeQueueSize > 10 drop-deltas` hack in webcodecs-backend.ts
+ *   - hang-mmt-fec's "reorderWindow = 0 // TODO" in video/decoder.ts
+ *
+ * Pacing uses FrameCountPacer (anchor on first decoded picture, frame count ×
+ * 1/fps) with threshold = mapper.pacingThresholdMs. Reorder window comes from
+ * mapper.d (groups per FEC block), which is the natural bound for out-of-order
+ * delivery within a block after converting catalog groups to frame count.
+ */
+
+import { MmtpDecodePipeline, type PipelineFecClient } from '@blockcast/mmt-container/mmtp-decode-pipeline'
+import type { MfuReassemblyPolicy } from '@blockcast/mmt-container/mfu-reassembly-policy'
+import type { FecBlockMapper } from '@blockcast/mmt-fec'
+import { FrameCountPacer } from '../pacing/frame-pacer.js'
+import { KeyframeGate } from './keyframe-gate.js'
+
+export interface VideoFrameMeta {
+  timestamp: number
+  keyframe: boolean
+  packetId: number
+}
+
+export type VideoFrameCallback = (frame: VideoFrame, meta: VideoFrameMeta) => void
+
+export function deriveVideoReorderWindow(
+  framerate: number,
+  mapper: Pick<FecBlockMapper, 'd' | 'gopDurationMs'>,
+): number {
+  const framesPerGroup = Math.ceil(framerate * mapper.gopDurationMs / 1000)
+  return mapper.d * framesPerGroup
+}
+
+export interface VideoRenderPipelineOpts {
+  codec: string
+  /** MMTP packet_id for this video track — from catalog. */
+  packetId?: number
+  description?: Uint8Array
+  width: number
+  height: number
+  /** Frames per second (used for pacing + default reorder window). */
+  framerate: number
+  mapper: FecBlockMapper
+  fecClient?: PipelineFecClient
+  /** Reliable MoQ is clock-free; datagram deadlines come from catalog/in-band timing. */
+  reassemblyPolicy: MfuReassemblyPolicy
+  /** Receives VideoFrames ready to render. */
+  onFrame: VideoFrameCallback
+  /** Called on decoder errors. */
+  onError?: (err: unknown) => void
+  /** Reorder window in frames. Defaults to one catalog-derived FEC block. */
+  reorderWindow?: number
+  /** Warmup frames before pacing engages. Default 3 (absorbs decoder init jitter). */
+  warmupFrames?: number
+  /** Optional decoder hardware acceleration hint. */
+  hardwareAcceleration?: HardwareAcceleration
+}
+
+export class VideoRenderPipeline {
+  readonly #pacer: FrameCountPacer
+  readonly #gate = new KeyframeGate()
+  readonly #opts: VideoRenderPipelineOpts
+
+  #decoder: VideoDecoder | null = null
+  #pipeline: MmtpDecodePipeline | null = null
+  #closed = false
+
+  constructor(opts: VideoRenderPipelineOpts) {
+    if (!(opts.framerate > 0)) throw new Error('framerate must be > 0')
+    this.#opts = opts
+    this.#pacer = new FrameCountPacer({
+      frameDurationMs: 1000 / opts.framerate,
+      mapper: opts.mapper,
+      warmupFrames: opts.warmupFrames ?? 3,
+    })
+    this.#init()
+  }
+
+  feedMoqObject(packet: Uint8Array): void {
+    this.#pipeline?.feedMoqObject(packet)
+  }
+
+  paceSleepMs(): number {
+    return this.#pacer.computeSleepMs()
+  }
+
+  get pacerSnapshot() {
+    return this.#pacer.snapshot
+  }
+
+  /** Request a fresh keyframe after a decoder error (or source switch). */
+  reset(): void {
+    this.#gate.reset()
+    this.#pacer.reset()
+    try { this.#decoder?.reset() } catch { /* may be closed */ }
+    // Re-configure so the decoder is ready for the next keyframe.
+    this.#configureDecoder()
+  }
+
+  close(): void {
+    if (this.#closed) return
+    this.#closed = true
+    try { this.#decoder?.close() } catch { /* noop */ }
+    try { this.#pipeline?.dispose() } catch { /* noop */ }
+    this.#decoder = null
+    this.#pipeline = null
+  }
+
+  // ─── Private ─────────────────────────────────────────────────────
+
+  #init(): void {
+    this.#configureDecoder()
+
+    const pipeline = new MmtpDecodePipeline()
+    const reorderWindow = this.#opts.reorderWindow
+      ?? deriveVideoReorderWindow(this.#opts.framerate, this.#opts.mapper)
+    pipeline.configure({
+      codec: this.#opts.codec,
+      packetId: this.#opts.packetId,
+      description: this.#opts.description,
+      reorderWindow,
+      framerate: this.#opts.framerate,
+      groupDurationMs: this.#opts.mapper.gopDurationMs,
+      reassemblyPolicy: this.#opts.reassemblyPolicy,
+    })
+    if (this.#opts.fecClient) pipeline.setFecClient(this.#opts.fecClient)
+
+    pipeline.onFrame((codecData, meta) => {
+      if (this.#closed || !this.#decoder) return
+      if (codecData.byteLength === 0) return
+      if (!this.#gate.allow(meta.keyframe)) return
+
+      this.#pacer.notifyFrame()
+      try {
+        this.#decoder.decode(new EncodedVideoChunk({
+          type: meta.keyframe ? 'key' : 'delta',
+          data: codecData,
+          timestamp: meta.timestamp,
+        }))
+      } catch (err) {
+        this.#opts.onError?.(err)
+        this.reset()
+      }
+    })
+    this.#pipeline = pipeline
+  }
+
+  #configureDecoder(): void {
+    try { this.#decoder?.close() } catch { /* noop */ }
+
+    const userMeta: VideoFrameMeta = { timestamp: 0, keyframe: false, packetId: 0 }
+    const decoder = new VideoDecoder({
+      output: (frame) => {
+        // WebCodecs VideoFrame doesn't carry our frame meta; the consumer
+        // rematches via frame.timestamp if needed.
+        userMeta.timestamp = frame.timestamp
+        this.#opts.onFrame(frame, userMeta)
+      },
+      error: (err) => {
+        this.#opts.onError?.(err)
+        this.#gate.reset()
+      },
+    })
+    const config: VideoDecoderConfig = {
+      codec: this.#opts.codec,
+      codedWidth: this.#opts.width,
+      codedHeight: this.#opts.height,
+      ...(this.#opts.description ? { description: this.#opts.description } : {}),
+      ...(this.#opts.hardwareAcceleration ? { hardwareAcceleration: this.#opts.hardwareAcceleration } : {}),
+    }
+    decoder.configure(config)
+    this.#decoder = decoder
+  }
+}