@blockcast/mmt-container 0.1.0-main.149f78a4adae

package/src/mmtp-decode-pipeline.ts

@@ -0,0 +1,315 @@
+/**
+ * MmtpDecodePipeline — unified MMTP decode pipeline for MoQ and multicast paths.
+ *
+ * Wraps MmtpContainerParser with optional FEC client integration and a keyframe gate.
+ * Designed for the MoQ decode path (hang-mmt-fec audio/video decoders).
+ *
+ * Two entry points:
+ * - feedMoqObject(): raw MMTP packets from MoQ QUIC streams
+ * - feedAssembledFrame(): pre-assembled codec frames from multicast (post-reassembly)
+ */
+
+import {
+  MmtpContainerParser,
+  type FrameMeta,
+  type MmtpReorderStats,
+  type TrackInfo,
+} from './mmtp-container-parser.js'
+import type {
+  MfuReassemblerBackend,
+  MmtpDeliveryProvenance,
+} from './mfu-reassembler.js'
+import type { MfuReassemblyPolicy } from './mfu-reassembly-policy.js'
+
+/** Minimal FEC client interface for source symbol feeding. */
+export interface PipelineFecClient {
+  addSourceSymbol?(ssId: number, data: Uint8Array): Uint8Array | null
+}
+
+export interface PipelineConfig {
+  codec: string
+  /** Catalog video cadence exposed to downstream media stages. */
+  framerate?: number
+  /** Catalog MMTP group duration exposed to downstream media stages. */
+  groupDurationMs?: number
+  /** MMTP packet_id for this track — from catalog. When omitted, parser accepts all packetIds. */
+  packetId?: number
+  description?: Uint8Array
+  reorderWindow?: number
+  /**
+   * MFU reassembly timeout (ms). Caller derives from catalog `fec.interleaveDepthMs`,
+   * already in milliseconds (FFmpeg `moqenc_mmt.c` converts depth-in-frames →
+   * `depth_ms = fec_depth * (1000/fps)` before set_fec_config). Defaults to
+   * 1000ms when omitted. Pass null for reliable, clock-free MoQ streams;
+   * incomplete MFUs then remain bounded by the reassembler's memory limit.
+   */
+  mfuTimeoutMs?: number | null
+  /** Explicit transport policy; preferred over mfuTimeoutMs. */
+  reassemblyPolicy?: MfuReassemblyPolicy
+  /** Shared Rust/WASM backend for browser playback. */
+  reassembler?: MfuReassemblerBackend
+  maxBufferedMfus?: number
+}
+
+type FrameCallback = (data: Uint8Array, meta: FrameMeta) => void
+type InitCallback = (data: Uint8Array, packetId: number) => void
+
+export class MmtpDecodePipeline {
+  #parser: MmtpContainerParser | null = null
+  #fecClient: PipelineFecClient | null = null
+  #frameCb: FrameCallback | null = null
+  #initCb: InitCallback | null = null
+  #needsKeyframe = false
+  #codec = ''
+  /** Diagnostic counters. The keyframe gate drops silently, so a parser that
+   *  never marks a frame as a keyframe stalls the whole pipeline with no
+   *  output anywhere: frames are fed, frames are parsed, and nothing is
+   *  emitted. Bounded logging only — these gate nothing. */
+  #gateDropped = 0
+  #gateEmitted = 0
+  #fedObjects = 0
+  /** Warned-once latch for feeds arriving with no parser. Kept separate from
+   *  #fedObjects so that counter stays a pure monotonic count. */
+  #warnedNoParser = false
+  /** Histogram of feed() outcomes. feed() already reports which of its ~7 exit
+   *  paths a packet took, via its return value — but the call below discards it,
+   *  so a packet that fails header parse, fails MFU extraction, or accumulates
+   *  forever as an unreassembled fragment is indistinguishable from one that
+   *  produced a frame. Tally the kinds instead of guessing. */
+  #feedOutcomes = new Map<string, number>()
+
+  /** Configure the inner parser. Call before feedMoqObject(). */
+  configure(config: PipelineConfig): void {
+    // Reconfiguration is an ordered boundary: emit frames accepted under the
+    // old configuration before replacing its parser and track metadata.
+    this.#parser?.flush()
+    this.#parser?.dispose()
+
+    // The diagnostic tallies are per-configuration, matching the boundary above.
+    // Carrying them across would blend two codecs into one histogram and let
+    // "keyframe gate OPENED after N drops" attribute drops accrued under the
+    // previous configuration.
+    this.#fedObjects = 0
+    this.#gateDropped = 0
+    this.#gateEmitted = 0
+    this.#feedOutcomes.clear()
+    this.#warnedNoParser = false
+
+    this.#codec = config.codec
+
+    const trackType: 'video' | 'audio' = isVideoCodec(config.codec) ? 'video' : 'audio'
+    const trackInfo: TrackInfo = {
+      type: trackType,
+      codec: config.codec,
+      ...(config.framerate !== undefined ? { framerate: config.framerate } : {}),
+      ...(config.groupDurationMs !== undefined
+        ? { groupDurationMs: config.groupDurationMs }
+        : {}),
+    }
+    const tracks = new Map<number, TrackInfo>()
+    if (config.packetId != null) {
+      tracks.set(config.packetId, trackInfo)
+    }
+    // When packetId omitted: empty tracks map → parser accepts all packets,
+    // codec/type derived from content. Caller should provide packetId from catalog.
+
+    this.#parser = new MmtpContainerParser({
+      tracks,
+      reorderWindow: config.reorderWindow ?? 0,
+      mfuTimeoutMs: config.mfuTimeoutMs,
+      reassemblyPolicy: config.reassemblyPolicy,
+      reassembler: config.reassembler,
+      maxBufferedMfus: config.maxBufferedMfus,
+      defaultTrack: trackInfo,
+    })
+
+    this.#parser.onFrame((data, meta) => {
+      this.#gatedEmit(data, meta)
+    })
+    this.#parser.onInit((data, packetId) => {
+      this.#initCb?.(data, packetId)
+    })
+  }
+
+  /** Inject a FEC client for source symbol feeding on the MoQ path. */
+  setFecClient(client: PipelineFecClient): void {
+    this.#fecClient = client
+  }
+
+  /** Register frame output callback. */
+  onFrame(cb: FrameCallback): void {
+    this.#frameCb = cb
+  }
+
+  /** Register init segment callback. */
+  onInit(cb: InitCallback): void {
+    this.#initCb = cb
+  }
+
+  /** Current shared-parser ordering/drop telemetry. */
+  get reorderStats(): Readonly<MmtpReorderStats> | null {
+    return this.#parser?.reorderStats ?? null
+  }
+
+  /**
+   * Feed a raw MoQ object payload (one MMTP packet per MoQ object).
+   * Optionally feeds source symbols to FEC client before parsing.
+   */
+  feedMoqObject(packet: Uint8Array): void {
+    if (!this.#parser) {
+      // Distinguishes "never configured" from "configured but emitting nothing".
+      // Silent today: the caller has no way to see that its feed went nowhere.
+      //
+      // The warned-once flag is deliberately separate from #fedObjects. Folding
+      // it in (a -1 sentinel) makes this guard test "no parser AND nothing ever
+      // fed", so a feed arriving after dispose() — which nulls #parser but
+      // leaves the count — would return here silently. That is the exact
+      // silent-drop class this file exists to expose, and teardown racing an
+      // in-flight feed is the likelier ordering in a live pipeline than
+      // feed-before-configure.
+      if (!this.#warnedNoParser) {
+        this.#warnedNoParser = true
+        console.warn('[mmtp-pipeline] feedMoqObject called with NO PARSER configured — packet dropped')
+      }
+      return
+    }
+    this.#fedObjects += 1
+    if (this.#fedObjects === 1) {
+      console.log(`[mmtp-pipeline] first object fed codec=${this.#codec} bytes=${packet.length}`)
+    }
+
+    if (this.#fecClient?.addSourceSymbol) {
+      const info = this.#parser.parseHeader(packet)
+      if (info?.fecPayloadId) {
+        this.#fecClient.addSourceSymbol(info.fecPayloadId.ssId, packet)
+      }
+    }
+
+    const record = this.#parser.feed(packet)
+    // `null` means feed() bailed at one of its early returns — header parse,
+    // MFU extraction, timedFlag, or a fragmentation-indicator guard.
+    const kind = record?.kind ?? 'null(parse-or-extract-failed)'
+    const seen = (this.#feedOutcomes.get(kind) ?? 0) + 1
+    this.#feedOutcomes.set(kind, seen)
+    if (seen === 1) {
+      console.log(
+        `[mmtp-pipeline] feed outcome FIRST '${kind}' at object #${this.#fedObjects} bytes=${packet.length}`,
+      )
+    }
+    if (this.#fedObjects % 200 === 0) {
+      const hist = [...this.#feedOutcomes].map(([k, v]) => `${k}=${v}`).join(' ')
+      console.log(
+        `[mmtp-pipeline] feed histogram @${this.#fedObjects}: ${hist} | emitted=${this.#gateEmitted} gateDropped=${this.#gateDropped}`,
+      )
+    }
+  }
+
+  /**
+   * Feed a pre-assembled codec frame (from multicast, post-reassembly).
+   * Bypasses MMTP parsing — emits directly through the keyframe gate.
+   *
+   * Callers MUST supply:
+   *   - packetSequenceNumber: unique per-AU identifier (typically the MMTP
+   *     packet_sequence_number of the first fragment). Required for
+   *     deterministic dedup against duplicates arriving via other paths.
+   *   - packetId: MMTP packet_id of the track (1=video, 2=audio by convention).
+   */
+  feedAssembledFrame(
+    data: Uint8Array,
+    meta: {
+      timestamp: number
+      keyframe: boolean
+      packetSequenceNumber: number
+      packetId: number
+      sourceDiscontinuity?: boolean
+      recovered?: boolean
+      /**
+       * Whole-frame delivery provenance, when the caller knows it. Absent means
+       * the caller never told us — which is NOT the same as 'unknown', a
+       * verdict the caller reached and is asserting. Keep them distinguishable.
+       */
+      deliveryProvenance?: MmtpDeliveryProvenance
+    },
+  ): void {
+    const frameMeta: FrameMeta = {
+      timestamp: meta.timestamp,
+      keyframe: meta.keyframe,
+      packetId: meta.packetId,
+      trackType: isVideoCodec(this.#codec) ? 'video' : 'audio',
+      packetSequenceNumber: meta.packetSequenceNumber,
+    }
+    if (meta.sourceDiscontinuity) frameMeta.sourceDiscontinuity = true
+    if (meta.recovered) frameMeta.recovered = true
+    // Unlike the boolean flags above, forward this whenever it is present —
+    // 'unknown' is a real verdict, not a falsy value to drop.
+    if (meta.deliveryProvenance !== undefined)
+      frameMeta.deliveryProvenance = meta.deliveryProvenance
+    this.#parser?.routeFrame(data, frameMeta)
+  }
+
+  /** Request keyframe — gates output until a keyframe arrives. */
+  requestKeyframe(): void {
+    this.#needsKeyframe = true
+  }
+
+  /** Emit buffered ordered frames. Finite consumers call this before dispose(). */
+  flush(): void {
+    this.#parser?.flush()
+  }
+
+  /** Clean up the pipeline. Finite inputs pass flush=true at EOS; live
+   * cancellation keeps the default drop-only teardown behavior. */
+  dispose(options: { flush?: boolean } = {}): void {
+    const droppedBefore = this.#parser?.reorderStats.droppedOnLiveDispose ?? 0
+    if (options.flush) this.#parser?.flush()
+    this.#parser?.dispose()
+    const droppedFrames = (this.#parser?.reorderStats.droppedOnLiveDispose ?? droppedBefore) -
+      droppedBefore
+    if (droppedFrames > 0) {
+      console.warn(
+        `[MmtpDecodePipeline] live dispose dropped ${droppedFrames} buffered frame(s); ` +
+        'finite inputs must use dispose({ flush: true })',
+      )
+    }
+    this.#parser = null
+    this.#frameCb = null
+    this.#initCb = null
+    this.#fecClient = null
+  }
+
+  #gatedEmit(data: Uint8Array, meta: FrameMeta): void {
+    let sourceDiscontinuity = false
+    if (this.#needsKeyframe) {
+      if (!meta.keyframe) {
+        // Silent drop: the caller keeps feeding, the parser keeps parsing, and
+        // no layer reports anything. If the parser never sets meta.keyframe the
+        // gate never opens and the decoder is never called.
+        this.#gateDropped += 1
+        if (this.#gateDropped <= 3 || this.#gateDropped % 200 === 0) {
+          console.warn(
+            `[mmtp-pipeline] keyframe gate DROP #${this.#gateDropped} codec=${this.#codec} bytes=${data.length} fed=${this.#fedObjects} emitted=${this.#gateEmitted}`,
+          )
+        }
+        return
+      }
+      this.#needsKeyframe = false
+      sourceDiscontinuity = true
+      console.log(
+        `[mmtp-pipeline] keyframe gate OPENED codec=${this.#codec} after ${this.#gateDropped} drops, fed=${this.#fedObjects}`,
+      )
+    }
+    this.#gateEmitted += 1
+    if (this.#gateEmitted === 1) {
+      console.log(`[mmtp-pipeline] first frame emitted codec=${this.#codec} bytes=${data.length}`)
+    }
+    const outMeta = sourceDiscontinuity ? { ...meta, sourceDiscontinuity: true } : meta
+    this.#frameCb?.(data, outMeta)
+  }
+}
+
+function isVideoCodec(codec: string): boolean {
+  return codec.startsWith('avc') ||
+    codec.startsWith('hev1') || codec.startsWith('hvc1') ||
+    codec.startsWith('av01') ||
+    codec.startsWith('vp09') || codec.startsWith('vp8')
+}