@blockcast/mmt-container 0.1.0-main.0399efef1f07

package/src/mmtp-decode-pipeline.ts

@@ -0,0 +1,389 @@
+/**
+ * MmtpDecodePipeline — unified MMTP decode pipeline for MoQ and multicast paths.
+ *
+ * Wraps MmtpContainerParser with optional FEC client integration and a keyframe gate.
+ * Designed for the MoQ decode path (hang-mmt-fec audio/video decoders).
+ *
+ * Two entry points:
+ * - feedMoqObject(): raw MMTP packets from MoQ QUIC streams
+ * - feedAssembledFrame(): pre-assembled codec frames from multicast (post-reassembly)
+ */
+
+import {
+  MmtpContainerParser,
+  type FrameMeta,
+  type MmtpReorderStats,
+  type TrackInfo,
+} from './mmtp-container-parser.js'
+import type {
+  MfuReassemblerBackend,
+  MmtpDeliveryProvenance,
+} from './mfu-reassembler.js'
+import type { MfuReassemblyPolicy } from './mfu-reassembly-policy.js'
+
+/** Minimal FEC client interface for source symbol feeding. */
+export interface PipelineFecClient {
+  addSourceSymbol?(ssId: number, data: Uint8Array): Uint8Array | null
+}
+
+export interface PipelineConfig {
+  codec: string
+  /** Catalog video cadence exposed to downstream media stages. */
+  framerate?: number
+  /** Catalog MMTP group duration exposed to downstream media stages. */
+  groupDurationMs?: number
+  /** MMTP packet_id for this track — from catalog. When omitted, parser accepts all packetIds. */
+  packetId?: number
+  description?: Uint8Array
+  reorderWindow?: number
+  /**
+   * MFU reassembly timeout (ms). Caller derives from catalog `fec.interleaveDepthMs`,
+   * already in milliseconds (FFmpeg `moqenc_mmt.c` converts depth-in-frames →
+   * `depth_ms = fec_depth * (1000/fps)` before set_fec_config). Defaults to
+   * 1000ms when omitted. Pass null for reliable, clock-free MoQ streams;
+   * incomplete MFUs then remain bounded by the reassembler's memory limit.
+   */
+  mfuTimeoutMs?: number | null
+  /** Explicit transport policy; preferred over mfuTimeoutMs. */
+  reassemblyPolicy?: MfuReassemblyPolicy
+  /** Shared Rust/WASM backend for browser playback. */
+  reassembler?: MfuReassemblerBackend
+  maxBufferedMfus?: number
+}
+
+type FrameCallback = (data: Uint8Array, meta: FrameMeta) => void
+type InitCallback = (data: Uint8Array, packetId: number) => void
+
+/**
+ * Snapshot of one pipeline's feed tallies.
+ *
+ * These counters were console-only, and a player builds one pipeline per track,
+ * so two instances logged indistinguishably and the audio receive path could not
+ * be read at all from a harness artifact. Expose them as a producer-owned typed
+ * counter — logs are diagnostic output, not an acceptance API.
+ */
+export interface MmtpFeedStats {
+  /** Track identity, so a consumer can tell two concurrent pipelines apart. */
+  trackType: 'video' | 'audio' | null
+  codec: string | null
+  fedObjects: number
+  emitted: number
+  gateDropped: number
+  /**
+   * feed() exit-path tallies, keyed by record kind, with
+   * 'null(parse-or-extract-failed)' for the early returns and 'threw' when
+   * parseHeader()/feed() raises. Mutually exclusive: the values sum to
+   * fedObjects on every path (the throw bucket is tallied in a finally), which
+   * is what makes a frozen media bucket beside a still-climbing non-media
+   * bucket a readable signal.
+   */
+  outcomes: Record<string, number>
+}
+
+export class MmtpDecodePipeline {
+  #parser: MmtpContainerParser | null = null
+  #fecClient: PipelineFecClient | null = null
+  #frameCb: FrameCallback | null = null
+  #initCb: InitCallback | null = null
+  #needsKeyframe = false
+  #codec: string | null = null
+  /** Set together with #codec at configure(). Also the diagnostic label: a
+   *  player builds one pipeline per track, and codec alone does not say which
+   *  role a line belongs to at a glance. */
+  #trackType: 'video' | 'audio' | null = null
+  /** Diagnostic counters. The keyframe gate drops silently, so a parser that
+   *  never marks a frame as a keyframe stalls the whole pipeline with no
+   *  output anywhere: frames are fed, frames are parsed, and nothing is
+   *  emitted. Bounded logging only — these gate nothing. */
+  #gateDropped = 0
+  #gateEmitted = 0
+  #fedObjects = 0
+  /** Warned-once latch for feeds arriving with no parser. Kept separate from
+   *  #fedObjects so that counter stays a pure monotonic count. */
+  #warnedNoParser = false
+  /** Histogram of feed() outcomes. feed() already reports which of its ~7 exit
+   *  paths a packet took, via its return value — but the call below discards it,
+   *  so a packet that fails header parse, fails MFU extraction, or accumulates
+   *  forever as an unreassembled fragment is indistinguishable from one that
+   *  produced a frame. Tally the kinds instead of guessing. */
+  #feedOutcomes = new Map<string, number>()
+
+  /** Configure the inner parser. Call before feedMoqObject(). */
+  configure(config: PipelineConfig): void {
+    // Reconfiguration is an ordered boundary: emit frames accepted under the
+    // old configuration before replacing its parser and track metadata.
+    this.#parser?.flush()
+    this.#parser?.dispose()
+
+    // The diagnostic tallies are per-configuration, matching the boundary above.
+    // Carrying them across would blend two codecs into one histogram and let
+    // "keyframe gate OPENED after N drops" attribute drops accrued under the
+    // previous configuration.
+    this.#fedObjects = 0
+    this.#gateDropped = 0
+    this.#gateEmitted = 0
+    this.#feedOutcomes.clear()
+    this.#warnedNoParser = false
+
+    this.#codec = config.codec
+
+    const trackType: 'video' | 'audio' = isVideoCodec(config.codec) ? 'video' : 'audio'
+    this.#trackType = trackType
+    const trackInfo: TrackInfo = {
+      type: trackType,
+      codec: config.codec,
+      ...(config.framerate !== undefined ? { framerate: config.framerate } : {}),
+      ...(config.groupDurationMs !== undefined
+        ? { groupDurationMs: config.groupDurationMs }
+        : {}),
+    }
+    const tracks = new Map<number, TrackInfo>()
+    if (config.packetId != null) {
+      tracks.set(config.packetId, trackInfo)
+    }
+    // When packetId omitted: empty tracks map → parser accepts all packets,
+    // codec/type derived from content. Caller should provide packetId from catalog.
+
+    this.#parser = new MmtpContainerParser({
+      tracks,
+      reorderWindow: config.reorderWindow ?? 0,
+      mfuTimeoutMs: config.mfuTimeoutMs,
+      reassemblyPolicy: config.reassemblyPolicy,
+      reassembler: config.reassembler,
+      maxBufferedMfus: config.maxBufferedMfus,
+      defaultTrack: trackInfo,
+    })
+
+    this.#parser.onFrame((data, meta) => {
+      this.#gatedEmit(data, meta)
+    })
+    this.#parser.onInit((data, packetId) => {
+      this.#initCb?.(data, packetId)
+    })
+  }
+
+  /** Inject a FEC client for source symbol feeding on the MoQ path. */
+  setFecClient(client: PipelineFecClient): void {
+    this.#fecClient = client
+  }
+
+  /** Register frame output callback. */
+  onFrame(cb: FrameCallback): void {
+    this.#frameCb = cb
+  }
+
+  /** Register init segment callback. */
+  onInit(cb: InitCallback): void {
+    this.#initCb = cb
+  }
+
+  /** Current shared-parser ordering/drop telemetry. */
+  get reorderStats(): Readonly<MmtpReorderStats> | null {
+    return this.#parser?.reorderStats ?? null
+  }
+
+  /**
+   * Feed tallies for this track. A fresh snapshot per call, not a live view.
+   *
+   * Unlike reorderStats this never returns null: the counters belong to the
+   * pipeline rather than the parser, so they survive dispose(). That is
+   * deliberate — a wedged pipeline is usually read after teardown, which is
+   * exactly when the parser is already gone.
+   */
+  get feedStats(): Readonly<MmtpFeedStats> {
+    return {
+      trackType: this.#trackType,
+      codec: this.#codec,
+      fedObjects: this.#fedObjects,
+      emitted: this.#gateEmitted,
+      gateDropped: this.#gateDropped,
+      outcomes: Object.fromEntries(this.#feedOutcomes),
+    }
+  }
+
+  /** Identity prefix for diagnostics: one pipeline per track means two
+   *  instances write to the same console, and unlabelled lines interleave into
+   *  a single unreadable stream. */
+  #tag(): string {
+    return `track=${this.#trackType ?? 'unconfigured'} codec=${this.#codec ?? 'none'}`
+  }
+
+  /**
+   * Feed a raw MoQ object payload (one MMTP packet per MoQ object).
+   * Optionally feeds source symbols to FEC client before parsing.
+   */
+  feedMoqObject(packet: Uint8Array): void {
+    if (!this.#parser) {
+      // Distinguishes "never configured" from "configured but emitting nothing".
+      // Silent today: the caller has no way to see that its feed went nowhere.
+      //
+      // The warned-once flag is deliberately separate from #fedObjects. Folding
+      // it in (a -1 sentinel) makes this guard test "no parser AND nothing ever
+      // fed", so a feed arriving after dispose() — which nulls #parser but
+      // leaves the count — would return here silently. That is the exact
+      // silent-drop class this file exists to expose, and teardown racing an
+      // in-flight feed is the likelier ordering in a live pipeline than
+      // feed-before-configure.
+      if (!this.#warnedNoParser) {
+        this.#warnedNoParser = true
+        console.warn(`[mmtp-pipeline] feedMoqObject called with NO PARSER configured ${this.#tag()} — packet dropped`)
+      }
+      return
+    }
+    this.#fedObjects += 1
+    if (this.#fedObjects === 1) {
+      console.log(`[mmtp-pipeline] first object fed ${this.#tag()} bytes=${packet.length}`)
+    }
+
+    // Pessimistic default: any throw from parseHeader()/feed() below leaves
+    // this as 'threw', which the finally tallies. That keeps the histogram
+    // exact: sum(outcomes) === fedObjects on every path, including the wedge
+    // that is exactly when malformed input arrives and the invariant is most
+    // load-bearing.
+    let outcomeKind = 'threw'
+    try {
+      if (this.#fecClient?.addSourceSymbol) {
+        const info = this.#parser.parseHeader(packet)
+        if (info?.fecPayloadId) {
+          this.#fecClient.addSourceSymbol(info.fecPayloadId.ssId, packet)
+        }
+      }
+
+      const record = this.#parser.feed(packet)
+      // `null` means feed() bailed at one of its early returns: header parse,
+      // MFU extraction, timedFlag, or a fragmentation-indicator guard.
+      outcomeKind = record?.kind ?? 'null(parse-or-extract-failed)'
+    } finally {
+      const seen = (this.#feedOutcomes.get(outcomeKind) ?? 0) + 1
+      this.#feedOutcomes.set(outcomeKind, seen)
+      if (seen === 1) {
+        console.log(
+          `[mmtp-pipeline] feed outcome FIRST '${outcomeKind}' ${this.#tag()} at object #${this.#fedObjects} bytes=${packet.length}`,
+        )
+      }
+    }
+    if (this.#fedObjects % 200 === 0) {
+      const hist = [...this.#feedOutcomes].map(([k, v]) => `${k}=${v}`).join(' ')
+      console.log(
+        `[mmtp-pipeline] feed histogram ${this.#tag()} @${this.#fedObjects}: ${hist} | emitted=${this.#gateEmitted} gateDropped=${this.#gateDropped}`,
+      )
+    }
+  }
+
+  /**
+   * Feed a pre-assembled codec frame (from multicast, post-reassembly).
+   * Bypasses MMTP parsing — emits directly through the keyframe gate.
+   *
+   * Callers MUST supply:
+   *   - packetSequenceNumber: unique per-AU identifier (typically the MMTP
+   *     packet_sequence_number of the first fragment). Required for
+   *     deterministic dedup against duplicates arriving via other paths.
+   *   - packetId: MMTP packet_id of the track (1=video, 2=audio by convention).
+   */
+  feedAssembledFrame(
+    data: Uint8Array,
+    meta: {
+      timestamp: number
+      /** Raw MMTP short-NTP header timestamp; timing telemetry only. */
+      mmtpTimestamp?: bigint
+      keyframe: boolean
+      packetSequenceNumber: number
+      packetId: number
+      sourceDiscontinuity?: boolean
+      recovered?: boolean
+      /**
+       * Whole-frame delivery provenance, when the caller knows it. Absent means
+       * the caller never told us — which is NOT the same as 'unknown', a
+       * verdict the caller reached and is asserting. Keep them distinguishable.
+       */
+      deliveryProvenance?: MmtpDeliveryProvenance
+    },
+  ): void {
+    const frameMeta: FrameMeta = {
+      timestamp: meta.timestamp,
+      ...(meta.mmtpTimestamp === undefined
+        ? {}
+        : { mmtpTimestamp: meta.mmtpTimestamp }),
+      keyframe: meta.keyframe,
+      packetId: meta.packetId,
+      // Same derivation as configure(), read from the field it set rather than
+      // recomputed. Pre-configure the parser is null and routeFrame below is a
+      // no-op, so the fallback is unreachable in practice.
+      trackType: this.#trackType ?? 'audio',
+      packetSequenceNumber: meta.packetSequenceNumber,
+    }
+    if (meta.sourceDiscontinuity) frameMeta.sourceDiscontinuity = true
+    if (meta.recovered) frameMeta.recovered = true
+    // Unlike the boolean flags above, forward this whenever it is present —
+    // 'unknown' is a real verdict, not a falsy value to drop.
+    if (meta.deliveryProvenance !== undefined)
+      frameMeta.deliveryProvenance = meta.deliveryProvenance
+    this.#parser?.routeFrame(data, frameMeta)
+  }
+
+  /** Request keyframe — gates output until a keyframe arrives. */
+  requestKeyframe(): void {
+    this.#needsKeyframe = true
+  }
+
+  /** Emit buffered ordered frames. Finite consumers call this before dispose(). */
+  flush(): void {
+    this.#parser?.flush()
+  }
+
+  /** Clean up the pipeline. Finite inputs pass flush=true at EOS; live
+   * cancellation keeps the default drop-only teardown behavior. */
+  dispose(options: { flush?: boolean } = {}): void {
+    const droppedBefore = this.#parser?.reorderStats.droppedOnLiveDispose ?? 0
+    if (options.flush) this.#parser?.flush()
+    this.#parser?.dispose()
+    const droppedFrames = (this.#parser?.reorderStats.droppedOnLiveDispose ?? droppedBefore) -
+      droppedBefore
+    if (droppedFrames > 0) {
+      console.warn(
+        `[MmtpDecodePipeline] live dispose ${this.#tag()} dropped ${droppedFrames} buffered frame(s); ` +
+        'finite inputs must use dispose({ flush: true })',
+      )
+    }
+    this.#parser = null
+    this.#frameCb = null
+    this.#initCb = null
+    this.#fecClient = null
+  }
+
+  #gatedEmit(data: Uint8Array, meta: FrameMeta): void {
+    let sourceDiscontinuity = false
+    if (this.#needsKeyframe) {
+      if (!meta.keyframe) {
+        // Silent drop: the caller keeps feeding, the parser keeps parsing, and
+        // no layer reports anything. If the parser never sets meta.keyframe the
+        // gate never opens and the decoder is never called.
+        this.#gateDropped += 1
+        if (this.#gateDropped <= 3 || this.#gateDropped % 200 === 0) {
+          console.warn(
+            `[mmtp-pipeline] keyframe gate DROP #${this.#gateDropped} ${this.#tag()} bytes=${data.length} fed=${this.#fedObjects} emitted=${this.#gateEmitted}`,
+          )
+        }
+        return
+      }
+      this.#needsKeyframe = false
+      sourceDiscontinuity = true
+      console.log(
+        `[mmtp-pipeline] keyframe gate OPENED ${this.#tag()} after ${this.#gateDropped} drops, fed=${this.#fedObjects}`,
+      )
+    }
+    this.#gateEmitted += 1
+    if (this.#gateEmitted === 1) {
+      console.log(`[mmtp-pipeline] first frame emitted ${this.#tag()} bytes=${data.length}`)
+    }
+    const outMeta = sourceDiscontinuity ? { ...meta, sourceDiscontinuity: true } : meta
+    this.#frameCb?.(data, outMeta)
+  }
+}
+
+function isVideoCodec(codec: string): boolean {
+  return codec.startsWith('avc') ||
+    codec.startsWith('hev1') || codec.startsWith('hvc1') ||
+    codec.startsWith('av01') ||
+    codec.startsWith('vp09') || codec.startsWith('vp8')
+}