@blockcast/mmt-container 0.1.0 → 0.3.0-main.36c5f6abe867

package/src/duplicate-ledger.ts

@@ -0,0 +1,372 @@
+/**
+ * MmtpRouter's per-stream duplicate ledgers, retained for the stream's FEC
+ * horizon (BLO-36599).
+ *
+ * The router remembers three kinds of wire number per physical stream, so a
+ * second copy of a packet is not parsed or delivered twice:
+ *
+ * - packet_sequence_numbers received on the direct source path, which a
+ *   recovered FEC block must not deliver again;
+ * - packet_sequence_numbers already handed to the track packet callback;
+ * - Source FEC Payload IDs (SS_ID) already parsed for playback, which dedup an
+ *   overlap copy against its multicast twin.
+ *
+ * Each of these is useful for as long as FECManager can still re-deliver the
+ * number's packet, which ends when its FEC block goes terminal; the ledger
+ * keeps one further block lifetime of margin. A block's symbols all arrive
+ * within its span, the catalog's `interleaveDepthMs` (draft-ramadan-moq-fec
+ * §5.1). After its last symbol FECManager makes the block terminal within one
+ * block lifetime (the inactivity fail bound plus the D-28 repair grace, the
+ * value `blockLifetimeMsForPacket()` returns). FECManager keeps the terminal
+ * record longer than that (`terminalRetentionMsForPacket()`, for the in-order
+ * release gate), and that record itself counts late symbols as duplicates, so
+ * the ledger need not match it. Measured from a number's first arrival that is
+ *
+ *   retention = interleaveDepthMs + blockLifetime + blockLifetime
+ *
+ * The bound is a time, not a count, on purpose. A count would need the
+ * stream's symbol rate, and the catalog does not give one: the producer closes
+ * a block after D media units or after K source symbols, whichever comes
+ * first, so one span can hold several blocks (every large keyframe does) and K
+ * per span is not an upper bound on the rate. The fixed 2000-entry window this
+ * replaces held about 4 s of the 467 packet/s AMT stream and about 40 s of a
+ * 47 packet/s audio stream; this one holds each stream for its own horizon.
+ *
+ * The clock is read only when a stream moves to a new FEC block (or for a
+ * packet without an FEC coordinate), never per packet, and no timer is armed:
+ * the ledgers rotate lazily.
+ *
+ * A stream whose horizon cannot be derived while FEC still runs (the FEC
+ * manager reports no per-stream geometry, or the catalog gives no block span,
+ * in which case FECManager falls back to its fail bound and still recovers
+ * blocks) is still re-delivered packets by recovery, so it keeps the count
+ * window the router kept for every stream before this change: each ledger
+ * holds its newest COUNT_BOUNDED_GENERATION to 2 * COUNT_BOUNDED_GENERATION
+ * numbers. Only a router without an FEC manager keeps no ledger: nothing
+ * re-delivers a packet.
+ */
+
+/**
+ * Generation length, in numbers, of a ledger whose stream has no derivable FEC
+ * horizon. This is not a new bound: it is the window every stream kept before
+ * BLO-36599 (trim to the newest 2000 once past 4000), kept for exactly the
+ * streams the FEC horizon cannot describe.
+ */
+export const COUNT_BOUNDED_GENERATION = 2000
+
+/**
+ * RFC 1982 serial-number order on a 32-bit wire counter (MMTP
+ * packet_sequence_number, SS_ID): true when `a` precedes `b`. The counters
+ * wrap, so plain `<` would report every wrap as a regression.
+ */
+export function serial32Precedes(a: number, b: number): boolean {
+  const distance = (b - a) >>> 0
+  return distance !== 0 && distance < 0x80000000
+}
+
+/** The two FECManager values a stream's duplicate retention is derived from. */
+export interface DuplicateHorizonGeometry {
+  /** Catalog `fec.interleaveDepthMs`: the full FEC block span, in ms. */
+  readonly blockSpanMs: number
+  /**
+   * FECManager's `blockLifetimeMsForPacket()`: the longest a block stays open
+   * after its last symbol. Its terminal record is kept at least this long.
+   */
+  readonly blockLifetimeMs: number
+}
+
+/**
+ * How long a stream's duplicate ledgers keep a number, or why no FEC horizon
+ * is derivable. `'no-fec-manager'`: the router decodes no FEC, so nothing
+ * re-delivers a packet and the stream keeps no ledger. `'no-fec-geometry'`:
+ * the FEC manager cannot report per-stream geometry. `'no-block-span'`: the
+ * stream has no catalog FEC block span (FECManager reports interleaveDepthMs
+ * 0, which is also what a stream without FEC reports). FEC may still recover
+ * blocks in the last two cases, so they keep count-bounded ledgers.
+ */
+export type DuplicateHorizon =
+  | { readonly kind: 'fec-horizon'; readonly retentionMs: number }
+  | {
+      readonly kind: 'none'
+      readonly reason: 'no-fec-manager' | 'no-fec-geometry' | 'no-block-span'
+    }
+
+function assertPositiveMs(name: keyof DuplicateHorizonGeometry, value: number): void {
+  if (!Number.isFinite(value) || value <= 0) {
+    throw new Error(
+      `[DuplicateLedger] ${name}=${value}. Derive a positive ${name} from the ` +
+        'stream catalog through FECManager.',
+    )
+  }
+}
+
+/** Derive a stream's duplicate retention from its FEC geometry; see the module comment. */
+export function duplicateRetentionMs(geometry: DuplicateHorizonGeometry): number {
+  assertPositiveMs('blockSpanMs', geometry.blockSpanMs)
+  assertPositiveMs('blockLifetimeMs', geometry.blockLifetimeMs)
+  const untilTerminalMs = geometry.blockLifetimeMs
+  const terminalRecordMs = geometry.blockLifetimeMs
+  return geometry.blockSpanMs + untilTerminalMs + terminalRecordMs
+}
+
+/**
+ * A set of wire numbers kept in two generations. Rotating retires the older
+ * generation and starts a new one, so a number stays a member from the moment
+ * it is added until the second rotation after that: at least one full
+ * generation. A number already present keeps its first-arrival generation, as
+ * a duplicate must be timed from the original. Additions are dropped while
+ * `gate.retaining` is false. While `gate.countBounded` is true the set rotates
+ * itself each time its newer generation reaches COUNT_BOUNDED_GENERATION.
+ *
+ * One Set holds both generations. A number is added once, so the Set's
+ * insertion order is generation order and the older generation is exactly its
+ * first `#olderCount` members; rotating deletes those in place. Two Sets with
+ * a clear() per rotation would reallocate and regrow a Set every generation.
+ */
+export class GenerationalNumberSet {
+  readonly #members = new Set<number>()
+  #olderCount = 0
+  #newerCount = 0
+  #rotations = 0
+  #floor: number | undefined = undefined
+  readonly #gate: { readonly retaining: boolean; readonly countBounded: boolean }
+
+  constructor(gate: { readonly retaining: boolean; readonly countBounded: boolean }) {
+    this.#gate = gate
+  }
+
+  has(value: number): boolean {
+    return this.#members.has(value)
+  }
+
+  add(value: number): void {
+    if (!this.#gate.retaining || this.#members.has(value)) return
+    this.#members.add(value)
+    if (++this.#newerCount >= COUNT_BOUNDED_GENERATION && this.#gate.countBounded) this.rotate()
+  }
+
+  /** Retire the older generation; the current one becomes the older. */
+  rotate(): void {
+    let retiring = this.#olderCount
+    if (retiring > 0) {
+      for (const value of this.#members) {
+        this.#members.delete(value)
+        if (--retiring === 0) break
+      }
+    }
+    this.#olderCount = this.#newerCount
+    this.#newerCount = 0
+    this.#rotations++
+    // Members are only appended between rotations, so the earliest-arrived
+    // survivor is fixed until the next rotation; read it once here rather than
+    // allocating an iterator per lookup.
+    this.#floor = undefined
+    for (const value of this.#members) {
+      this.#floor = value
+      break
+    }
+  }
+
+  clear(): void {
+    this.#members.clear()
+    this.#olderCount = 0
+    this.#newerCount = 0
+    this.restartAging()
+  }
+
+  /**
+   * Keep every member but forget how many rotations have passed, so no floor
+   * is reported until two more rotations. A stream that starts timing its
+   * generations calls this: generations rotated by count say nothing about
+   * how long ago their members arrived.
+   */
+  restartAging(): void {
+    this.#rotations = 0
+    this.#floor = undefined
+  }
+
+  /**
+   * The earliest-arrived member, once it is known to have arrived at least one
+   * full generation ago; `undefined` before that.
+   *
+   * A rotation is triggered by an arriving packet, which is normally the
+   * first member of the new generation (a live stream adds its next packet
+   * within one packet interval otherwise). From the second rotation on, the
+   * older generation therefore began at a rotation, and its first member
+   * arrived about one generation length before now. Before the second
+   * rotation the first member may be a number that simply arrived early out of
+   * order, so no floor is reported.
+   */
+  get horizonFloor(): number | undefined {
+    if (this.#rotations < 2 || this.#olderCount === 0) return undefined
+    return this.#floor
+  }
+
+  get size(): number {
+    return this.#members.size
+  }
+}
+
+/**
+ * One physical stream's three duplicate ledgers and their shared generation
+ * clock. `advance()` runs once per packet (and once per recovered block) before
+ * any lookup, `predatesHorizon()` included. It does work only when the packet
+ * belongs to a different FEC block than the previous one, or carries no FEC
+ * coordinate on a stream with an FEC horizon; everything else is a Set lookup.
+ */
+export class StreamDuplicateLedgers {
+  /** packet_sequence_numbers received on the direct source path. */
+  readonly sourceSeqs: GenerationalNumberSet
+  /** packet_sequence_numbers already handed to the track packet callback. */
+  readonly routedSeqs: GenerationalNumberSet
+  /** SS_IDs already parsed for playback. */
+  readonly parsedSymbols: GenerationalNumberSet
+
+  readonly #horizon: () => DuplicateHorizon
+  readonly #now: () => number
+  #retentionMs: number | null = null
+  #countBounded = false
+  /** Whether the horizon was read since construction or the last clear(). */
+  #horizonRead = false
+  #generationStartMs = 0
+  #lastSbn: number | undefined = undefined
+
+  /**
+   * @param horizon resolves the stream's current horizon from its FEC
+   *   geometry. It is read when the stream moves to a new block, so a catalog
+   *   change applies from the next block.
+   * @param now the clock FECManager's deadlines use.
+   */
+  constructor(horizon: () => DuplicateHorizon, now: () => number = () => performance.now()) {
+    this.#horizon = horizon
+    this.#now = now
+    this.sourceSeqs = new GenerationalNumberSet(this)
+    this.routedSeqs = new GenerationalNumberSet(this)
+    this.parsedSymbols = new GenerationalNumberSet(this)
+  }
+
+  /**
+   * Note the FEC block (SBN) of the packet about to be looked up, or
+   * `undefined` for a packet without a Source FEC Payload ID. Returns whether
+   * the stream keeps ledgers; while it does not, lookups find nothing and
+   * additions are dropped. The horizon is read on the first packet after
+   * construction or clear(), and again whenever the block changes; a packet
+   * without a coordinate never re-reads it, so a stream without FEC
+   * coordinates resolves its horizon once, not per packet.
+   */
+  advance(sbn: number | undefined): boolean {
+    if (sbn !== undefined) {
+      if (sbn === this.#lastSbn) return this.retaining
+      this.#lastSbn = sbn
+    } else if (this.#horizonRead) {
+      // A packet outside every FEC block (MPU metadata, for one) does not move
+      // the stream to a new block, but it still ages timed generations, so a
+      // stream that stops carrying FEC coordinates cannot grow its ledgers.
+      // Count-bounded ledgers age on their own.
+      if (this.#retentionMs !== null) this.#rotateIfDue(this.#now())
+      return this.retaining
+    }
+    this.#horizonRead = true
+    const horizon = this.#horizon()
+    if (horizon.kind === 'none') {
+      if (horizon.reason === 'no-fec-manager') {
+        this.#forget()
+      } else {
+        // FEC still runs, but its horizon is not derivable: keep what the
+        // ledgers hold and bound them by count from here on.
+        this.#retentionMs = null
+        this.#countBounded = true
+      }
+      return this.retaining
+    }
+    const nowMs = this.#now()
+    this.#countBounded = false
+    if (this.#retentionMs === null) {
+      this.#retentionMs = horizon.retentionMs
+      this.#generationStartMs = nowMs
+      this.sourceSeqs.restartAging()
+      this.routedSeqs.restartAging()
+      this.parsedSymbols.restartAging()
+      return true
+    }
+    this.#retentionMs = horizon.retentionMs
+    this.#rotateIfDue(nowMs)
+    return true
+  }
+
+  /**
+   * Whether a packet with these wire numbers predates every number the stream
+   * has kept for at least one full FEC horizon (BLO-36599).
+   *
+   * Within one publisher epoch this cannot happen to a usable packet: it
+   * would have been sent before a packet that arrived a whole horizon ago, so
+   * no FEC block could still recover it and no reorder window could still place
+   * it. A publisher that restarts resets packet_sequence_number and SS_ID,
+   * and its first packets land exactly here. The evidence is in-band: two
+   * wire counters compared against numbers the stream actually received, with
+   * the horizon derived from the catalog. No timer is armed and no count
+   * threshold is guessed.
+   *
+   * Limits, by construction: a stream without an FEC horizon (count-bounded
+   * ledgers, or none) reports nothing, and so does a stream younger than two
+   * timed generations; a restart whose new counters land inside the retained
+   * range is not evidence, and its old numbers age out of the ledgers within
+   * one horizon.
+   */
+  predatesHorizon(packetSequenceNumber: number, ssId: number | undefined): boolean {
+    if (this.#retentionMs === null) return false
+    const seqFloor = this.sourceSeqs.horizonFloor
+    if (
+      seqFloor !== undefined &&
+      serial32Precedes(packetSequenceNumber, seqFloor) &&
+      !this.sourceSeqs.has(packetSequenceNumber)
+    ) {
+      return true
+    }
+    if (ssId === undefined) return false
+    const ssIdFloor = this.parsedSymbols.horizonFloor
+    return (
+      ssIdFloor !== undefined &&
+      serial32Precedes(ssId, ssIdFloor) &&
+      !this.parsedSymbols.has(ssId)
+    )
+  }
+
+  /** Whether the stream keeps ledgers, for its FEC horizon or by count. */
+  get retaining(): boolean {
+    return this.#retentionMs !== null || this.#countBounded
+  }
+
+  /** Whether the ledgers are bounded by count because no FEC horizon is derivable. */
+  get countBounded(): boolean {
+    return this.#countBounded
+  }
+
+  /** Current retention in ms, or null while the stream has no FEC horizon. */
+  get retentionMs(): number | null {
+    return this.#retentionMs
+  }
+
+  /** Forget every number (a track reset); the next packet re-derives the horizon. */
+  clear(): void {
+    this.#forget()
+    this.#lastSbn = undefined
+    this.#horizonRead = false
+  }
+
+  #forget(): void {
+    this.sourceSeqs.clear()
+    this.routedSeqs.clear()
+    this.parsedSymbols.clear()
+    this.#retentionMs = null
+    this.#countBounded = false
+  }
+
+  #rotateIfDue(nowMs: number): void {
+    if (this.#retentionMs === null || nowMs - this.#generationStartMs < this.#retentionMs) return
+    this.sourceSeqs.rotate()
+    this.routedSeqs.rotate()
+    this.parsedSymbols.rotate()
+    this.#generationStartMs = nowMs
+  }
+}

package/src/fec-codec.ts

@@ -0,0 +1,61 @@
+/**
+ * Generic FEC codec interface. Transport-agnostic, algorithm-pluggable.
+ * Implementations: NoneCodec (passthrough), RaptorQCodec (wraps WASM),
+ * ReedSolomonCodec (placeholder).
+ */
+export interface FecCodec {
+  readonly algorithm: 'raptorq' | 'reed-solomon' | 'none'
+  addSymbol(blockId: number, symbolId: number, data: Uint8Array, isRepair: boolean): Uint8Array | null
+  configure(params: Uint8Array): void
+  cleanup(now: number): Uint8Array
+}
+
+/** Passthrough codec for reliable transports (QUIC streams). Returns data immediately. */
+export class NoneCodec implements FecCodec {
+  readonly algorithm = 'none' as const
+
+  addSymbol(_blockId: number, _symbolId: number, data: Uint8Array, _isRepair: boolean): Uint8Array | null {
+    return data
+  }
+
+  configure(_params: Uint8Array): void {}
+
+  cleanup(_now: number): Uint8Array {
+    return new Uint8Array(0)
+  }
+}
+
+/**
+ * Create a FecCodec from a catalog algorithm name.
+ * RaptorQ is not returned here — it requires a WASM module and is injected
+ * via FECManager.setFecDecoder(). This factory covers non-WASM codecs.
+ */
+export function createFecCodec(algorithm: string): FecCodec {
+  switch (algorithm) {
+    case 'none': return new NoneCodec()
+    case 'reed-solomon': return new ReedSolomonCodec()
+    case 'raptorq':
+      // RaptorQ requires WASM — caller must use FECManager.setFecDecoder() instead.
+      // Return NoneCodec as a safe fallback (passthrough, no recovery).
+      console.warn('[FEC] RaptorQ requested via createFecCodec — use FECManager.setFecDecoder() for WASM RaptorQ. Falling back to NoneCodec.')
+      return new NoneCodec()
+    default:
+      console.warn(`[FEC] Unknown algorithm "${algorithm}", falling back to NoneCodec`)
+      return new NoneCodec()
+  }
+}
+
+/** Placeholder for Reed-Solomon GF(2^8) per RFC 5510. Not yet implemented. */
+export class ReedSolomonCodec implements FecCodec {
+  readonly algorithm = 'reed-solomon' as const
+
+  addSymbol(_blockId: number, _symbolId: number, _data: Uint8Array, _isRepair: boolean): Uint8Array | null {
+    return null
+  }
+
+  configure(_params: Uint8Array): void {}
+
+  cleanup(_now: number): Uint8Array {
+    return new Uint8Array(0)
+  }
+}