@blockcast/fec-worker 0.1.0

package/src/__tests__/fec-worker-wasm-contract.test.ts

@@ -0,0 +1,84 @@
+/**
+ * Contract tests: FEC Worker ↔ WASM decoder signatures.
+ *
+ * Prevents the regression fixed in session 08-08: fec-worker.ts used to call
+ *   wasmDecoder.add_source(sbn, esi, data, ts)       // 4 args, wrong
+ *   wasmDecoder.add_repair(sbn, esi, data, ts)       // 4 args, wrong
+ * while a fresh external libmmt `mmt_wasm.d.ts` exports
+ *   add_source(ss_id, data, timestamp)               // 3 args
+ *   add_repair(ss_start, ssb_length, rs_id, data, timestamp)   // 5 args
+ * The misalignment silently turned symbol bytes into numbers, which
+ * wasm-bindgen's passArray8ToWasm0 converted to empty allocations. Result:
+ * moqtail's FEC worker received empty symbols for every feed, so
+ * blocksRecovered stayed at 0 while repairPackets piled up.
+ *
+ * These tests pin the protocol fields carried in the worker commands so a
+ * future rename/restructure fails loudly at compile time.
+ */
+
+import { describe, it, expect, assertType } from "vitest";
+import type { FecWorkerCommand } from "../fec-worker-types.js";
+
+describe("FEC Worker ↔ WASM signature contract", () => {
+	it("feedSource command exposes ssId (flat 32-bit SS_ID) — matches WASM add_source(ss_id, data, timestamp)", () => {
+		const cmd: Extract<FecWorkerCommand, { type: "feedSource" }> = {
+			type: "feedSource",
+			ssId: 42,
+			data: new ArrayBuffer(1440),
+			ts: 1000,
+		};
+		// ssId must be a number. Any drift (e.g., renaming back to sbn+esi)
+		// breaks this line at compile time.
+		assertType<number>(cmd.ssId);
+		assertType<ArrayBuffer>(cmd.data);
+		assertType<number>(cmd.ts);
+		// @ts-expect-error — sbn must not exist on feedSource; that would
+		// resurrect the collapsed (sbn, esi) shape that misaligned WASM args.
+		const _hasSbn = cmd.sbn;
+		// @ts-expect-error — esi must not exist on feedSource either.
+		const _hasEsi = cmd.esi;
+		void _hasSbn;
+		void _hasEsi;
+		expect(cmd.ssId).toBe(42);
+	});
+
+	it("feedRepair command exposes ssStart/ssbLength/rsId — matches WASM add_repair(ss_start, ssb_length, rs_id, data, timestamp)", () => {
+		const cmd: Extract<FecWorkerCommand, { type: "feedRepair" }> = {
+			type: "feedRepair",
+			ssStart: 40,   // SBN * K (e.g. SBN=10, K=4)
+			ssbLength: 4,  // K
+			rsId: 2,       // 3rd repair symbol in the block
+			data: new ArrayBuffer(1440),
+			ts: 1000,
+		};
+		assertType<number>(cmd.ssStart);
+		assertType<number>(cmd.ssbLength);
+		assertType<number>(cmd.rsId);
+		assertType<ArrayBuffer>(cmd.data);
+		// @ts-expect-error — sbn must not exist on feedRepair; that field was
+		// the pre-computed collapse that broke recovery.
+		const _hasSbn = cmd.sbn;
+		// @ts-expect-error — esi must not exist on feedRepair either.
+		const _hasEsi = cmd.esi;
+		void _hasSbn;
+		void _hasEsi;
+		expect(cmd.ssStart).toBe(40);
+		expect(cmd.ssbLength).toBe(4);
+		expect(cmd.rsId).toBe(2);
+	});
+
+	it("WASM derivation is reversible — (ssStart=SBN*K, ssbLength=K, rsId) round-trips", () => {
+		// Pins the WASM-side derivation documented in mmt_wasm.d.ts:
+		//   SBN = ss_start / ssb_length
+		//   repair ESI = ssb_length + rs_id
+		// If those formulas change, the worker protocol must change with them.
+		const k = 4;
+		const sbn = 7;
+		const ssStart = sbn * k;
+		const rsId = 1;
+		const derivedSbn = Math.floor(ssStart / k);
+		const derivedRepairEsi = k + rsId;
+		expect(derivedSbn).toBe(sbn);
+		expect(derivedRepairEsi).toBe(5);
+	});
+});

package/src/__tests__/shred-fec-worker.test.ts

@@ -0,0 +1,448 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import type {
+	ShredWorkerCommand,
+	ShredWorkerEvent,
+	ShredWorkerConfig,
+} from "../shred-worker-types.js";
+import {
+	SHRED_WIRE_VERSION,
+	SHRED_HEADER_LEN,
+	SHRED_FLAG_IS_CODING,
+	SHRED_FLAG_DATA_COMPLETE,
+} from "../shred-worker-types.js";
+
+// --- Synthetic v3 wire frames -----------------------------------------------
+
+interface ShredFields {
+	version?: number;
+	slot?: bigint;
+	fecSetIndex?: number;
+	localIndex?: number;
+	numData?: number;
+	numCoding?: number;
+	sendTsUs?: bigint;
+	isCoding?: boolean;
+	dataComplete?: boolean;
+	payload?: Uint8Array;
+}
+
+/** Build a v3 (28-byte header) shred frame, matching the deployed player wire. */
+function makeV3Frame(f: ShredFields = {}): ArrayBuffer {
+	const payload = f.payload ?? new Uint8Array([1, 2, 3, 4]);
+	const buf = new Uint8Array(SHRED_HEADER_LEN + payload.byteLength);
+	const dv = new DataView(buf.buffer);
+	dv.setUint8(0, f.version ?? SHRED_WIRE_VERSION);
+	dv.setBigUint64(1, f.slot ?? 100n, true);
+	dv.setUint32(9, f.fecSetIndex ?? 0, true);
+	dv.setUint32(13, f.localIndex ?? 0, true);
+	let flags = 0;
+	if (f.dataComplete) flags |= SHRED_FLAG_DATA_COMPLETE;
+	if (f.isCoding) flags |= SHRED_FLAG_IS_CODING;
+	dv.setUint8(17, flags);
+	dv.setUint8(18, f.numData ?? 4);
+	dv.setUint8(19, f.numCoding ?? 2);
+	dv.setBigUint64(20, f.sendTsUs ?? 0n, true);
+	buf.set(payload, SHRED_HEADER_LEN);
+	return buf.buffer;
+}
+
+function nextTick(): Promise<void> {
+	return new Promise((resolve) => setTimeout(resolve, 0));
+}
+
+function last<T>(arr: T[]): T | undefined {
+	return arr[arr.length - 1];
+}
+
+// --- Worker harness (stubbed self + injected shred WASM bindings) -----------
+
+interface WorkerHarnessOptions {
+	addShredResult?: Uint8Array | null;
+	recoverResult?: Uint8Array | null;
+	/**
+	 * When set, the mock ShredFecBuffer exposes `try_recover_all_with_identity`
+	 * (the identity-carrying drain) returning this set once. Leaving it undefined
+	 * keeps the method absent so the worker falls back to `try_recover_all`.
+	 */
+	recoverIdentity?: { slot: bigint; fecSetIndex: number; payload: Uint8Array } | null;
+	pending?: number;
+	config?: Partial<ShredWorkerConfig>;
+	/** Inject the legacy 5-arg ShredReassembler instead of the 8-arg ShredFecBuffer. */
+	legacyReassembler?: boolean;
+}
+
+async function setupWorker(options: WorkerHarnessOptions = {}) {
+	const events: ShredWorkerEvent[] = [];
+	const addShredCalls: unknown[][] = [];
+	const tryRecoverCalls: number[] = [];
+	const tryRecoverIdentityCalls: number[] = [];
+	const ctorArgs: Array<[bigint, number]> = [];
+	const free = vi.fn();
+	const recoveredSetFree = vi.fn();
+
+	// 8-arg RS decoder — Function.length === 8 drives the RS dispatch branch.
+	class MockShredFecBuffer {
+		free = free;
+		constructor(maxSlots: bigint, symbolSize: number) {
+			ctorArgs.push([maxSlots, symbolSize]);
+		}
+		add_shred(
+			slot: bigint,
+			fecSetIndex: number,
+			localIndex: number,
+			numData: number,
+			numCoding: number,
+			isCoding: boolean,
+			payload: Uint8Array,
+			lastInSet: boolean,
+		): Uint8Array | null {
+			addShredCalls.push([
+				slot,
+				fecSetIndex,
+				localIndex,
+				numData,
+				numCoding,
+				isCoding,
+				payload,
+				lastInSet,
+			]);
+			return options.addShredResult ?? null;
+		}
+		try_recover_all(nowMs: number): Uint8Array | null {
+			tryRecoverCalls.push(nowMs);
+			return options.recoverResult ?? null;
+		}
+		pending_fec_sets(): number {
+			return options.pending ?? 0;
+		}
+	}
+
+	// Only expose the identity drain when the test opts in, so tests that leave
+	// `recoverIdentity` undefined exercise the legacy `try_recover_all` fallback
+	// (the worker branches on typeof ..._with_identity === "function").
+	if (options.recoverIdentity !== undefined) {
+		(
+			MockShredFecBuffer.prototype as unknown as {
+				try_recover_all_with_identity: (nowMs: number) => unknown;
+			}
+		).try_recover_all_with_identity = (nowMs: number) => {
+			tryRecoverIdentityCalls.push(nowMs);
+			const r = options.recoverIdentity;
+			if (!r) return null;
+			return {
+				slot: r.slot,
+				fec_set_index: r.fecSetIndex,
+				payload: r.payload,
+				free: recoveredSetFree,
+			};
+		};
+	}
+
+	// 5-arg legacy decoder — Function.length === 5 → data-only dispatch branch.
+	class MockShredReassembler {
+		free = free;
+		constructor(maxSlots: bigint, symbolSize: number) {
+			ctorArgs.push([maxSlots, symbolSize]);
+		}
+		add_shred(
+			slot: bigint,
+			fecSetIndex: number,
+			localIndex: number,
+			payload: Uint8Array,
+			lastInSet: boolean,
+		): Uint8Array | null {
+			addShredCalls.push([slot, fecSetIndex, localIndex, payload, lastInSet]);
+			return options.addShredResult ?? null;
+		}
+		pending_fec_sets(): number {
+			return options.pending ?? 0;
+		}
+	}
+
+	const bindings = options.legacyReassembler
+		? { ShredReassembler: MockShredReassembler }
+		: { ShredFecBuffer: MockShredFecBuffer };
+
+	const workerSelf = {
+		__SHRED_WASM_BINDINGS__: bindings,
+		onmessage: undefined as
+			| ((e: MessageEvent<ShredWorkerCommand>) => void)
+			| undefined,
+		onmessageerror: undefined as (() => void) | undefined,
+		postMessage: vi.fn((event: ShredWorkerEvent) => {
+			events.push(event);
+		}),
+	};
+
+	vi.stubGlobal("self", workerSelf);
+	vi.resetModules();
+	await import("../shred-fec-worker.ts");
+
+	const post = async (cmd: ShredWorkerCommand) => {
+		workerSelf.onmessage?.({ data: cmd } as MessageEvent<ShredWorkerCommand>);
+		await nextTick();
+		await nextTick();
+	};
+
+	// Synchronous dispatch (no await) — for testing back-to-back re-entrant commands.
+	const fire = (cmd: ShredWorkerCommand) =>
+		workerSelf.onmessage?.({ data: cmd } as MessageEvent<ShredWorkerCommand>);
+
+	await post({
+		type: "configure",
+		config: {
+			maxSlots: 5000,
+			symbolSize: 512,
+			statsIntervalMs: 0,
+			recoverIntervalMs: 0,
+			...options.config,
+		},
+	});
+
+	return {
+		events,
+		post,
+		fire,
+		addShredCalls,
+		tryRecoverCalls,
+		tryRecoverIdentityCalls,
+		free,
+		recoveredSetFree,
+		ctorArgs,
+	};
+}
+
+afterEach(() => {
+	vi.unstubAllGlobals();
+	vi.useRealTimers();
+});
+
+// --- Tests ------------------------------------------------------------------
+
+describe("shred FEC worker (v3 wire)", () => {
+	it("constructs the decoder with (BigInt(maxSlots), symbolSize)", async () => {
+		const w = await setupWorker({ config: { maxSlots: 5000, symbolSize: 512 } });
+		expect(w.ctorArgs).toEqual([[5000n, 512]]);
+		await w.post({ type: "dispose" });
+	});
+
+	it("parses a v3 frame and forwards exact fields to add_shred (DATA_COMPLETE → lastInSet)", async () => {
+		const w = await setupWorker();
+		const payload = new Uint8Array([10, 20, 30]);
+		await w.post({
+			type: "feedShred",
+			frame: makeV3Frame({
+				slot: 42n,
+				fecSetIndex: 5,
+				localIndex: 9,
+				numData: 8,
+				numCoding: 4,
+				isCoding: true,
+				dataComplete: true,
+				payload,
+			}),
+		});
+		expect(w.addShredCalls).toHaveLength(1);
+		const [slot, fecSet, localIdx, numData, numCoding, isCoding, gotPayload, last8] =
+			w.addShredCalls[0];
+		expect(slot).toBe(42n);
+		expect(fecSet).toBe(5);
+		expect(localIdx).toBe(9);
+		expect(numData).toBe(8);
+		expect(numCoding).toBe(4);
+		expect(isCoding).toBe(true);
+		expect(last8).toBe(true);
+		expect(Array.from(gotPayload as Uint8Array)).toEqual([10, 20, 30]);
+		await w.post({ type: "dispose" });
+	});
+
+	it("emits a recovered event with metadata when a set completes directly", async () => {
+		const recovered = new Uint8Array([0xaa, 0xbb, 0xcc]);
+		const w = await setupWorker({ addShredResult: recovered });
+		await w.post({
+			type: "feedShred",
+			frame: makeV3Frame({ slot: 77n, fecSetIndex: 3, sendTsUs: 123456n }),
+		});
+		const ev = w.events.find((e) => e.type === "recovered");
+		if (ev?.type !== "recovered") throw new Error("expected recovered");
+		expect(Array.from(new Uint8Array(ev.data))).toEqual([0xaa, 0xbb, 0xcc]);
+		expect(ev.meta.slot).toBe(77n);
+		expect(ev.meta.fecSetIndex).toBe(3);
+		expect(ev.meta.sendTsUs).toBe(123456n);
+		expect(ev.meta.recovered).toBe(false);
+		await w.post({ type: "dispose" });
+	});
+
+	it("drops a non-v3 frame (version guard) and counts it, not add_shred", async () => {
+		const w = await setupWorker({ config: { statsIntervalMs: 10 } });
+		await w.post({ type: "feedShred", frame: makeV3Frame({ version: 2 }) });
+		expect(w.addShredCalls).toHaveLength(0);
+		await new Promise((r) => setTimeout(r, 25));
+		const stats = last(w.events.filter((e) => e.type === "stats"));
+		if (stats?.type !== "stats") throw new Error("expected stats");
+		expect(stats.stats.versionMismatch).toBe(1);
+		expect(stats.stats.shredsRx).toBe(0);
+		await w.post({ type: "dispose" });
+	});
+
+	it("drops frames shorter than the v3 header and counts them malformed", async () => {
+		const w = await setupWorker({ config: { statsIntervalMs: 10 } });
+		await w.post({ type: "feedShred", frame: new Uint8Array(10).buffer });
+		expect(w.addShredCalls).toHaveLength(0);
+		await new Promise((r) => setTimeout(r, 25));
+		const stats = last(w.events.filter((e) => e.type === "stats"));
+		if (stats?.type !== "stats") throw new Error("expected stats");
+		expect(stats.stats.malformed).toBe(1);
+		await w.post({ type: "dispose" });
+	});
+
+	it("dispatches to the 5-arg legacy reassembler and drops coding shreds", async () => {
+		const w = await setupWorker({ legacyReassembler: true });
+		// Data shred → forwarded with 5 args.
+		await w.post({ type: "feedShred", frame: makeV3Frame({ isCoding: false }) });
+		// Coding shred → dropped by the legacy path (can't use it).
+		await w.post({ type: "feedShred", frame: makeV3Frame({ isCoding: true }) });
+		expect(w.addShredCalls).toHaveLength(1);
+		expect(w.addShredCalls[0]).toHaveLength(5); // 5-arg legacy signature
+		await w.post({ type: "dispose" });
+	});
+
+	it("errors when feedShred arrives before configure", async () => {
+		const events: ShredWorkerEvent[] = [];
+		const workerSelf = {
+			__SHRED_WASM_BINDINGS__: {
+				ShredFecBuffer: class {
+					add_shred() {
+						return null;
+					}
+					try_recover_all() {
+						return null;
+					}
+					pending_fec_sets() {
+						return 0;
+					}
+				},
+			},
+			onmessage: undefined as
+				| ((e: MessageEvent<ShredWorkerCommand>) => void)
+				| undefined,
+			postMessage: vi.fn((e: ShredWorkerEvent) => events.push(e)),
+		};
+		vi.stubGlobal("self", workerSelf);
+		vi.resetModules();
+		await import("../shred-fec-worker.ts");
+		workerSelf.onmessage?.({
+			data: { type: "feedShred", frame: makeV3Frame() },
+		} as MessageEvent<ShredWorkerCommand>);
+		await nextTick();
+		const err = events.find((e) => e.type === "error");
+		expect(err?.type === "error" && err.code).toBe("not-configured");
+	});
+
+	it("recoverAll drives try_recover_all and emits recovered:true", async () => {
+		const late = new Uint8Array([1, 2]);
+		const w = await setupWorker({ recoverResult: late });
+		await w.post({ type: "recoverAll", nowMs: 1000 });
+		expect(w.tryRecoverCalls).toEqual([1000]);
+		const ev = w.events.find((e) => e.type === "recovered");
+		if (ev?.type !== "recovered") throw new Error("expected recovered");
+		expect(ev.meta.recovered).toBe(true);
+		expect(ev.meta.slot).toBe(0n);
+		await w.post({ type: "dispose" });
+	});
+
+	it("recover path attributes ingest-recorded send_ts to the recovered set", async () => {
+		const payload = new Uint8Array([9, 8, 7]);
+		// Identity drain present: worker prefers it and attributes the send_ts it
+		// recorded at ingest for (slot 55, fecSet 2) rather than emitting 0n.
+		const w = await setupWorker({
+			recoverIdentity: { slot: 55n, fecSetIndex: 2, payload },
+		});
+		// Feed a data shred for the set so its send_ts is on record; addShredResult
+		// defaults to null, so the set does NOT complete inline — it must recover.
+		await w.post({
+			type: "feedShred",
+			frame: makeV3Frame({ slot: 55n, fecSetIndex: 2, sendTsUs: 999000n }),
+		});
+		expect(w.events.some((e) => e.type === "recovered")).toBe(false);
+		await w.post({ type: "recoverAll", nowMs: 2000 });
+		expect(w.tryRecoverIdentityCalls).toEqual([2000]);
+		expect(w.tryRecoverCalls).toEqual([]); // identity path taken, not the legacy one
+		const ev = w.events.find((e) => e.type === "recovered");
+		if (ev?.type !== "recovered") throw new Error("expected recovered");
+		expect(Array.from(new Uint8Array(ev.data))).toEqual([9, 8, 7]);
+		expect(ev.meta.slot).toBe(55n);
+		expect(ev.meta.fecSetIndex).toBe(2);
+		expect(ev.meta.sendTsUs).toBe(999000n);
+		expect(ev.meta.recovered).toBe(true);
+		expect(w.recoveredSetFree).toHaveBeenCalledTimes(1); // WASM-owned set freed
+		await w.post({ type: "dispose" });
+	});
+
+	it("batch-drains a transferred ReadableStream of frames", async () => {
+		const recovered = new Uint8Array([5]);
+		const w = await setupWorker({ addShredResult: recovered });
+		const frames = [
+			makeV3Frame({ slot: 1n }),
+			makeV3Frame({ slot: 2n }),
+			makeV3Frame({ slot: 3n }),
+		];
+		const readable = new ReadableStream<Uint8Array>({
+			start(controller) {
+				for (const f of frames) controller.enqueue(new Uint8Array(f));
+				controller.close();
+			},
+		});
+		await w.post({ type: "attachStream", readable });
+		await new Promise((r) => setTimeout(r, 20));
+		expect(w.addShredCalls).toHaveLength(3);
+		expect(w.events.filter((e) => e.type === "recovered")).toHaveLength(3);
+		await w.post({ type: "dispose" });
+	});
+
+	it("emits a coalesced stats heartbeat with counters + slot samples", async () => {
+		const w = await setupWorker({ pending: 4, config: { statsIntervalMs: 15 } });
+		await w.post({ type: "feedShred", frame: makeV3Frame({ slot: 100n, sendTsUs: 555n, isCoding: false }) });
+		await w.post({ type: "feedShred", frame: makeV3Frame({ slot: 100n, isCoding: true }) });
+		await w.post({ type: "feedShred", frame: makeV3Frame({ slot: 101n, sendTsUs: 777n, isCoding: false }) });
+		await new Promise((r) => setTimeout(r, 30));
+		const statsEvents = w.events.filter((e) => e.type === "stats");
+		const stats = last(statsEvents);
+		if (stats?.type !== "stats") throw new Error("expected stats");
+		expect(stats.stats.shredsRx).toBe(3);
+		expect(stats.stats.sourceShreds).toBe(2);
+		expect(stats.stats.repairShreds).toBe(1);
+		expect(stats.stats.pendingFecSets).toBe(4);
+		expect(stats.stats.currentSlot).toBe("101");
+		// Samples drain per-heartbeat; a consumer aggregates across heartbeats.
+		// One sample per distinct slot (first shred of each), not per shred.
+		const allSamples = statsEvents.flatMap((e) =>
+			e.type === "stats" ? e.stats.slotSamples : [],
+		);
+		expect(allSamples.map((s) => [s.slot, s.sendTsUs])).toEqual([
+			[100n, 555n],
+			[101n, 777n],
+		]);
+		await w.post({ type: "dispose" });
+	});
+
+	it("frees the decoder on dispose", async () => {
+		const w = await setupWorker();
+		await w.post({ type: "dispose" });
+		expect(w.free).toHaveBeenCalled();
+	});
+
+	it("guards re-entrant configure(): the superseded call bails before constructing a decoder", async () => {
+		// After the harness's initial configure (1 ctor), fire two more back-to-back
+		// without awaiting. The first of the two is superseded mid-load by the second,
+		// so it must bail before `new Ctor` — only the winner constructs (1 more ctor,
+		// not 2), and no orphaned timer is armed.
+		const w = await setupWorker({ config: { statsIntervalMs: 0, recoverIntervalMs: 0 } });
+		const cfg = { maxSlots: 5000, symbolSize: 512, statsIntervalMs: 0, recoverIntervalMs: 0 };
+		w.fire({ type: "configure", config: cfg });
+		w.fire({ type: "configure", config: cfg });
+		await new Promise((r) => setTimeout(r, 10));
+		expect(w.ctorArgs).toHaveLength(2); // initial + winner; the superseded call bailed
+		await w.post({ type: "dispose" });
+	});
+});