@blockcast/fec-worker 0.1.0 → 0.3.0-main.d9354616fdf2

package/src/index.ts

@@ -6,8 +6,11 @@
 
 export type {
 	FecWorkerCommand,
+	FecSymbolCommand,
+	FecWorkerOptionalCommand,
 	FecWorkerEvent,
 	FecWorkerErrorCode,
+	FecBlockProfile,
 	FecTrackConfig,
 	FecTrackStats,
 	FecBlockSnapshot,

package/src/shred-fec-worker.ts

@@ -28,6 +28,7 @@ import type {
 } from "./shred-worker-types.js";
 import {
 	SHRED_WIRE_VERSION,
+	SHRED_WIRE_VERSION_FULL,
 	SHRED_HEADER_LEN,
 	SHRED_FLAG_IS_CODING,
 	SHRED_FLAG_DATA_COMPLETE,
@@ -216,10 +217,33 @@ function postStats(): void {
 
 // --- Wire parse + decode ----------------------------------------------------
 
+/**
+ * The v3 erasure-shard view of a v4 body (a full canonical shred), or null when
+ * the body is not a shred the forwarder could have emitted. Mirrors the
+ * forwarder's `parse_shred()` checks and `erasure_shard()` slice exactly, so a
+ * v4 frame reaches `add_shred` byte-identical to its v3 twin. A view, not a
+ * copy: the hot path allocates nothing extra for v4.
+ */
+function erasureShardOfFullShred(shred: Uint8Array, isCoding: boolean): Uint8Array | null {
+	if (shred.byteLength !== (isCoding ? 1228 : 1203)) return null;
+	const variant = shred[64];
+	const family = variant & 0xf0;
+	const variantIsCoding = family === 0x60 || family === 0x70;
+	const variantIsData = family === 0x90 || family === 0xb0;
+	if (isCoding ? !variantIsCoding : !variantIsData) return null;
+	const resigned = family === 0xb0 || family === 0x70;
+	const trailer = 32 + (variant & 0x0f) * 20 + (resigned ? 64 : 0);
+	const start = isCoding ? 89 : 64;
+	const end = shred.byteLength - trailer;
+	if (end <= start) return null;
+	return shred.subarray(start, end);
+}
+
 /**
  * Parse one raw shred frame and feed the decoder. Mirrors the deployed player's
  * `onShredBytes` (v3 wire, version-guarded) so the off-thread path is
- * bit-identical to the proven on-thread path. Emits a `recovered` event when a
+ * bit-identical to the proven on-thread path, and accepts v4 by reducing its
+ * full shred to that same v3 erasure shard. Emits a `recovered` event when a
  * set completes directly.
  */
 function ingestFrame(frame: Uint8Array): void {
@@ -228,7 +252,8 @@ function ingestFrame(frame: Uint8Array): void {
 		return;
 	}
 	const dv = new DataView(frame.buffer, frame.byteOffset, frame.byteLength);
-	if (dv.getUint8(0) !== SHRED_WIRE_VERSION) {
+	const version = dv.getUint8(0);
+	if (version !== SHRED_WIRE_VERSION && version !== SHRED_WIRE_VERSION_FULL) {
 		// Forwarder/worker wire-version skew — drop rather than misframe every shred.
 		stats.versionMismatch++;
 		return;
@@ -242,7 +267,13 @@ function ingestFrame(frame: Uint8Array): void {
 	const numData = dv.getUint8(18);
 	const numCoding = dv.getUint8(19);
 	const sendTsUs = dv.getBigUint64(20, true);
-	const payload = frame.subarray(SHRED_HEADER_LEN);
+	const body = frame.subarray(SHRED_HEADER_LEN);
+	const payload =
+		version === SHRED_WIRE_VERSION ? body : erasureShardOfFullShred(body, isCoding);
+	if (payload === null) {
+		stats.malformed++;
+		return;
+	}
 
 	stats.shredsRx++;
 	if (isCoding) stats.repairShreds++;

package/src/shred-worker-types.ts

@@ -28,7 +28,7 @@
  * path is bit-identical to the proven on-thread path.
  *
  * v3 (28-byte header):
- *   [0]      version       u8   (== 3; non-3 frames are dropped, not misframed)
+ *   [0]      version       u8   (3 or 4; any other version is dropped, not misframed)
  *   [1-8]    slot          u64 LE
  *   [9-12]   fecSetIndex   u32 LE
  *   [13-16]  localIndex    u32 LE   (data: 0..numData; coding: numData+position)
@@ -41,8 +41,23 @@
  * DATA_COMPLETE (last data shred in the set) is passed to `add_shred` as its
  * `lastInSet` argument — the WASM uses it to infer numData for a set that
  * received only data shreds (no coding).
+ *
+ * v4 has the same 28-byte header, but its body is the full canonical Agave
+ * shred (signature, headers, erasure shard, chained Merkle root, proof, and
+ * the retransmitter signature on resigned variants) — the form a validator TVU
+ * accepts (BLO-41383). The worker recovers the exact v3 erasure-shard view
+ * from it with the forwarder's own `erasure_shard()` slice, so both versions
+ * reach `add_shred` byte-identical:
+ *   variant      = shred[64]; data 0x9_/0xb_, coding 0x6_/0x7_ (chained merkle;
+ *                  0xb_/0x7_ resigned), must agree with IS_CODING_SHRED
+ *   length       = 1203 (data) / 1228 (coding), exactly
+ *   shard start  = 64 (data: after the signature) / 89 (coding headers)
+ *   shard end    = length - (32 root + 20 * (variant & 0x0f) proof
+ *                            + 64 retransmitter signature if resigned)
  */
 export const SHRED_WIRE_VERSION = 3;
+/** Wire version whose body is the full canonical shred (see above). */
+export const SHRED_WIRE_VERSION_FULL = 4;
 export const SHRED_HEADER_LEN = 28;
 export const SHRED_FLAG_DATA_COMPLETE = 0x01;
 export const SHRED_FLAG_IS_CODING = 0x02;

package/src/__tests__/fec-worker-alta.test.ts

@@ -1,231 +0,0 @@
-/**
- * Unit tests for ALTA verification behavior in FEC Worker.
- *
- * These tests validate the verification logic rules from D-05:
- *   - altaVerified increments on valid packets
- *   - altaFailed increments on invalid packets
- *   - Packets are NEVER dropped regardless of ALTA outcome (count + play)
- *   - FrameMeta.altaVerified reflects actual result (true/false/null)
- *
- * The FEC Worker runs in DedicatedWorkerGlobalScope. We test the
- * verification logic by directly simulating the branching behavior
- * documented in fec-worker.ts (lines ~236-263).
- *
- * Note: Full Worker integration tests (with postMessage) require a DOM
- * environment. These unit tests focus on the verification logic rules only.
- */
-
-import { describe, it, expect, vi } from "vitest";
-
-// --- Mock JsAltaVerifier interface (mirrors alta_rs.d.ts) ---
-
-interface MockAltaVerifyResult {
-	valid: boolean;
-	sequence_number: number;
-	payload?: Uint8Array;
-	error?: string | null;
-}
-
-interface MockJsAltaVerifier {
-	verify(packet: Uint8Array): MockAltaVerifyResult;
-	free(): void;
-}
-
-function makeVerifier(valid: boolean, throws?: boolean): MockJsAltaVerifier {
-	return {
-		verify(_packet: Uint8Array): MockAltaVerifyResult {
-			if (throws) throw new Error("WASM error");
-			return {
-				valid,
-				sequence_number: 1,
-				payload: valid ? new Uint8Array(100) : undefined,
-				error: valid ? null : "invalid signature",
-			};
-		},
-		free: vi.fn(),
-	};
-}
-
-// --- Simulate the fec-worker verification logic ---
-// Extracted from fec-worker.ts handleFeedSource() body.
-// This mirrors the real implementation for unit-testability without a Worker context.
-
-interface VerifyState {
-	altaEnabled: boolean;
-	altaVerifier: MockJsAltaVerifier | null;
-	altaVerified: number;
-	altaFailed: number;
-}
-
-interface VerifyResult {
-	symbolAltaVerified: boolean | null;
-	packetDropped: boolean;  // Must always be false (D-05)
-}
-
-function simulateVerification(
-	state: VerifyState,
-	packet: Uint8Array,
-): VerifyResult {
-	let symbolAltaVerified: boolean | null = null;
-	let packetDropped = false;
-
-	if (state.altaEnabled && state.altaVerifier) {
-		try {
-			const result = state.altaVerifier.verify(packet);
-			if (result.valid) {
-				state.altaVerified++;
-				symbolAltaVerified = true;
-			} else {
-				state.altaFailed++;
-				symbolAltaVerified = false;
-				// D-05: count + play — do NOT return or drop the packet
-			}
-		} catch (_e) {
-			state.altaFailed++;
-			symbolAltaVerified = false;
-			// WASM error: packet still passes through (D-05)
-		}
-	} else if (state.altaEnabled && !state.altaVerifier) {
-		// Public key present but WASM init failed — not a tamper indicator
-		symbolAltaVerified = null;
-	}
-	// If not altaEnabled: symbolAltaVerified stays null
-
-	return { symbolAltaVerified, packetDropped };
-}
-
-// --- Tests ---
-
-describe("ALTA verification in FEC Worker (D-05 compliance)", () => {
-	const packet = new Uint8Array(1320);
-
-	it("Case 1: altaEnabled=false — no verification attempted, counters stay 0", () => {
-		const state: VerifyState = {
-			altaEnabled: false,
-			altaVerifier: makeVerifier(true),
-			altaVerified: 0,
-			altaFailed: 0,
-		};
-		const result = simulateVerification(state, packet);
-		expect(result.symbolAltaVerified).toBeNull();
-		expect(result.packetDropped).toBe(false);
-		expect(state.altaVerified).toBe(0);
-		expect(state.altaFailed).toBe(0);
-	});
-
-	it("Case 2: altaEnabled=true, valid packet — altaVerified increments", () => {
-		const state: VerifyState = {
-			altaEnabled: true,
-			altaVerifier: makeVerifier(true),
-			altaVerified: 0,
-			altaFailed: 0,
-		};
-		const result = simulateVerification(state, packet);
-		expect(result.symbolAltaVerified).toBe(true);
-		expect(result.packetDropped).toBe(false);
-		expect(state.altaVerified).toBe(1);
-		expect(state.altaFailed).toBe(0);
-	});
-
-	it("Case 3: altaEnabled=true, invalid packet — altaFailed increments, packet NOT dropped (D-05)", () => {
-		const state: VerifyState = {
-			altaEnabled: true,
-			altaVerifier: makeVerifier(false),
-			altaVerified: 0,
-			altaFailed: 0,
-		};
-		const result = simulateVerification(state, packet);
-		expect(result.symbolAltaVerified).toBe(false);
-		expect(result.packetDropped).toBe(false);  // D-05: count + play
-		expect(state.altaVerified).toBe(0);
-		expect(state.altaFailed).toBe(1);
-	});
-
-	it("Case 4: altaEnabled=true, altaVerifier=null (WASM init failed) — no crash, no altaFailed increment", () => {
-		const state: VerifyState = {
-			altaEnabled: true,
-			altaVerifier: null,  // WASM init failed
-			altaVerified: 0,
-			altaFailed: 0,
-		};
-		const result = simulateVerification(state, packet);
-		// altaVerified is null (unverified due to init failure, not tamper)
-		expect(result.symbolAltaVerified).toBeNull();
-		expect(result.packetDropped).toBe(false);
-		expect(state.altaVerified).toBe(0);
-		// CRITICAL: altaFailed must NOT increment for init failures (not a tamper indicator)
-		expect(state.altaFailed).toBe(0);
-	});
-
-	it("Case 5: altaVerifier.verify() throws — altaFailed increments, packet still passes (D-05)", () => {
-		const state: VerifyState = {
-			altaEnabled: true,
-			altaVerifier: makeVerifier(true, /* throws */ true),
-			altaVerified: 0,
-			altaFailed: 0,
-		};
-		const result = simulateVerification(state, packet);
-		expect(result.symbolAltaVerified).toBe(false);
-		expect(result.packetDropped).toBe(false);  // D-05: even on WASM error
-		expect(state.altaVerified).toBe(0);
-		expect(state.altaFailed).toBe(1);
-	});
-
-	it("Case 6: FrameMeta.altaVerified reflects actual result (true/false/null)", () => {
-		// Valid packet → true
-		const stateValid: VerifyState = { altaEnabled: true, altaVerifier: makeVerifier(true), altaVerified: 0, altaFailed: 0 };
-		expect(simulateVerification(stateValid, packet).symbolAltaVerified).toBe(true);
-
-		// Invalid packet → false
-		const stateInvalid: VerifyState = { altaEnabled: true, altaVerifier: makeVerifier(false), altaVerified: 0, altaFailed: 0 };
-		expect(simulateVerification(stateInvalid, packet).symbolAltaVerified).toBe(false);
-
-		// ALTA not enabled → null
-		const stateDisabled: VerifyState = { altaEnabled: false, altaVerifier: makeVerifier(true), altaVerified: 0, altaFailed: 0 };
-		expect(simulateVerification(stateDisabled, packet).symbolAltaVerified).toBeNull();
-
-		// ALTA enabled but WASM init failed → null (not false)
-		const stateNoWasm: VerifyState = { altaEnabled: true, altaVerifier: null, altaVerified: 0, altaFailed: 0 };
-		expect(simulateVerification(stateNoWasm, packet).symbolAltaVerified).toBeNull();
-	});
-
-	it("Multiple valid packets — altaVerified accumulates correctly", () => {
-		const state: VerifyState = {
-			altaEnabled: true,
-			altaVerifier: makeVerifier(true),
-			altaVerified: 0,
-			altaFailed: 0,
-		};
-		for (let i = 0; i < 32; i++) {
-			simulateVerification(state, packet);
-		}
-		expect(state.altaVerified).toBe(32);
-		expect(state.altaFailed).toBe(0);
-	});
-
-	it("Mixed valid/invalid packets — counters track independently", () => {
-		const state: VerifyState = {
-			altaEnabled: true,
-			altaVerifier: null as unknown as MockJsAltaVerifier,
-			altaVerified: 0,
-			altaFailed: 0,
-		};
-		// 20 valid, 5 invalid, 3 throws
-		let callCount = 0;
-		state.altaVerifier = {
-			verify(_p: Uint8Array): MockAltaVerifyResult {
-				callCount++;
-				if (callCount <= 20) return { valid: true, sequence_number: callCount };
-				if (callCount <= 25) return { valid: false, sequence_number: callCount, error: "tampered" };
-				throw new Error("WASM failure");
-			},
-			free: vi.fn(),
-		};
-
-		for (let i = 0; i < 28; i++) {
-			simulateVerification(state, packet);
-		}
-		expect(state.altaVerified).toBe(20);
-		expect(state.altaFailed).toBe(8);  // 5 invalid + 3 throws
-	});
-});

package/src/__tests__/fec-worker-cleanup-integration.test.ts

@@ -1,250 +0,0 @@
-import { afterEach, describe, expect, it, vi } from "vitest";
-import { FecWorkerClient } from "../fec-worker-client.js";
-import type {
-	FecWorkerCommand,
-	FecWorkerEvent,
-	FecTrackConfig,
-} from "../fec-worker-types.js";
-
-const EMPTY_WASM_MODULE = new WebAssembly.Module(
-	new Uint8Array([0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00]),
-);
-
-const CONFIG: FecTrackConfig = {
-	codec: "avc1.64001f",
-	trackName: "video",
-	trackType: "video",
-	algorithm: "raptorq",
-	k: 4,
-	repairCount: 2,
-	symbolSize: 8,
-	interleaveDepth: 4,
-	interleaveMs: 100,
-	deliveryWindowMs: 200,
-	fecMode: "subframe",
-	altaEnabled: false,
-	relayBlockSigEnabled: false,
-};
-
-function nextTick(): Promise<void> {
-	return new Promise((resolve) => setTimeout(resolve, 0));
-}
-
-function frameCleanupRecords(
-	records: Array<{ sbn: number; data: Uint8Array }>,
-): Uint8Array {
-	const size = records.reduce((total, record) => total + 8 + record.data.byteLength, 0);
-	const framed = new Uint8Array(size);
-	const view = new DataView(framed.buffer);
-	let offset = 0;
-	for (const record of records) {
-		view.setUint32(offset, record.sbn, false);
-		view.setUint32(offset + 4, record.data.byteLength, false);
-		offset += 8;
-		framed.set(record.data, offset);
-		offset += record.data.byteLength;
-	}
-	return framed;
-}
-
-afterEach(() => {
-	vi.unstubAllGlobals();
-});
-
-describe("FEC worker cleanup integration", () => {
-	it("keeps per-SBN green-fill ownership through a coalesced client/worker round trip", async () => {
-		const fill16 = new Uint8Array([0, 0, 0, 1, 0x0d]);
-		const fill18 = new Uint8Array([0, 0, 0, 1, 0x0d]);
-		let cleanupPass = 0;
-			const cleanupBlocksDetailed = vi.fn((sbns: Uint32Array) => {
-				cleanupPass++;
-				return cleanupPass === 1
-					? new Uint8Array()
-					: frameCleanupRecords([
-							...Array.from(sbns, (sbn) => ({
-								sbn,
-								data: sbn === 16 ? fill16 : fill18,
-							})),
-						]);
-			});
-
-		class MockDecoder {
-			static last: MockDecoder | undefined;
-			constructor() {
-				MockDecoder.last = this;
-			}
-			add_source = vi.fn(() => undefined);
-			add_repair = vi.fn(() => undefined);
-			configure_params = vi.fn();
-				cleanup_blocks_detailed = cleanupBlocksDetailed;
-				cleanup_by_sbn = vi.fn(() => new Uint32Array());
-			flush = vi.fn();
-			pending_blocks = vi.fn(() => 0);
-			reset_stats = vi.fn();
-			get_stats = vi.fn(() => ({
-				source_packets: 0n,
-				repair_packets: 0n,
-				blocks_complete: 0n,
-				blocks_recovered: 0n,
-				blocks_failed: 0n,
-				bytes_recovered: 0n,
-				recovery_rate: () => 0,
-			}));
-			free = vi.fn();
-		}
-
-		let workerOnMessage:
-			| ((event: MessageEvent<FecWorkerCommand>) => void)
-			| undefined;
-		const workerCommands: FecWorkerCommand[] = [];
-		const pageWorker = {
-			onmessage: null as ((event: MessageEvent<FecWorkerEvent>) => void) | null,
-			onerror: null as ((event: ErrorEvent) => void) | null,
-			onmessageerror: null as ((event: MessageEvent) => void) | null,
-			postMessage: vi.fn((command: FecWorkerCommand) => {
-				workerCommands.push(command);
-				workerOnMessage?.({ data: command } as MessageEvent<FecWorkerCommand>);
-			}),
-			terminate: vi.fn(),
-		};
-		const workerScope = {
-			__MMT_WASM_BINDINGS__: {
-				initSync: vi.fn(),
-				MmtFecDecoder: MockDecoder,
-			},
-			get onmessage() {
-				return workerOnMessage;
-			},
-			set onmessage(handler) {
-				workerOnMessage = handler;
-			},
-			postMessage: vi.fn((event: FecWorkerEvent) => {
-				pageWorker.onmessage?.({ data: event } as MessageEvent<FecWorkerEvent>);
-			}),
-		};
-
-		vi.stubGlobal("self", workerScope);
-		vi.resetModules();
-		await import("../fec-worker.ts");
-
-		const client = new FecWorkerClient(pageWorker as unknown as Worker);
-		const completed: number[] = [];
-		const fills: Array<{ sbn: number; data: number[] }> = [];
-		client.onCleanupComplete((sbn) => completed.push(sbn));
-		client.onGreenFill((sbn, data) => {
-			fills.push({ sbn, data: [...new Uint8Array(data)] });
-		});
-
-		client.configure(CONFIG, EMPTY_WASM_MODULE);
-		await nextTick();
-		await nextTick();
-		client.cleanup(15);
-		client.cleanup(16);
-		client.cleanup(18);
-		for (let i = 0; i < 6; i++) await nextTick();
-
-		expect(
-			workerCommands
-				.filter((command) => command.type === "cleanup")
-				.map((command) => command.type === "cleanup" ? command.sbns : []),
-		).toEqual([[15], [16, 18]]);
-			expect(cleanupBlocksDetailed).toHaveBeenCalledTimes(2);
-			expect(Array.from(cleanupBlocksDetailed.mock.calls[0]![0])).toEqual([15]);
-			expect(Array.from(cleanupBlocksDetailed.mock.calls[1]![0])).toEqual([16, 18]);
-		expect(fills).toEqual([
-			{ sbn: 16, data: [...fill16] },
-			{ sbn: 18, data: [...fill18] },
-		]);
-		expect(completed).toEqual([15, 16, 18]);
-
-		client.dispose();
-		await nextTick();
-		await nextTick();
-	});
-
-	it("keeps reset quarantined after surfacing a decoder reset failure", async () => {
-		class MockDecoder {
-			static last: MockDecoder | undefined;
-			constructor() {
-				MockDecoder.last = this;
-			}
-			add_source = vi.fn(() => undefined);
-			add_repair = vi.fn(() => undefined);
-			configure_params = vi.fn();
-			flush = vi.fn(() => {
-				throw new Error("flush failed");
-			});
-			pending_blocks = vi.fn(() => 0);
-			reset_stats = vi.fn();
-			get_stats = vi.fn(() => ({
-				source_packets: 0n,
-				repair_packets: 0n,
-				blocks_complete: 0n,
-				blocks_recovered: 0n,
-				blocks_failed: 0n,
-				bytes_recovered: 0n,
-				recovery_rate: () => 0,
-			}));
-			free = vi.fn();
-		}
-
-		let workerOnMessage:
-			| ((event: MessageEvent<FecWorkerCommand>) => void)
-			| undefined;
-		const pageWorker = {
-			onmessage: null as ((event: MessageEvent<FecWorkerEvent>) => void) | null,
-			onerror: null as ((event: ErrorEvent) => void) | null,
-			onmessageerror: null as ((event: MessageEvent) => void) | null,
-			postMessage: vi.fn((command: FecWorkerCommand) => {
-				workerOnMessage?.({ data: command } as MessageEvent<FecWorkerCommand>);
-			}),
-			terminate: vi.fn(),
-		};
-		const workerScope = {
-			__MMT_WASM_BINDINGS__: {
-				initSync: vi.fn(),
-				MmtFecDecoder: MockDecoder,
-			},
-			get onmessage() {
-				return workerOnMessage;
-			},
-			set onmessage(handler) {
-				workerOnMessage = handler;
-			},
-			postMessage: vi.fn((event: FecWorkerEvent) => {
-				pageWorker.onmessage?.({ data: event } as MessageEvent<FecWorkerEvent>);
-			}),
-		};
-
-		vi.stubGlobal("self", workerScope);
-		vi.resetModules();
-		await import("../fec-worker.ts");
-
-		const client = new FecWorkerClient(pageWorker as unknown as Worker);
-		const errors: Array<{ message: string; code: string | undefined }> = [];
-		const snapshots: unknown[] = [];
-		client.onError((message, code) => errors.push({ message, code }));
-		client.onSnapshot((stats) => snapshots.push(stats));
-
-		client.configure(CONFIG, EMPTY_WASM_MODULE);
-		await nextTick();
-		await nextTick();
-		snapshots.length = 0;
-
-		client.reset();
-		for (let i = 0; i < 4; i++) await nextTick();
-
-		expect(MockDecoder.last?.flush).toHaveBeenCalledOnce();
-		expect(errors).toEqual([
-			{
-				message: "[FecWorker] reset failed: flush failed",
-				code: "reset-failed",
-			},
-		]);
-		expect(snapshots).toHaveLength(0);
-
-		client.dispose();
-		await nextTick();
-		await nextTick();
-	});
-});