@blockcast/fec-worker 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/README.md +213 -0
  2. package/dist/.build-stamp +0 -0
  3. package/dist/fec-worker-client.d.ts +80 -0
  4. package/dist/fec-worker-client.d.ts.map +1 -0
  5. package/dist/fec-worker-client.js +270 -0
  6. package/dist/fec-worker-client.js.map +1 -0
  7. package/dist/fec-worker-types.d.ts +252 -0
  8. package/dist/fec-worker-types.d.ts.map +1 -0
  9. package/dist/fec-worker-types.js +11 -0
  10. package/dist/fec-worker-types.js.map +1 -0
  11. package/dist/fec-worker.d.ts +14 -0
  12. package/dist/fec-worker.d.ts.map +1 -0
  13. package/dist/fec-worker.js +565 -0
  14. package/dist/fec-worker.js.map +7 -0
  15. package/dist/index.d.ts +10 -0
  16. package/dist/index.d.ts.map +1 -0
  17. package/dist/index.js +8 -0
  18. package/dist/index.js.map +1 -0
  19. package/dist/shred-fec-worker.d.ts +20 -0
  20. package/dist/shred-fec-worker.d.ts.map +1 -0
  21. package/dist/shred-fec-worker.js +349 -0
  22. package/dist/shred-fec-worker.js.map +7 -0
  23. package/dist/shred-worker-client.d.ts +104 -0
  24. package/dist/shred-worker-client.d.ts.map +1 -0
  25. package/dist/shred-worker-client.js +181 -0
  26. package/dist/shred-worker-client.js.map +1 -0
  27. package/dist/shred-worker-types.d.ts +179 -0
  28. package/dist/shred-worker-types.d.ts.map +1 -0
  29. package/dist/shred-worker-types.js +47 -0
  30. package/dist/shred-worker-types.js.map +1 -0
  31. package/dist/transfer.d.ts +9 -0
  32. package/dist/transfer.d.ts.map +1 -0
  33. package/dist/transfer.js +13 -0
  34. package/dist/transfer.js.map +1 -0
  35. package/package.json +54 -0
  36. package/src/__tests__/fec-worker-alta.test.ts +231 -0
  37. package/src/__tests__/fec-worker-cleanup-integration.test.ts +250 -0
  38. package/src/__tests__/fec-worker-pending-queue.test.ts +315 -0
  39. package/src/__tests__/fec-worker-repair-size.test.ts +1029 -0
  40. package/src/__tests__/fec-worker-wasm-contract.test.ts +84 -0
  41. package/src/__tests__/shred-fec-worker.test.ts +448 -0
  42. package/src/fec-worker-client.test.ts +530 -0
  43. package/src/fec-worker-client.ts +309 -0
  44. package/src/fec-worker-types.test.ts +400 -0
  45. package/src/fec-worker-types.ts +268 -0
  46. package/src/fec-worker.ts +1048 -0
  47. package/src/index.ts +32 -0
  48. package/src/shred-fec-worker.ts +558 -0
  49. package/src/shred-worker-client.test.ts +182 -0
  50. package/src/shred-worker-client.ts +222 -0
  51. package/src/shred-worker-types.ts +194 -0
  52. package/src/transfer.test.ts +24 -0
  53. package/src/transfer.ts +12 -0
@@ -0,0 +1,182 @@
1
+ import { describe, it, expect } from "vitest";
2
+ import type {
3
+ ShredWorkerCommand,
4
+ ShredWorkerEvent,
5
+ ShredWorkerStats,
6
+ } from "./shred-worker-types.js";
7
+ import { ShredWorkerClient } from "./shred-worker-client.js";
8
+
9
+ class MockWorker {
10
+ onmessage: ((e: MessageEvent) => void) | null = null;
11
+ onerror: ((e: ErrorEvent) => void) | null = null;
12
+ onmessageerror: ((e: MessageEvent) => void) | null = null;
13
+ posted: Array<{ data: ShredWorkerCommand; transfer: Transferable[] }> = [];
14
+ terminated = false;
15
+
16
+ postMessage(data: ShredWorkerCommand, transfer?: Transferable[]): void {
17
+ this.posted.push({ data, transfer: transfer ?? [] });
18
+ }
19
+ terminate(): void {
20
+ this.terminated = true;
21
+ }
22
+ emit(event: ShredWorkerEvent): void {
23
+ this.onmessage?.({ data: event } as MessageEvent);
24
+ }
25
+ }
26
+
27
+ function makeStats(overrides: Partial<ShredWorkerStats> = {}): ShredWorkerStats {
28
+ return {
29
+ shredsRx: 0,
30
+ sourceShreds: 0,
31
+ repairShreds: 0,
32
+ fecSetsCompleted: 0,
33
+ bytesReassembled: 0,
34
+ fecSetsRecovered: 0,
35
+ pendingFecSets: -1,
36
+ malformed: 0,
37
+ versionMismatch: 0,
38
+ currentSlot: "0",
39
+ slotSamples: [],
40
+ ...overrides,
41
+ };
42
+ }
43
+
44
+ describe("ShredWorkerClient", () => {
45
+ it("posts configure with the config", () => {
46
+ const worker = new MockWorker();
47
+ const client = new ShredWorkerClient(worker as unknown as Worker);
48
+ client.configure({ maxSlots: 5000, symbolSize: 512, wasmUrl: "/vendor/fec/mmt_wasm.js" });
49
+ expect(worker.posted[0].data).toEqual({
50
+ type: "configure",
51
+ config: { maxSlots: 5000, symbolSize: 512, wasmUrl: "/vendor/fec/mmt_wasm.js" },
52
+ });
53
+ });
54
+
55
+ it("transfers the readable stream on attachStream", () => {
56
+ const worker = new MockWorker();
57
+ const client = new ShredWorkerClient(worker as unknown as Worker);
58
+ const readable = new ReadableStream<Uint8Array>();
59
+ client.attachStream(readable);
60
+ const msg = worker.posted[0];
61
+ expect(msg.data.type).toBe("attachStream");
62
+ expect(msg.transfer).toEqual([readable]);
63
+ });
64
+
65
+ it("transfers the frame buffer on feedShred", () => {
66
+ const worker = new MockWorker();
67
+ const client = new ShredWorkerClient(worker as unknown as Worker);
68
+ const frame = new Uint8Array([1, 2, 3]).buffer;
69
+ client.feedShred(frame);
70
+ const msg = worker.posted[0];
71
+ expect(msg.data).toEqual({ type: "feedShred", frame });
72
+ expect(msg.transfer).toEqual([frame]);
73
+ });
74
+
75
+ it("bounds feedShred to maxInflight when the worker never drains", () => {
76
+ const worker = new MockWorker();
77
+ const client = new ShredWorkerClient(worker as unknown as Worker, { maxInflight: 4 });
78
+ for (let i = 0; i < 10; i++) client.feedShred(new Uint8Array([i]).buffer);
79
+ const fed = worker.posted.filter((p) => p.data.type === "feedShred");
80
+ expect(fed.length).toBe(4); // only maxInflight frames posted; the rest dropped
81
+ expect(client.droppedBackpressure).toBe(6);
82
+ expect(client.inflight).toBe(4);
83
+ });
84
+
85
+ it("does not transfer (detach) a frame dropped by the in-flight guard", () => {
86
+ const worker = new MockWorker();
87
+ const client = new ShredWorkerClient(worker as unknown as Worker, { maxInflight: 1 });
88
+ client.feedShred(new Uint8Array([1]).buffer); // posted (outstanding 0 -> 1)
89
+ const dropped = new Uint8Array([9, 9, 9]).buffer;
90
+ client.feedShred(dropped); // dropped (outstanding 1 >= 1)
91
+ // A dropped frame is never added to a transfer list, so the caller's buffer
92
+ // is untouched — no attempt to reuse a detached ArrayBuffer.
93
+ expect(dropped.byteLength).toBe(3);
94
+ expect(worker.posted.filter((p) => p.transfer.includes(dropped)).length).toBe(0);
95
+ });
96
+
97
+ it("resumes feeding after the worker reports drain progress via stats", () => {
98
+ const worker = new MockWorker();
99
+ const client = new ShredWorkerClient(worker as unknown as Worker, { maxInflight: 4 });
100
+ for (let i = 0; i < 6; i++) client.feedShred(new Uint8Array([i]).buffer);
101
+ expect(worker.posted.filter((p) => p.data.type === "feedShred").length).toBe(4);
102
+ expect(client.droppedBackpressure).toBe(2);
103
+
104
+ // Worker heartbeat: it consumed all 4 queued frames -> window reopens.
105
+ worker.emit({ type: "stats", stats: makeStats({ shredsRx: 4 }) });
106
+ expect(client.inflight).toBe(0);
107
+
108
+ for (let i = 0; i < 3; i++) client.feedShred(new Uint8Array([100 + i]).buffer);
109
+ expect(worker.posted.filter((p) => p.data.type === "feedShred").length).toBe(7);
110
+ expect(client.droppedBackpressure).toBe(2); // no NEW drops after the window reopened
111
+ });
112
+
113
+ it("counts malformed + versionMismatch as drained so the window can't inflate", () => {
114
+ const worker = new MockWorker();
115
+ const client = new ShredWorkerClient(worker as unknown as Worker, { maxInflight: 4 });
116
+ for (let i = 0; i < 4; i++) client.feedShred(new Uint8Array([i]).buffer);
117
+ // Worker drained all 4, but only 1 was valid; 2 malformed + 1 version skew.
118
+ worker.emit({
119
+ type: "stats",
120
+ stats: makeStats({ shredsRx: 1, malformed: 2, versionMismatch: 1 }),
121
+ });
122
+ expect(client.inflight).toBe(0);
123
+ });
124
+
125
+ it("maxInflight <= 0 disables the bound entirely", () => {
126
+ const worker = new MockWorker();
127
+ const client = new ShredWorkerClient(worker as unknown as Worker, { maxInflight: 0 });
128
+ for (let i = 0; i < 50; i++) client.feedShred(new Uint8Array([i]).buffer);
129
+ expect(worker.posted.filter((p) => p.data.type === "feedShred").length).toBe(50);
130
+ expect(client.droppedBackpressure).toBe(0);
131
+ });
132
+
133
+ it("posts recoverAll with the provided time", () => {
134
+ const worker = new MockWorker();
135
+ const client = new ShredWorkerClient(worker as unknown as Worker);
136
+ client.recoverAll(9999);
137
+ expect(worker.posted[0].data).toEqual({ type: "recoverAll", nowMs: 9999 });
138
+ });
139
+
140
+ it("dispatches recovered and stats events to callbacks", () => {
141
+ const worker = new MockWorker();
142
+ const client = new ShredWorkerClient(worker as unknown as Worker);
143
+ const recovered: Array<{ len: number; recovered: boolean }> = [];
144
+ const statsSeen: ShredWorkerStats[] = [];
145
+ client.onRecovered((data, meta) => recovered.push({ len: data.byteLength, recovered: meta.recovered }));
146
+ client.onStats((s) => statsSeen.push(s));
147
+
148
+ worker.emit({
149
+ type: "recovered",
150
+ data: new Uint8Array([1, 2, 3, 4]).buffer,
151
+ meta: { slot: 5n, fecSetIndex: 1, sendTsUs: 0n, recovered: true },
152
+ });
153
+ worker.emit({ type: "stats", stats: makeStats({ shredsRx: 7 }) });
154
+
155
+ expect(recovered).toEqual([{ len: 4, recovered: true }]);
156
+ expect(statsSeen[0].shredsRx).toBe(7);
157
+ });
158
+
159
+ it("routes error events with their code", () => {
160
+ const worker = new MockWorker();
161
+ const client = new ShredWorkerClient(worker as unknown as Worker);
162
+ const errs: Array<[string, string | undefined]> = [];
163
+ client.onError((m, c) => errs.push([m, c]));
164
+ worker.emit({ type: "error", message: "boom", code: "wasm-load-failed" });
165
+ expect(errs).toEqual([["boom", "wasm-load-failed"]]);
166
+ });
167
+
168
+ it("dispose posts dispose, terminates, and is idempotent + no-ops afterward", () => {
169
+ const worker = new MockWorker();
170
+ const client = new ShredWorkerClient(worker as unknown as Worker);
171
+ client.dispose();
172
+ expect(worker.posted[worker.posted.length - 1]?.data).toEqual({ type: "dispose" });
173
+ expect(worker.terminated).toBe(true);
174
+ const postedCount = worker.posted.length;
175
+ // Post-dispose calls are no-ops.
176
+ client.feedShred(new Uint8Array([1]).buffer);
177
+ client.attachStream(new ReadableStream<Uint8Array>());
178
+ client.recoverAll();
179
+ client.dispose();
180
+ expect(worker.posted.length).toBe(postedCount);
181
+ });
182
+ });
@@ -0,0 +1,222 @@
1
+ /**
2
+ * @blockcast/fec-worker — Shred Worker Main-Thread Client
3
+ *
4
+ * Thin wrapper over the shred (Reed-Solomon) worker, mirroring
5
+ * {@link FecWorkerClient}'s shape. Spawns one Worker, speaks the
6
+ * {@link ShredWorkerCommand}/{@link ShredWorkerEvent} protocol, and keeps the
7
+ * datagram read loop + wire parse + WASM FEC off the main thread.
8
+ *
9
+ * Typical wiring for a WebTransport datagram source (fully off-thread):
10
+ *
11
+ * const client = new ShredWorkerClient(new URL("./shred-fec-worker.js", import.meta.url));
12
+ * client.onRecovered((data, meta) => renderRecoveredSet(data, meta));
13
+ * client.configure({ maxSlots: 5000, symbolSize: 512, wasmUrl: "/vendor/fec/mmt_wasm.js" });
14
+ * client.attachStream(wt.datagrams.readable); // transferred; main does no read() per packet
15
+ *
16
+ * For sources that demux on the main thread (moqtail OBJECT_DATAGRAM callback,
17
+ * IWA multicast bridge), push each raw frame with {@link feedShred} instead.
18
+ *
19
+ * {@link feedShred} is a fire-and-forget `postMessage` with no native
20
+ * backpressure, so a consumer that decodes slower than the ingest rate (a
21
+ * backgrounded/throttled tab, WASM RS falling behind a ~4k shred/s feed) would
22
+ * grow the worker's inbound message queue without limit — the queue lives in the
23
+ * renderer process and has been observed to reach multiple GB, GC-thrashing the
24
+ * page into a wedge. The client bounds it with an in-flight window keyed off the
25
+ * worker's own periodic stats heartbeat (no per-frame ack round-trip); see
26
+ * {@link ShredWorkerClientOptions.maxInflight}.
27
+ */
28
+
29
+ import type {
30
+ ShredWorkerCommand,
31
+ ShredWorkerConfig,
32
+ ShredWorkerEvent,
33
+ ShredWorkerStats,
34
+ ShredWorkerErrorCode,
35
+ RecoveredMeta,
36
+ } from "./shred-worker-types.js";
37
+
38
+ /** Construction-time options for {@link ShredWorkerClient}. */
39
+ export interface ShredWorkerClientOptions {
40
+ /**
41
+ * Cap on frames posted via {@link ShredWorkerClient.feedShred} but not yet
42
+ * drained by the worker, before further frames are dropped ("skip-to-live" at
43
+ * the worker boundary). Bounds the worker's inbound message queue — the sole
44
+ * unbounded structure on the main-thread-demux (`feedShred`) path.
45
+ *
46
+ * The drain signal is the worker's periodic `stats` heartbeat: the monotonic
47
+ * total of frames it has taken off the queue (`shredsRx + malformed +
48
+ * versionMismatch`). No per-frame acknowledgement crosses the boundary, so the
49
+ * bound is heartbeat-granular — size it above one heartbeat's worth of ingest
50
+ * (default 250ms heartbeat × ~4k frames/s ≈ ~1k frames).
51
+ *
52
+ * Default {@link DEFAULT_MAX_INFLIGHT} (8192 frames ≈ ~10 MB at ~1.3 KB/frame).
53
+ * Set to `0` (or any value `<= 0`) to disable the bound entirely — only do so
54
+ * for a source with its own backpressure (e.g. a bounded {@link
55
+ * ShredWorkerClient.attachStream} producer). Note the bound assumes `feedShred`
56
+ * is the sole ingest path; mixing it with `attachStream` (whose frames also
57
+ * advance the heartbeat total) makes the window under-count and benignly
58
+ * disables the drop.
59
+ */
60
+ maxInflight?: number;
61
+ }
62
+
63
+ /** Default {@link ShredWorkerClientOptions.maxInflight} (~10 MB of queued frames). */
64
+ export const DEFAULT_MAX_INFLIGHT = 8192;
65
+
66
+ export class ShredWorkerClient {
67
+ readonly #worker: Worker;
68
+ #onRecovered: ((data: ArrayBuffer, meta: RecoveredMeta) => void) | null = null;
69
+ #onStats: ((stats: ShredWorkerStats) => void) | null = null;
70
+ #onError: ((message: string, code?: ShredWorkerErrorCode) => void) | null = null;
71
+ #disposed = false;
72
+
73
+ /** feedShred in-flight backpressure accounting (see ShredWorkerClientOptions.maxInflight). */
74
+ readonly #maxInflight: number;
75
+ #sent = 0; // frames posted via feedShred (monotonic)
76
+ #processed = 0; // worker-consumed total from the last stats heartbeat (monotonic)
77
+ #droppedBackpressure = 0; // frames dropped by the in-flight guard (monotonic)
78
+
79
+ /**
80
+ * @param workerOrUrl - A URL (a Worker is created internally, `type: module`)
81
+ * or a pre-created Worker (for testing via constructor injection).
82
+ * @param opts - Optional {@link ShredWorkerClientOptions}.
83
+ */
84
+ constructor(workerOrUrl: Worker | URL, opts: ShredWorkerClientOptions = {}) {
85
+ this.#maxInflight = opts.maxInflight ?? DEFAULT_MAX_INFLIGHT;
86
+ this.#worker =
87
+ workerOrUrl instanceof URL
88
+ ? new Worker(workerOrUrl, { type: "module" })
89
+ : workerOrUrl;
90
+ this.#worker.onmessage = (e: MessageEvent<ShredWorkerEvent>) =>
91
+ this.#handleEvent(e.data);
92
+ this.#worker.onerror = (e) =>
93
+ this.#onError?.(formatWorkerError(e), "worker-runtime");
94
+ this.#worker.onmessageerror = () =>
95
+ this.#onError?.("failed to deserialize worker message", "worker-message");
96
+ }
97
+
98
+ /** Configure the decoder (WASM load + ShredFecBuffer construction happen in the worker). */
99
+ configure(config: ShredWorkerConfig): void {
100
+ if (this.#disposed) throw new Error("[ShredWorkerClient] disposed");
101
+ this.#post({ type: "configure", config });
102
+ }
103
+
104
+ /**
105
+ * Hand the worker a stream of raw shred frames to batch-drain. The stream is
106
+ * transferred — the caller must not use it afterward. One active stream at a
107
+ * time; attaching a new one cancels the previous.
108
+ */
109
+ attachStream(readable: ReadableStream<Uint8Array>): void {
110
+ if (this.#disposed) return;
111
+ this.#post({ type: "attachStream", readable }, [readable]);
112
+ }
113
+
114
+ /**
115
+ * Feed one already-demuxed raw shred frame. `frame` is transferred zero-copy;
116
+ * the caller must not access it afterward.
117
+ *
118
+ * Dropped (frame NOT transferred, so the caller's buffer stays intact) when the
119
+ * in-flight window is full — i.e. the worker is more than `maxInflight` frames
120
+ * behind. This is the memory bound: a slow/wedged consumer costs bounded
121
+ * skip-to-live drops (see {@link droppedBackpressure}), never an unbounded
122
+ * renderer-heap blow-up.
123
+ */
124
+ feedShred(frame: ArrayBuffer): void {
125
+ if (this.#disposed) return;
126
+ if (this.#maxInflight > 0 && this.#sent - this.#processed >= this.#maxInflight) {
127
+ this.#droppedBackpressure++;
128
+ return;
129
+ }
130
+ this.#sent++;
131
+ this.#post({ type: "feedShred", frame }, [frame]);
132
+ }
133
+
134
+ /** Explicitly drive loss recovery (in addition to the worker's internal timer). */
135
+ recoverAll(nowMs: number = Date.now()): void {
136
+ if (this.#disposed) return;
137
+ this.#post({ type: "recoverAll", nowMs });
138
+ }
139
+
140
+ /** Register callback for recovered/completed FEC sets. `data` is a transferred buffer. */
141
+ onRecovered(cb: (data: ArrayBuffer, meta: RecoveredMeta) => void): void {
142
+ this.#onRecovered = cb;
143
+ }
144
+
145
+ /** Register callback for the coalesced stats heartbeat. */
146
+ onStats(cb: (stats: ShredWorkerStats) => void): void {
147
+ this.#onStats = cb;
148
+ }
149
+
150
+ /** Register callback for worker errors. */
151
+ onError(cb: (message: string, code?: ShredWorkerErrorCode) => void): void {
152
+ this.#onError = cb;
153
+ }
154
+
155
+ /**
156
+ * Frames dropped by the in-flight backpressure guard since construction
157
+ * (monotonic). Surface this in the UI so sustained loss is visible rather than
158
+ * silent — a rising counter means the consumer can't keep pace with ingest.
159
+ */
160
+ get droppedBackpressure(): number {
161
+ return this.#droppedBackpressure;
162
+ }
163
+
164
+ /**
165
+ * Frames posted via {@link feedShred} but not yet reported drained by the
166
+ * worker's last stats heartbeat (best-effort, heartbeat-granular; never < 0).
167
+ */
168
+ get inflight(): number {
169
+ return Math.max(0, this.#sent - this.#processed);
170
+ }
171
+
172
+ /** Dispose the worker. Idempotent. */
173
+ dispose(): void {
174
+ if (this.#disposed) return;
175
+ this.#disposed = true;
176
+ try {
177
+ this.#worker.postMessage({ type: "dispose" } satisfies ShredWorkerCommand);
178
+ } catch {
179
+ /* worker may already be gone */
180
+ }
181
+ this.#worker.terminate();
182
+ }
183
+
184
+ #post(cmd: ShredWorkerCommand, transfer: Transferable[] = []): void {
185
+ this.#worker.postMessage(cmd, transfer);
186
+ }
187
+
188
+ #handleEvent(event: ShredWorkerEvent): void {
189
+ switch (event.type) {
190
+ case "recovered":
191
+ this.#onRecovered?.(event.data, event.meta);
192
+ break;
193
+ case "stats":
194
+ // The worker's monotonic consumed-frame total is the drain signal for
195
+ // the feedShred in-flight guard. shredsRx counts only valid frames;
196
+ // malformed + versionMismatch are consumed off the queue too, so the
197
+ // sum is the true "frames taken off the queue" and keeps the window
198
+ // from inflating (and false-dropping) on a malformed/skewed source.
199
+ this.#processed =
200
+ event.stats.shredsRx + event.stats.malformed + event.stats.versionMismatch;
201
+ this.#onStats?.(event.stats);
202
+ break;
203
+ case "error":
204
+ if (event.code) this.#onError?.(event.message, event.code);
205
+ else this.#onError?.(event.message);
206
+ break;
207
+ }
208
+ }
209
+ }
210
+
211
+ function formatWorkerError(event: ErrorEvent): string {
212
+ const parts = ["Worker error"];
213
+ if (event.message) parts.push(event.message);
214
+ if (event.filename) {
215
+ const loc = [event.filename];
216
+ if (event.lineno) loc.push(String(event.lineno));
217
+ if (event.colno) loc.push(String(event.colno));
218
+ parts.push(`at ${loc.join(":")}`);
219
+ }
220
+ if (event.error instanceof Error && event.error.stack) parts.push(event.error.stack);
221
+ return parts.join(": ");
222
+ }
@@ -0,0 +1,194 @@
1
+ /**
2
+ * @blockcast/fec-worker — Shred (Reed-Solomon) Worker Protocol
3
+ *
4
+ * A sibling codec to the RaptorQ/MMTP worker (`fec-worker.ts`). Solana-style
5
+ * shred streams carry a different FEC payload identity than MMTP source/repair
6
+ * symbols — a shred is `(slot, fecSetIndex, localIndex, numData, numCoding,
7
+ * isCoding, lastInSet)`, not a flat `ssId` — so they need their own command
8
+ * shape. The transport harness (Worker spawn, onmessage plumbing, zero-copy
9
+ * `toOwnedArrayBuffer` transfers) is shared; the decoder WASM is not.
10
+ *
11
+ * The worker owns the batch-drain: the consumer transfers a WebTransport
12
+ * `datagrams.readable` (or any `ReadableStream<Uint8Array>` of raw shred
13
+ * frames) into the Worker via {@link ShredWorkerCommand} `attachStream`, and
14
+ * the read loop + wire parse + `ShredFecBuffer.add_shred` all run off the main
15
+ * thread. Only recovered FEC sets (and coalesced stats) cross back. A
16
+ * `feedShred` command covers sources that have already demuxed the shred bytes
17
+ * on the main thread (e.g. a moqtail OBJECT_DATAGRAM callback, or the IWA
18
+ * multicast bridge).
19
+ *
20
+ * Types here are importable from both main thread and Worker contexts.
21
+ */
22
+
23
+ // --- Wire format ------------------------------------------------------------
24
+
25
+ /**
26
+ * Wire header layout the worker parses. Mirrors the deployed shred player's
27
+ * `onShredBytes` exactly (libmmt shred-forwarder `emit_frame`) so the off-thread
28
+ * path is bit-identical to the proven on-thread path.
29
+ *
30
+ * v3 (28-byte header):
31
+ * [0] version u8 (== 3; non-3 frames are dropped, not misframed)
32
+ * [1-8] slot u64 LE
33
+ * [9-12] fecSetIndex u32 LE
34
+ * [13-16] localIndex u32 LE (data: 0..numData; coding: numData+position)
35
+ * [17] flags u8 (bit0 = DATA_COMPLETE, bit1 = IS_CODING_SHRED)
36
+ * [18] numData u8 (coding shreds only; 0 on data — WASM infers)
37
+ * [19] numCoding u8 (coding shreds only; 0 on data)
38
+ * [20-27] sendTsUs u64 LE (µs since epoch, 0 when unstamped)
39
+ * [28+] erasureShard (equal length across the FEC set)
40
+ *
41
+ * DATA_COMPLETE (last data shred in the set) is passed to `add_shred` as its
42
+ * `lastInSet` argument — the WASM uses it to infer numData for a set that
43
+ * received only data shreds (no coding).
44
+ */
45
+ export const SHRED_WIRE_VERSION = 3;
46
+ export const SHRED_HEADER_LEN = 28;
47
+ export const SHRED_FLAG_DATA_COMPLETE = 0x01;
48
+ export const SHRED_FLAG_IS_CODING = 0x02;
49
+
50
+ // --- Config -----------------------------------------------------------------
51
+
52
+ /** Per-decoder configuration for the shred FEC worker. */
53
+ export interface ShredWorkerConfig {
54
+ /** Ring capacity in slots (matches `new ShredFecBuffer(maxSlots, symbolSize)`). */
55
+ maxSlots: number;
56
+ /** Symbol size in bytes (Solana shred payloads are fixed-size within a set). */
57
+ symbolSize: number;
58
+ /**
59
+ * Module URL for the shred WASM (wasm-bindgen ESM exporting `default` init +
60
+ * `ShredFecBuffer`). The worker `import()`s it and calls `await default()`.
61
+ * Ignored when `self.__SHRED_WASM_BINDINGS__` is pre-injected (tests / custom
62
+ * loaders), mirroring the RaptorQ worker's `__MMT_WASM_BINDINGS__` contract.
63
+ */
64
+ wasmUrl?: string;
65
+ /**
66
+ * How often (ms) the worker coalesces and posts a `stats` event. The read
67
+ * loop never posts per-shred; only recovered frames and this heartbeat cross
68
+ * the boundary. Default 250ms.
69
+ */
70
+ statsIntervalMs?: number;
71
+ /**
72
+ * How often (ms) the worker runs `try_recover_all(now)` to drain timed-out
73
+ * FEC sets that `add_shred` didn't complete (loss scenarios). Default 50ms,
74
+ * matching the player's on-thread recovery timer. Set to 0 to disable (the
75
+ * consumer may instead drive it explicitly with `recoverAll`).
76
+ */
77
+ recoverIntervalMs?: number;
78
+ }
79
+
80
+ // --- Stats ------------------------------------------------------------------
81
+
82
+ /**
83
+ * First-shred-per-slot sample, collected by the worker and drained into each
84
+ * coalesced `stats` heartbeat. Lets the main thread run app-specific,
85
+ * slot-derived stats (e.g. the turbine-leg latency estimator, per-slot latency
86
+ * percentiles) WITHOUT re-parsing the wire header — the worker stays the single
87
+ * source of truth for parsing, and these cross the boundary aggregated (a
88
+ * handful per heartbeat), never per-packet.
89
+ */
90
+ export interface SlotSample {
91
+ /** Slot number (first shred of this slot the worker saw). */
92
+ slot: bigint;
93
+ /** Publisher send timestamp (µs since epoch) of that first shred, 0 when unstamped. */
94
+ sendTsUs: bigint;
95
+ /** Frame byte length of that first shred. */
96
+ bytes: number;
97
+ }
98
+
99
+ /** Aggregate shred-decode counters, emitted on the coalesced `stats` heartbeat. */
100
+ export interface ShredWorkerStats {
101
+ /** Total shred frames parsed (data + coding). */
102
+ shredsRx: number;
103
+ /** Data (source) shreds observed. */
104
+ sourceShreds: number;
105
+ /** Coding (repair) shreds observed. */
106
+ repairShreds: number;
107
+ /** FEC sets completed (direct completion + FEC recovery). */
108
+ fecSetsCompleted: number;
109
+ /** Bytes handed back as recovered/completed sets. */
110
+ bytesReassembled: number;
111
+ /** Sets recovered specifically via `try_recover_all` (loss recovery). */
112
+ fecSetsRecovered: number;
113
+ /** Current `ShredFecBuffer.pending_fec_sets()`, or -1 when unavailable. */
114
+ pendingFecSets: number;
115
+ /** Frames dropped for being shorter than the v3 header. */
116
+ malformed: number;
117
+ /** Frames dropped for carrying a non-v3 version byte (forwarder/worker skew). */
118
+ versionMismatch: number;
119
+ /** Highest slot number observed (as a string — bigint isn't JSON-safe for UI). */
120
+ currentSlot: string;
121
+ /** First-shred-per-slot samples parsed since the previous heartbeat (drained each emit). */
122
+ slotSamples: SlotSample[];
123
+ }
124
+
125
+ /** Stable identities for worker failures whose message text carries live values. */
126
+ export type ShredWorkerErrorCode =
127
+ | "wasm-load-failed"
128
+ | "wasm-bindings-missing"
129
+ | "not-configured"
130
+ | "stream-read-failed"
131
+ | "unknown-command"
132
+ | "worker-runtime"
133
+ | "worker-message";
134
+
135
+ // --- Recovered frame metadata ----------------------------------------------
136
+
137
+ /** Metadata attached to each recovered set the worker emits. */
138
+ export interface RecoveredMeta {
139
+ /**
140
+ * Slot the recovered set belongs to. Attributed on both the direct-completion
141
+ * path and the identity-carrying recovery drain
142
+ * (`try_recover_all_with_identity`); only 0n on the legacy `ShredReassembler`
143
+ * fallback, which has no identity variant. Do NOT branch on `slot === 0n` as
144
+ * "unattributed" — that would drop the majority of recovered sets.
145
+ */
146
+ slot: bigint;
147
+ /** FEC set index within the slot (0 only on the legacy identity-less fallback). */
148
+ fecSetIndex: number;
149
+ /**
150
+ * Publisher send timestamp (µs since epoch), 0 when unstamped/unknown. On
151
+ * direct completion this is the completing shred's stamp; on the recovery
152
+ * drain it is the set's first-seen shred stamp (shreds of a set are sent µs
153
+ * apart, so the two are equivalent latency baselines).
154
+ */
155
+ sendTsUs: bigint;
156
+ /** True when produced by loss recovery (`try_recover_all`) rather than direct completion. */
157
+ recovered: boolean;
158
+ }
159
+
160
+ // --- Commands (main -> worker) ----------------------------------------------
161
+
162
+ /** Discriminated union of all commands sent from the main thread to the shred worker. */
163
+ export type ShredWorkerCommand =
164
+ | { type: "configure"; config: ShredWorkerConfig }
165
+ | {
166
+ /**
167
+ * Hand the worker a stream of raw shred frames to batch-drain. Each
168
+ * chunk is one wire frame (one WebTransport datagram). The stream is
169
+ * transferred (add to the postMessage transfer list) — the main thread
170
+ * must not touch it afterward.
171
+ */
172
+ type: "attachStream";
173
+ readable: ReadableStream<Uint8Array>;
174
+ }
175
+ | {
176
+ /**
177
+ * Feed one already-demuxed raw shred frame. `frame` is transferred
178
+ * zero-copy — the caller must not access it after posting. Use this for
179
+ * sources whose demux stays on the main thread (moqtail OBJECT_DATAGRAM
180
+ * callback, IWA multicast bridge).
181
+ */
182
+ type: "feedShred";
183
+ frame: ArrayBuffer;
184
+ }
185
+ | { type: "recoverAll"; nowMs: number }
186
+ | { type: "dispose" };
187
+
188
+ // --- Events (worker -> main) ------------------------------------------------
189
+
190
+ /** Discriminated union of all events emitted from the shred worker to the main thread. */
191
+ export type ShredWorkerEvent =
192
+ | { type: "recovered"; data: ArrayBuffer; meta: RecoveredMeta }
193
+ | { type: "stats"; stats: ShredWorkerStats }
194
+ | { type: "error"; message: string; code?: ShredWorkerErrorCode };
@@ -0,0 +1,24 @@
1
+ import { describe, expect, it } from "vitest";
2
+ import { toOwnedArrayBuffer } from "./transfer.js";
3
+
4
+ describe("toOwnedArrayBuffer", () => {
5
+ it("copies exactly the bytes in an offset Uint8Array view", () => {
6
+ const backing = new Uint8Array([0, 1, 2, 3, 4, 5]);
7
+ const view = backing.subarray(2, 5);
8
+
9
+ const owned = toOwnedArrayBuffer(view);
10
+
11
+ expect([...new Uint8Array(owned)]).toEqual([2, 3, 4]);
12
+ expect(owned.byteLength).toBe(3);
13
+ expect(owned).not.toBe(backing.buffer);
14
+ });
15
+
16
+ it("does not alias the source buffer after copying", () => {
17
+ const source = new Uint8Array([9, 8, 7]);
18
+ const owned = toOwnedArrayBuffer(source);
19
+
20
+ source[1] = 1;
21
+
22
+ expect([...new Uint8Array(owned)]).toEqual([9, 8, 7]);
23
+ });
24
+ });
@@ -0,0 +1,12 @@
1
+ /**
2
+ * Return a fresh ArrayBuffer containing exactly the bytes in `view`.
3
+ *
4
+ * Worker outputs must not transfer buffers that might be owned by a WASM
5
+ * binding or by a larger shared packet. A fresh JS-owned buffer gives
6
+ * postMessage() exclusive ownership without detaching unrelated memory.
7
+ */
8
+ export function toOwnedArrayBuffer(view: Uint8Array): ArrayBuffer {
9
+ const copy = new Uint8Array(view.byteLength);
10
+ copy.set(view);
11
+ return copy.buffer;
12
+ }