@blockcast/fec-worker 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +213 -0
- package/dist/.build-stamp +0 -0
- package/dist/fec-worker-client.d.ts +80 -0
- package/dist/fec-worker-client.d.ts.map +1 -0
- package/dist/fec-worker-client.js +270 -0
- package/dist/fec-worker-client.js.map +1 -0
- package/dist/fec-worker-types.d.ts +252 -0
- package/dist/fec-worker-types.d.ts.map +1 -0
- package/dist/fec-worker-types.js +11 -0
- package/dist/fec-worker-types.js.map +1 -0
- package/dist/fec-worker.d.ts +14 -0
- package/dist/fec-worker.d.ts.map +1 -0
- package/dist/fec-worker.js +565 -0
- package/dist/fec-worker.js.map +7 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -0
- package/dist/shred-fec-worker.d.ts +20 -0
- package/dist/shred-fec-worker.d.ts.map +1 -0
- package/dist/shred-fec-worker.js +349 -0
- package/dist/shred-fec-worker.js.map +7 -0
- package/dist/shred-worker-client.d.ts +104 -0
- package/dist/shred-worker-client.d.ts.map +1 -0
- package/dist/shred-worker-client.js +181 -0
- package/dist/shred-worker-client.js.map +1 -0
- package/dist/shred-worker-types.d.ts +179 -0
- package/dist/shred-worker-types.d.ts.map +1 -0
- package/dist/shred-worker-types.js +47 -0
- package/dist/shred-worker-types.js.map +1 -0
- package/dist/transfer.d.ts +9 -0
- package/dist/transfer.d.ts.map +1 -0
- package/dist/transfer.js +13 -0
- package/dist/transfer.js.map +1 -0
- package/package.json +54 -0
- package/src/__tests__/fec-worker-alta.test.ts +231 -0
- package/src/__tests__/fec-worker-cleanup-integration.test.ts +250 -0
- package/src/__tests__/fec-worker-pending-queue.test.ts +315 -0
- package/src/__tests__/fec-worker-repair-size.test.ts +1029 -0
- package/src/__tests__/fec-worker-wasm-contract.test.ts +84 -0
- package/src/__tests__/shred-fec-worker.test.ts +448 -0
- package/src/fec-worker-client.test.ts +530 -0
- package/src/fec-worker-client.ts +309 -0
- package/src/fec-worker-types.test.ts +400 -0
- package/src/fec-worker-types.ts +268 -0
- package/src/fec-worker.ts +1048 -0
- package/src/index.ts +32 -0
- package/src/shred-fec-worker.ts +558 -0
- package/src/shred-worker-client.test.ts +182 -0
- package/src/shred-worker-client.ts +222 -0
- package/src/shred-worker-types.ts +194 -0
- package/src/transfer.test.ts +24 -0
- package/src/transfer.ts +12 -0
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
import { describe, it, expect } from "vitest";
|
|
2
|
+
import type {
|
|
3
|
+
ShredWorkerCommand,
|
|
4
|
+
ShredWorkerEvent,
|
|
5
|
+
ShredWorkerStats,
|
|
6
|
+
} from "./shred-worker-types.js";
|
|
7
|
+
import { ShredWorkerClient } from "./shred-worker-client.js";
|
|
8
|
+
|
|
9
|
+
class MockWorker {
|
|
10
|
+
onmessage: ((e: MessageEvent) => void) | null = null;
|
|
11
|
+
onerror: ((e: ErrorEvent) => void) | null = null;
|
|
12
|
+
onmessageerror: ((e: MessageEvent) => void) | null = null;
|
|
13
|
+
posted: Array<{ data: ShredWorkerCommand; transfer: Transferable[] }> = [];
|
|
14
|
+
terminated = false;
|
|
15
|
+
|
|
16
|
+
postMessage(data: ShredWorkerCommand, transfer?: Transferable[]): void {
|
|
17
|
+
this.posted.push({ data, transfer: transfer ?? [] });
|
|
18
|
+
}
|
|
19
|
+
terminate(): void {
|
|
20
|
+
this.terminated = true;
|
|
21
|
+
}
|
|
22
|
+
emit(event: ShredWorkerEvent): void {
|
|
23
|
+
this.onmessage?.({ data: event } as MessageEvent);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function makeStats(overrides: Partial<ShredWorkerStats> = {}): ShredWorkerStats {
|
|
28
|
+
return {
|
|
29
|
+
shredsRx: 0,
|
|
30
|
+
sourceShreds: 0,
|
|
31
|
+
repairShreds: 0,
|
|
32
|
+
fecSetsCompleted: 0,
|
|
33
|
+
bytesReassembled: 0,
|
|
34
|
+
fecSetsRecovered: 0,
|
|
35
|
+
pendingFecSets: -1,
|
|
36
|
+
malformed: 0,
|
|
37
|
+
versionMismatch: 0,
|
|
38
|
+
currentSlot: "0",
|
|
39
|
+
slotSamples: [],
|
|
40
|
+
...overrides,
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
describe("ShredWorkerClient", () => {
|
|
45
|
+
it("posts configure with the config", () => {
|
|
46
|
+
const worker = new MockWorker();
|
|
47
|
+
const client = new ShredWorkerClient(worker as unknown as Worker);
|
|
48
|
+
client.configure({ maxSlots: 5000, symbolSize: 512, wasmUrl: "/vendor/fec/mmt_wasm.js" });
|
|
49
|
+
expect(worker.posted[0].data).toEqual({
|
|
50
|
+
type: "configure",
|
|
51
|
+
config: { maxSlots: 5000, symbolSize: 512, wasmUrl: "/vendor/fec/mmt_wasm.js" },
|
|
52
|
+
});
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
it("transfers the readable stream on attachStream", () => {
|
|
56
|
+
const worker = new MockWorker();
|
|
57
|
+
const client = new ShredWorkerClient(worker as unknown as Worker);
|
|
58
|
+
const readable = new ReadableStream<Uint8Array>();
|
|
59
|
+
client.attachStream(readable);
|
|
60
|
+
const msg = worker.posted[0];
|
|
61
|
+
expect(msg.data.type).toBe("attachStream");
|
|
62
|
+
expect(msg.transfer).toEqual([readable]);
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
it("transfers the frame buffer on feedShred", () => {
|
|
66
|
+
const worker = new MockWorker();
|
|
67
|
+
const client = new ShredWorkerClient(worker as unknown as Worker);
|
|
68
|
+
const frame = new Uint8Array([1, 2, 3]).buffer;
|
|
69
|
+
client.feedShred(frame);
|
|
70
|
+
const msg = worker.posted[0];
|
|
71
|
+
expect(msg.data).toEqual({ type: "feedShred", frame });
|
|
72
|
+
expect(msg.transfer).toEqual([frame]);
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
it("bounds feedShred to maxInflight when the worker never drains", () => {
|
|
76
|
+
const worker = new MockWorker();
|
|
77
|
+
const client = new ShredWorkerClient(worker as unknown as Worker, { maxInflight: 4 });
|
|
78
|
+
for (let i = 0; i < 10; i++) client.feedShred(new Uint8Array([i]).buffer);
|
|
79
|
+
const fed = worker.posted.filter((p) => p.data.type === "feedShred");
|
|
80
|
+
expect(fed.length).toBe(4); // only maxInflight frames posted; the rest dropped
|
|
81
|
+
expect(client.droppedBackpressure).toBe(6);
|
|
82
|
+
expect(client.inflight).toBe(4);
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
it("does not transfer (detach) a frame dropped by the in-flight guard", () => {
|
|
86
|
+
const worker = new MockWorker();
|
|
87
|
+
const client = new ShredWorkerClient(worker as unknown as Worker, { maxInflight: 1 });
|
|
88
|
+
client.feedShred(new Uint8Array([1]).buffer); // posted (outstanding 0 -> 1)
|
|
89
|
+
const dropped = new Uint8Array([9, 9, 9]).buffer;
|
|
90
|
+
client.feedShred(dropped); // dropped (outstanding 1 >= 1)
|
|
91
|
+
// A dropped frame is never added to a transfer list, so the caller's buffer
|
|
92
|
+
// is untouched — no attempt to reuse a detached ArrayBuffer.
|
|
93
|
+
expect(dropped.byteLength).toBe(3);
|
|
94
|
+
expect(worker.posted.filter((p) => p.transfer.includes(dropped)).length).toBe(0);
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
it("resumes feeding after the worker reports drain progress via stats", () => {
|
|
98
|
+
const worker = new MockWorker();
|
|
99
|
+
const client = new ShredWorkerClient(worker as unknown as Worker, { maxInflight: 4 });
|
|
100
|
+
for (let i = 0; i < 6; i++) client.feedShred(new Uint8Array([i]).buffer);
|
|
101
|
+
expect(worker.posted.filter((p) => p.data.type === "feedShred").length).toBe(4);
|
|
102
|
+
expect(client.droppedBackpressure).toBe(2);
|
|
103
|
+
|
|
104
|
+
// Worker heartbeat: it consumed all 4 queued frames -> window reopens.
|
|
105
|
+
worker.emit({ type: "stats", stats: makeStats({ shredsRx: 4 }) });
|
|
106
|
+
expect(client.inflight).toBe(0);
|
|
107
|
+
|
|
108
|
+
for (let i = 0; i < 3; i++) client.feedShred(new Uint8Array([100 + i]).buffer);
|
|
109
|
+
expect(worker.posted.filter((p) => p.data.type === "feedShred").length).toBe(7);
|
|
110
|
+
expect(client.droppedBackpressure).toBe(2); // no NEW drops after the window reopened
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
it("counts malformed + versionMismatch as drained so the window can't inflate", () => {
|
|
114
|
+
const worker = new MockWorker();
|
|
115
|
+
const client = new ShredWorkerClient(worker as unknown as Worker, { maxInflight: 4 });
|
|
116
|
+
for (let i = 0; i < 4; i++) client.feedShred(new Uint8Array([i]).buffer);
|
|
117
|
+
// Worker drained all 4, but only 1 was valid; 2 malformed + 1 version skew.
|
|
118
|
+
worker.emit({
|
|
119
|
+
type: "stats",
|
|
120
|
+
stats: makeStats({ shredsRx: 1, malformed: 2, versionMismatch: 1 }),
|
|
121
|
+
});
|
|
122
|
+
expect(client.inflight).toBe(0);
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
it("maxInflight <= 0 disables the bound entirely", () => {
|
|
126
|
+
const worker = new MockWorker();
|
|
127
|
+
const client = new ShredWorkerClient(worker as unknown as Worker, { maxInflight: 0 });
|
|
128
|
+
for (let i = 0; i < 50; i++) client.feedShred(new Uint8Array([i]).buffer);
|
|
129
|
+
expect(worker.posted.filter((p) => p.data.type === "feedShred").length).toBe(50);
|
|
130
|
+
expect(client.droppedBackpressure).toBe(0);
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
it("posts recoverAll with the provided time", () => {
|
|
134
|
+
const worker = new MockWorker();
|
|
135
|
+
const client = new ShredWorkerClient(worker as unknown as Worker);
|
|
136
|
+
client.recoverAll(9999);
|
|
137
|
+
expect(worker.posted[0].data).toEqual({ type: "recoverAll", nowMs: 9999 });
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
it("dispatches recovered and stats events to callbacks", () => {
|
|
141
|
+
const worker = new MockWorker();
|
|
142
|
+
const client = new ShredWorkerClient(worker as unknown as Worker);
|
|
143
|
+
const recovered: Array<{ len: number; recovered: boolean }> = [];
|
|
144
|
+
const statsSeen: ShredWorkerStats[] = [];
|
|
145
|
+
client.onRecovered((data, meta) => recovered.push({ len: data.byteLength, recovered: meta.recovered }));
|
|
146
|
+
client.onStats((s) => statsSeen.push(s));
|
|
147
|
+
|
|
148
|
+
worker.emit({
|
|
149
|
+
type: "recovered",
|
|
150
|
+
data: new Uint8Array([1, 2, 3, 4]).buffer,
|
|
151
|
+
meta: { slot: 5n, fecSetIndex: 1, sendTsUs: 0n, recovered: true },
|
|
152
|
+
});
|
|
153
|
+
worker.emit({ type: "stats", stats: makeStats({ shredsRx: 7 }) });
|
|
154
|
+
|
|
155
|
+
expect(recovered).toEqual([{ len: 4, recovered: true }]);
|
|
156
|
+
expect(statsSeen[0].shredsRx).toBe(7);
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
it("routes error events with their code", () => {
|
|
160
|
+
const worker = new MockWorker();
|
|
161
|
+
const client = new ShredWorkerClient(worker as unknown as Worker);
|
|
162
|
+
const errs: Array<[string, string | undefined]> = [];
|
|
163
|
+
client.onError((m, c) => errs.push([m, c]));
|
|
164
|
+
worker.emit({ type: "error", message: "boom", code: "wasm-load-failed" });
|
|
165
|
+
expect(errs).toEqual([["boom", "wasm-load-failed"]]);
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
it("dispose posts dispose, terminates, and is idempotent + no-ops afterward", () => {
|
|
169
|
+
const worker = new MockWorker();
|
|
170
|
+
const client = new ShredWorkerClient(worker as unknown as Worker);
|
|
171
|
+
client.dispose();
|
|
172
|
+
expect(worker.posted[worker.posted.length - 1]?.data).toEqual({ type: "dispose" });
|
|
173
|
+
expect(worker.terminated).toBe(true);
|
|
174
|
+
const postedCount = worker.posted.length;
|
|
175
|
+
// Post-dispose calls are no-ops.
|
|
176
|
+
client.feedShred(new Uint8Array([1]).buffer);
|
|
177
|
+
client.attachStream(new ReadableStream<Uint8Array>());
|
|
178
|
+
client.recoverAll();
|
|
179
|
+
client.dispose();
|
|
180
|
+
expect(worker.posted.length).toBe(postedCount);
|
|
181
|
+
});
|
|
182
|
+
});
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @blockcast/fec-worker — Shred Worker Main-Thread Client
|
|
3
|
+
*
|
|
4
|
+
* Thin wrapper over the shred (Reed-Solomon) worker, mirroring
|
|
5
|
+
* {@link FecWorkerClient}'s shape. Spawns one Worker, speaks the
|
|
6
|
+
* {@link ShredWorkerCommand}/{@link ShredWorkerEvent} protocol, and keeps the
|
|
7
|
+
* datagram read loop + wire parse + WASM FEC off the main thread.
|
|
8
|
+
*
|
|
9
|
+
* Typical wiring for a WebTransport datagram source (fully off-thread):
|
|
10
|
+
*
|
|
11
|
+
* const client = new ShredWorkerClient(new URL("./shred-fec-worker.js", import.meta.url));
|
|
12
|
+
* client.onRecovered((data, meta) => renderRecoveredSet(data, meta));
|
|
13
|
+
* client.configure({ maxSlots: 5000, symbolSize: 512, wasmUrl: "/vendor/fec/mmt_wasm.js" });
|
|
14
|
+
* client.attachStream(wt.datagrams.readable); // transferred; main does no read() per packet
|
|
15
|
+
*
|
|
16
|
+
* For sources that demux on the main thread (moqtail OBJECT_DATAGRAM callback,
|
|
17
|
+
* IWA multicast bridge), push each raw frame with {@link feedShred} instead.
|
|
18
|
+
*
|
|
19
|
+
* {@link feedShred} is a fire-and-forget `postMessage` with no native
|
|
20
|
+
* backpressure, so a consumer that decodes slower than the ingest rate (a
|
|
21
|
+
* backgrounded/throttled tab, WASM RS falling behind a ~4k shred/s feed) would
|
|
22
|
+
* grow the worker's inbound message queue without limit — the queue lives in the
|
|
23
|
+
* renderer process and has been observed to reach multiple GB, GC-thrashing the
|
|
24
|
+
* page into a wedge. The client bounds it with an in-flight window keyed off the
|
|
25
|
+
* worker's own periodic stats heartbeat (no per-frame ack round-trip); see
|
|
26
|
+
* {@link ShredWorkerClientOptions.maxInflight}.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import type {
|
|
30
|
+
ShredWorkerCommand,
|
|
31
|
+
ShredWorkerConfig,
|
|
32
|
+
ShredWorkerEvent,
|
|
33
|
+
ShredWorkerStats,
|
|
34
|
+
ShredWorkerErrorCode,
|
|
35
|
+
RecoveredMeta,
|
|
36
|
+
} from "./shred-worker-types.js";
|
|
37
|
+
|
|
38
|
+
/** Construction-time options for {@link ShredWorkerClient}. */
|
|
39
|
+
export interface ShredWorkerClientOptions {
|
|
40
|
+
/**
|
|
41
|
+
* Cap on frames posted via {@link ShredWorkerClient.feedShred} but not yet
|
|
42
|
+
* drained by the worker, before further frames are dropped ("skip-to-live" at
|
|
43
|
+
* the worker boundary). Bounds the worker's inbound message queue — the sole
|
|
44
|
+
* unbounded structure on the main-thread-demux (`feedShred`) path.
|
|
45
|
+
*
|
|
46
|
+
* The drain signal is the worker's periodic `stats` heartbeat: the monotonic
|
|
47
|
+
* total of frames it has taken off the queue (`shredsRx + malformed +
|
|
48
|
+
* versionMismatch`). No per-frame acknowledgement crosses the boundary, so the
|
|
49
|
+
* bound is heartbeat-granular — size it above one heartbeat's worth of ingest
|
|
50
|
+
* (default 250ms heartbeat × ~4k frames/s ≈ ~1k frames).
|
|
51
|
+
*
|
|
52
|
+
* Default {@link DEFAULT_MAX_INFLIGHT} (8192 frames ≈ ~10 MB at ~1.3 KB/frame).
|
|
53
|
+
* Set to `0` (or any value `<= 0`) to disable the bound entirely — only do so
|
|
54
|
+
* for a source with its own backpressure (e.g. a bounded {@link
|
|
55
|
+
* ShredWorkerClient.attachStream} producer). Note the bound assumes `feedShred`
|
|
56
|
+
* is the sole ingest path; mixing it with `attachStream` (whose frames also
|
|
57
|
+
* advance the heartbeat total) makes the window under-count and benignly
|
|
58
|
+
* disables the drop.
|
|
59
|
+
*/
|
|
60
|
+
maxInflight?: number;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** Default {@link ShredWorkerClientOptions.maxInflight} (~10 MB of queued frames). */
|
|
64
|
+
export const DEFAULT_MAX_INFLIGHT = 8192;
|
|
65
|
+
|
|
66
|
+
export class ShredWorkerClient {
|
|
67
|
+
readonly #worker: Worker;
|
|
68
|
+
#onRecovered: ((data: ArrayBuffer, meta: RecoveredMeta) => void) | null = null;
|
|
69
|
+
#onStats: ((stats: ShredWorkerStats) => void) | null = null;
|
|
70
|
+
#onError: ((message: string, code?: ShredWorkerErrorCode) => void) | null = null;
|
|
71
|
+
#disposed = false;
|
|
72
|
+
|
|
73
|
+
/** feedShred in-flight backpressure accounting (see ShredWorkerClientOptions.maxInflight). */
|
|
74
|
+
readonly #maxInflight: number;
|
|
75
|
+
#sent = 0; // frames posted via feedShred (monotonic)
|
|
76
|
+
#processed = 0; // worker-consumed total from the last stats heartbeat (monotonic)
|
|
77
|
+
#droppedBackpressure = 0; // frames dropped by the in-flight guard (monotonic)
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* @param workerOrUrl - A URL (a Worker is created internally, `type: module`)
|
|
81
|
+
* or a pre-created Worker (for testing via constructor injection).
|
|
82
|
+
* @param opts - Optional {@link ShredWorkerClientOptions}.
|
|
83
|
+
*/
|
|
84
|
+
constructor(workerOrUrl: Worker | URL, opts: ShredWorkerClientOptions = {}) {
|
|
85
|
+
this.#maxInflight = opts.maxInflight ?? DEFAULT_MAX_INFLIGHT;
|
|
86
|
+
this.#worker =
|
|
87
|
+
workerOrUrl instanceof URL
|
|
88
|
+
? new Worker(workerOrUrl, { type: "module" })
|
|
89
|
+
: workerOrUrl;
|
|
90
|
+
this.#worker.onmessage = (e: MessageEvent<ShredWorkerEvent>) =>
|
|
91
|
+
this.#handleEvent(e.data);
|
|
92
|
+
this.#worker.onerror = (e) =>
|
|
93
|
+
this.#onError?.(formatWorkerError(e), "worker-runtime");
|
|
94
|
+
this.#worker.onmessageerror = () =>
|
|
95
|
+
this.#onError?.("failed to deserialize worker message", "worker-message");
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** Configure the decoder (WASM load + ShredFecBuffer construction happen in the worker). */
|
|
99
|
+
configure(config: ShredWorkerConfig): void {
|
|
100
|
+
if (this.#disposed) throw new Error("[ShredWorkerClient] disposed");
|
|
101
|
+
this.#post({ type: "configure", config });
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Hand the worker a stream of raw shred frames to batch-drain. The stream is
|
|
106
|
+
* transferred — the caller must not use it afterward. One active stream at a
|
|
107
|
+
* time; attaching a new one cancels the previous.
|
|
108
|
+
*/
|
|
109
|
+
attachStream(readable: ReadableStream<Uint8Array>): void {
|
|
110
|
+
if (this.#disposed) return;
|
|
111
|
+
this.#post({ type: "attachStream", readable }, [readable]);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Feed one already-demuxed raw shred frame. `frame` is transferred zero-copy;
|
|
116
|
+
* the caller must not access it afterward.
|
|
117
|
+
*
|
|
118
|
+
* Dropped (frame NOT transferred, so the caller's buffer stays intact) when the
|
|
119
|
+
* in-flight window is full — i.e. the worker is more than `maxInflight` frames
|
|
120
|
+
* behind. This is the memory bound: a slow/wedged consumer costs bounded
|
|
121
|
+
* skip-to-live drops (see {@link droppedBackpressure}), never an unbounded
|
|
122
|
+
* renderer-heap blow-up.
|
|
123
|
+
*/
|
|
124
|
+
feedShred(frame: ArrayBuffer): void {
|
|
125
|
+
if (this.#disposed) return;
|
|
126
|
+
if (this.#maxInflight > 0 && this.#sent - this.#processed >= this.#maxInflight) {
|
|
127
|
+
this.#droppedBackpressure++;
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
this.#sent++;
|
|
131
|
+
this.#post({ type: "feedShred", frame }, [frame]);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** Explicitly drive loss recovery (in addition to the worker's internal timer). */
|
|
135
|
+
recoverAll(nowMs: number = Date.now()): void {
|
|
136
|
+
if (this.#disposed) return;
|
|
137
|
+
this.#post({ type: "recoverAll", nowMs });
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** Register callback for recovered/completed FEC sets. `data` is a transferred buffer. */
|
|
141
|
+
onRecovered(cb: (data: ArrayBuffer, meta: RecoveredMeta) => void): void {
|
|
142
|
+
this.#onRecovered = cb;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** Register callback for the coalesced stats heartbeat. */
|
|
146
|
+
onStats(cb: (stats: ShredWorkerStats) => void): void {
|
|
147
|
+
this.#onStats = cb;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** Register callback for worker errors. */
|
|
151
|
+
onError(cb: (message: string, code?: ShredWorkerErrorCode) => void): void {
|
|
152
|
+
this.#onError = cb;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Frames dropped by the in-flight backpressure guard since construction
|
|
157
|
+
* (monotonic). Surface this in the UI so sustained loss is visible rather than
|
|
158
|
+
* silent — a rising counter means the consumer can't keep pace with ingest.
|
|
159
|
+
*/
|
|
160
|
+
get droppedBackpressure(): number {
|
|
161
|
+
return this.#droppedBackpressure;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* Frames posted via {@link feedShred} but not yet reported drained by the
|
|
166
|
+
* worker's last stats heartbeat (best-effort, heartbeat-granular; never < 0).
|
|
167
|
+
*/
|
|
168
|
+
get inflight(): number {
|
|
169
|
+
return Math.max(0, this.#sent - this.#processed);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** Dispose the worker. Idempotent. */
|
|
173
|
+
dispose(): void {
|
|
174
|
+
if (this.#disposed) return;
|
|
175
|
+
this.#disposed = true;
|
|
176
|
+
try {
|
|
177
|
+
this.#worker.postMessage({ type: "dispose" } satisfies ShredWorkerCommand);
|
|
178
|
+
} catch {
|
|
179
|
+
/* worker may already be gone */
|
|
180
|
+
}
|
|
181
|
+
this.#worker.terminate();
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
#post(cmd: ShredWorkerCommand, transfer: Transferable[] = []): void {
|
|
185
|
+
this.#worker.postMessage(cmd, transfer);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
#handleEvent(event: ShredWorkerEvent): void {
|
|
189
|
+
switch (event.type) {
|
|
190
|
+
case "recovered":
|
|
191
|
+
this.#onRecovered?.(event.data, event.meta);
|
|
192
|
+
break;
|
|
193
|
+
case "stats":
|
|
194
|
+
// The worker's monotonic consumed-frame total is the drain signal for
|
|
195
|
+
// the feedShred in-flight guard. shredsRx counts only valid frames;
|
|
196
|
+
// malformed + versionMismatch are consumed off the queue too, so the
|
|
197
|
+
// sum is the true "frames taken off the queue" and keeps the window
|
|
198
|
+
// from inflating (and false-dropping) on a malformed/skewed source.
|
|
199
|
+
this.#processed =
|
|
200
|
+
event.stats.shredsRx + event.stats.malformed + event.stats.versionMismatch;
|
|
201
|
+
this.#onStats?.(event.stats);
|
|
202
|
+
break;
|
|
203
|
+
case "error":
|
|
204
|
+
if (event.code) this.#onError?.(event.message, event.code);
|
|
205
|
+
else this.#onError?.(event.message);
|
|
206
|
+
break;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function formatWorkerError(event: ErrorEvent): string {
|
|
212
|
+
const parts = ["Worker error"];
|
|
213
|
+
if (event.message) parts.push(event.message);
|
|
214
|
+
if (event.filename) {
|
|
215
|
+
const loc = [event.filename];
|
|
216
|
+
if (event.lineno) loc.push(String(event.lineno));
|
|
217
|
+
if (event.colno) loc.push(String(event.colno));
|
|
218
|
+
parts.push(`at ${loc.join(":")}`);
|
|
219
|
+
}
|
|
220
|
+
if (event.error instanceof Error && event.error.stack) parts.push(event.error.stack);
|
|
221
|
+
return parts.join(": ");
|
|
222
|
+
}
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @blockcast/fec-worker — Shred (Reed-Solomon) Worker Protocol
|
|
3
|
+
*
|
|
4
|
+
* A sibling codec to the RaptorQ/MMTP worker (`fec-worker.ts`). Solana-style
|
|
5
|
+
* shred streams carry a different FEC payload identity than MMTP source/repair
|
|
6
|
+
* symbols — a shred is `(slot, fecSetIndex, localIndex, numData, numCoding,
|
|
7
|
+
* isCoding, lastInSet)`, not a flat `ssId` — so they need their own command
|
|
8
|
+
* shape. The transport harness (Worker spawn, onmessage plumbing, zero-copy
|
|
9
|
+
* `toOwnedArrayBuffer` transfers) is shared; the decoder WASM is not.
|
|
10
|
+
*
|
|
11
|
+
* The worker owns the batch-drain: the consumer transfers a WebTransport
|
|
12
|
+
* `datagrams.readable` (or any `ReadableStream<Uint8Array>` of raw shred
|
|
13
|
+
* frames) into the Worker via {@link ShredWorkerCommand} `attachStream`, and
|
|
14
|
+
* the read loop + wire parse + `ShredFecBuffer.add_shred` all run off the main
|
|
15
|
+
* thread. Only recovered FEC sets (and coalesced stats) cross back. A
|
|
16
|
+
* `feedShred` command covers sources that have already demuxed the shred bytes
|
|
17
|
+
* on the main thread (e.g. a moqtail OBJECT_DATAGRAM callback, or the IWA
|
|
18
|
+
* multicast bridge).
|
|
19
|
+
*
|
|
20
|
+
* Types here are importable from both main thread and Worker contexts.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
// --- Wire format ------------------------------------------------------------
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Wire header layout the worker parses. Mirrors the deployed shred player's
|
|
27
|
+
* `onShredBytes` exactly (libmmt shred-forwarder `emit_frame`) so the off-thread
|
|
28
|
+
* path is bit-identical to the proven on-thread path.
|
|
29
|
+
*
|
|
30
|
+
* v3 (28-byte header):
|
|
31
|
+
* [0] version u8 (== 3; non-3 frames are dropped, not misframed)
|
|
32
|
+
* [1-8] slot u64 LE
|
|
33
|
+
* [9-12] fecSetIndex u32 LE
|
|
34
|
+
* [13-16] localIndex u32 LE (data: 0..numData; coding: numData+position)
|
|
35
|
+
* [17] flags u8 (bit0 = DATA_COMPLETE, bit1 = IS_CODING_SHRED)
|
|
36
|
+
* [18] numData u8 (coding shreds only; 0 on data — WASM infers)
|
|
37
|
+
* [19] numCoding u8 (coding shreds only; 0 on data)
|
|
38
|
+
* [20-27] sendTsUs u64 LE (µs since epoch, 0 when unstamped)
|
|
39
|
+
* [28+] erasureShard (equal length across the FEC set)
|
|
40
|
+
*
|
|
41
|
+
* DATA_COMPLETE (last data shred in the set) is passed to `add_shred` as its
|
|
42
|
+
* `lastInSet` argument — the WASM uses it to infer numData for a set that
|
|
43
|
+
* received only data shreds (no coding).
|
|
44
|
+
*/
|
|
45
|
+
export const SHRED_WIRE_VERSION = 3;
|
|
46
|
+
export const SHRED_HEADER_LEN = 28;
|
|
47
|
+
export const SHRED_FLAG_DATA_COMPLETE = 0x01;
|
|
48
|
+
export const SHRED_FLAG_IS_CODING = 0x02;
|
|
49
|
+
|
|
50
|
+
// --- Config -----------------------------------------------------------------
|
|
51
|
+
|
|
52
|
+
/** Per-decoder configuration for the shred FEC worker. */
|
|
53
|
+
export interface ShredWorkerConfig {
|
|
54
|
+
/** Ring capacity in slots (matches `new ShredFecBuffer(maxSlots, symbolSize)`). */
|
|
55
|
+
maxSlots: number;
|
|
56
|
+
/** Symbol size in bytes (Solana shred payloads are fixed-size within a set). */
|
|
57
|
+
symbolSize: number;
|
|
58
|
+
/**
|
|
59
|
+
* Module URL for the shred WASM (wasm-bindgen ESM exporting `default` init +
|
|
60
|
+
* `ShredFecBuffer`). The worker `import()`s it and calls `await default()`.
|
|
61
|
+
* Ignored when `self.__SHRED_WASM_BINDINGS__` is pre-injected (tests / custom
|
|
62
|
+
* loaders), mirroring the RaptorQ worker's `__MMT_WASM_BINDINGS__` contract.
|
|
63
|
+
*/
|
|
64
|
+
wasmUrl?: string;
|
|
65
|
+
/**
|
|
66
|
+
* How often (ms) the worker coalesces and posts a `stats` event. The read
|
|
67
|
+
* loop never posts per-shred; only recovered frames and this heartbeat cross
|
|
68
|
+
* the boundary. Default 250ms.
|
|
69
|
+
*/
|
|
70
|
+
statsIntervalMs?: number;
|
|
71
|
+
/**
|
|
72
|
+
* How often (ms) the worker runs `try_recover_all(now)` to drain timed-out
|
|
73
|
+
* FEC sets that `add_shred` didn't complete (loss scenarios). Default 50ms,
|
|
74
|
+
* matching the player's on-thread recovery timer. Set to 0 to disable (the
|
|
75
|
+
* consumer may instead drive it explicitly with `recoverAll`).
|
|
76
|
+
*/
|
|
77
|
+
recoverIntervalMs?: number;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// --- Stats ------------------------------------------------------------------
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* First-shred-per-slot sample, collected by the worker and drained into each
|
|
84
|
+
* coalesced `stats` heartbeat. Lets the main thread run app-specific,
|
|
85
|
+
* slot-derived stats (e.g. the turbine-leg latency estimator, per-slot latency
|
|
86
|
+
* percentiles) WITHOUT re-parsing the wire header — the worker stays the single
|
|
87
|
+
* source of truth for parsing, and these cross the boundary aggregated (a
|
|
88
|
+
* handful per heartbeat), never per-packet.
|
|
89
|
+
*/
|
|
90
|
+
export interface SlotSample {
|
|
91
|
+
/** Slot number (first shred of this slot the worker saw). */
|
|
92
|
+
slot: bigint;
|
|
93
|
+
/** Publisher send timestamp (µs since epoch) of that first shred, 0 when unstamped. */
|
|
94
|
+
sendTsUs: bigint;
|
|
95
|
+
/** Frame byte length of that first shred. */
|
|
96
|
+
bytes: number;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** Aggregate shred-decode counters, emitted on the coalesced `stats` heartbeat. */
|
|
100
|
+
export interface ShredWorkerStats {
|
|
101
|
+
/** Total shred frames parsed (data + coding). */
|
|
102
|
+
shredsRx: number;
|
|
103
|
+
/** Data (source) shreds observed. */
|
|
104
|
+
sourceShreds: number;
|
|
105
|
+
/** Coding (repair) shreds observed. */
|
|
106
|
+
repairShreds: number;
|
|
107
|
+
/** FEC sets completed (direct completion + FEC recovery). */
|
|
108
|
+
fecSetsCompleted: number;
|
|
109
|
+
/** Bytes handed back as recovered/completed sets. */
|
|
110
|
+
bytesReassembled: number;
|
|
111
|
+
/** Sets recovered specifically via `try_recover_all` (loss recovery). */
|
|
112
|
+
fecSetsRecovered: number;
|
|
113
|
+
/** Current `ShredFecBuffer.pending_fec_sets()`, or -1 when unavailable. */
|
|
114
|
+
pendingFecSets: number;
|
|
115
|
+
/** Frames dropped for being shorter than the v3 header. */
|
|
116
|
+
malformed: number;
|
|
117
|
+
/** Frames dropped for carrying a non-v3 version byte (forwarder/worker skew). */
|
|
118
|
+
versionMismatch: number;
|
|
119
|
+
/** Highest slot number observed (as a string — bigint isn't JSON-safe for UI). */
|
|
120
|
+
currentSlot: string;
|
|
121
|
+
/** First-shred-per-slot samples parsed since the previous heartbeat (drained each emit). */
|
|
122
|
+
slotSamples: SlotSample[];
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** Stable identities for worker failures whose message text carries live values. */
|
|
126
|
+
export type ShredWorkerErrorCode =
|
|
127
|
+
| "wasm-load-failed"
|
|
128
|
+
| "wasm-bindings-missing"
|
|
129
|
+
| "not-configured"
|
|
130
|
+
| "stream-read-failed"
|
|
131
|
+
| "unknown-command"
|
|
132
|
+
| "worker-runtime"
|
|
133
|
+
| "worker-message";
|
|
134
|
+
|
|
135
|
+
// --- Recovered frame metadata ----------------------------------------------
|
|
136
|
+
|
|
137
|
+
/** Metadata attached to each recovered set the worker emits. */
|
|
138
|
+
export interface RecoveredMeta {
|
|
139
|
+
/**
|
|
140
|
+
* Slot the recovered set belongs to. Attributed on both the direct-completion
|
|
141
|
+
* path and the identity-carrying recovery drain
|
|
142
|
+
* (`try_recover_all_with_identity`); only 0n on the legacy `ShredReassembler`
|
|
143
|
+
* fallback, which has no identity variant. Do NOT branch on `slot === 0n` as
|
|
144
|
+
* "unattributed" — that would drop the majority of recovered sets.
|
|
145
|
+
*/
|
|
146
|
+
slot: bigint;
|
|
147
|
+
/** FEC set index within the slot (0 only on the legacy identity-less fallback). */
|
|
148
|
+
fecSetIndex: number;
|
|
149
|
+
/**
|
|
150
|
+
* Publisher send timestamp (µs since epoch), 0 when unstamped/unknown. On
|
|
151
|
+
* direct completion this is the completing shred's stamp; on the recovery
|
|
152
|
+
* drain it is the set's first-seen shred stamp (shreds of a set are sent µs
|
|
153
|
+
* apart, so the two are equivalent latency baselines).
|
|
154
|
+
*/
|
|
155
|
+
sendTsUs: bigint;
|
|
156
|
+
/** True when produced by loss recovery (`try_recover_all`) rather than direct completion. */
|
|
157
|
+
recovered: boolean;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// --- Commands (main -> worker) ----------------------------------------------
|
|
161
|
+
|
|
162
|
+
/** Discriminated union of all commands sent from the main thread to the shred worker. */
|
|
163
|
+
export type ShredWorkerCommand =
|
|
164
|
+
| { type: "configure"; config: ShredWorkerConfig }
|
|
165
|
+
| {
|
|
166
|
+
/**
|
|
167
|
+
* Hand the worker a stream of raw shred frames to batch-drain. Each
|
|
168
|
+
* chunk is one wire frame (one WebTransport datagram). The stream is
|
|
169
|
+
* transferred (add to the postMessage transfer list) — the main thread
|
|
170
|
+
* must not touch it afterward.
|
|
171
|
+
*/
|
|
172
|
+
type: "attachStream";
|
|
173
|
+
readable: ReadableStream<Uint8Array>;
|
|
174
|
+
}
|
|
175
|
+
| {
|
|
176
|
+
/**
|
|
177
|
+
* Feed one already-demuxed raw shred frame. `frame` is transferred
|
|
178
|
+
* zero-copy — the caller must not access it after posting. Use this for
|
|
179
|
+
* sources whose demux stays on the main thread (moqtail OBJECT_DATAGRAM
|
|
180
|
+
* callback, IWA multicast bridge).
|
|
181
|
+
*/
|
|
182
|
+
type: "feedShred";
|
|
183
|
+
frame: ArrayBuffer;
|
|
184
|
+
}
|
|
185
|
+
| { type: "recoverAll"; nowMs: number }
|
|
186
|
+
| { type: "dispose" };
|
|
187
|
+
|
|
188
|
+
// --- Events (worker -> main) ------------------------------------------------
|
|
189
|
+
|
|
190
|
+
/** Discriminated union of all events emitted from the shred worker to the main thread. */
|
|
191
|
+
export type ShredWorkerEvent =
|
|
192
|
+
| { type: "recovered"; data: ArrayBuffer; meta: RecoveredMeta }
|
|
193
|
+
| { type: "stats"; stats: ShredWorkerStats }
|
|
194
|
+
| { type: "error"; message: string; code?: ShredWorkerErrorCode };
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import { toOwnedArrayBuffer } from "./transfer.js";
|
|
3
|
+
|
|
4
|
+
describe("toOwnedArrayBuffer", () => {
|
|
5
|
+
it("copies exactly the bytes in an offset Uint8Array view", () => {
|
|
6
|
+
const backing = new Uint8Array([0, 1, 2, 3, 4, 5]);
|
|
7
|
+
const view = backing.subarray(2, 5);
|
|
8
|
+
|
|
9
|
+
const owned = toOwnedArrayBuffer(view);
|
|
10
|
+
|
|
11
|
+
expect([...new Uint8Array(owned)]).toEqual([2, 3, 4]);
|
|
12
|
+
expect(owned.byteLength).toBe(3);
|
|
13
|
+
expect(owned).not.toBe(backing.buffer);
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
it("does not alias the source buffer after copying", () => {
|
|
17
|
+
const source = new Uint8Array([9, 8, 7]);
|
|
18
|
+
const owned = toOwnedArrayBuffer(source);
|
|
19
|
+
|
|
20
|
+
source[1] = 1;
|
|
21
|
+
|
|
22
|
+
expect([...new Uint8Array(owned)]).toEqual([9, 8, 7]);
|
|
23
|
+
});
|
|
24
|
+
});
|
package/src/transfer.ts
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Return a fresh ArrayBuffer containing exactly the bytes in `view`.
|
|
3
|
+
*
|
|
4
|
+
* Worker outputs must not transfer buffers that might be owned by a WASM
|
|
5
|
+
* binding or by a larger shared packet. A fresh JS-owned buffer gives
|
|
6
|
+
* postMessage() exclusive ownership without detaching unrelated memory.
|
|
7
|
+
*/
|
|
8
|
+
export function toOwnedArrayBuffer(view: Uint8Array): ArrayBuffer {
|
|
9
|
+
const copy = new Uint8Array(view.byteLength);
|
|
10
|
+
copy.set(view);
|
|
11
|
+
return copy.buffer;
|
|
12
|
+
}
|