@decentnetwork/peer 0.1.72 → 0.1.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/compat/filetransfer.d.ts +1 -0
- package/dist/compat/filetransfer.js +204 -20
- package/dist/compat/reed-solomon.d.ts +9 -0
- package/dist/compat/reed-solomon.js +198 -0
- package/dist/peer.js +4 -3
- package/package.json +1 -1
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
export declare const PACKET_ID_FILE_SENDREQUEST = 80;
|
|
2
2
|
export declare const PACKET_ID_FILE_CONTROL = 81;
|
|
3
3
|
export declare const PACKET_ID_FILE_DATA = 82;
|
|
4
|
+
export declare const PACKET_ID_FILE_FEC = 83;
|
|
4
5
|
export declare const FILE_ID_LENGTH = 32;
|
|
5
6
|
export declare const MAX_FILENAME_LENGTH = 255;
|
|
6
7
|
export declare const MAX_CONCURRENT_FILE_PIPES = 256;
|
|
@@ -23,14 +23,44 @@ import { concatBytes, randomBytes } from "../utils/bytes.js";
|
|
|
23
23
|
import { createHash } from "node:crypto";
|
|
24
24
|
import { existsSync, statSync, readFileSync, appendFileSync, writeFileSync, unlinkSync, mkdirSync } from "node:fs";
|
|
25
25
|
import { join } from "node:path";
|
|
26
|
+
import { rsEncode, rsDecode } from "./reed-solomon.js";
|
|
26
27
|
export const PACKET_ID_FILE_SENDREQUEST = 80;
|
|
27
28
|
export const PACKET_ID_FILE_CONTROL = 81;
|
|
28
29
|
export const PACKET_ID_FILE_DATA = 82;
|
|
30
|
+
export const PACKET_ID_FILE_FEC = 83; // FEC parity shard (forward error correction)
|
|
29
31
|
export const FILE_ID_LENGTH = 32;
|
|
30
32
|
export const MAX_FILENAME_LENGTH = 255;
|
|
31
33
|
export const MAX_CONCURRENT_FILE_PIPES = 256;
|
|
32
34
|
// MAX_CRYPTO_DATA_SIZE (1373) − 1 (kind) − 1 (filenum) − 4 (offset).
|
|
33
35
|
export const MAX_FILE_DATA_SIZE = 1367;
|
|
36
|
+
// FEC: a block is K data chunks; the sender adds M parity chunks so the receiver
|
|
37
|
+
// can reconstruct up to M lost chunks per block locally (no retransmit) — the
|
|
38
|
+
// key to bulk transfer over a lossy GFW path (measured 15-50% loss). ~37%
|
|
39
|
+
// overhead handles bursty ~15% loss with margin; the retransmit path remains a
|
|
40
|
+
// backstop for blocks that exceed FEC capacity.
|
|
41
|
+
const FEC_K = 16;
|
|
42
|
+
const FEC_M = 6;
|
|
43
|
+
// Adaptive: only spend parity bandwidth while the path is actually losing
|
|
44
|
+
// packets (a dup-ack was seen within this window). A clean path sends zero
|
|
45
|
+
// parity → no overhead; a lossy path (lili) gets full protection.
|
|
46
|
+
const FEC_ACTIVE_WINDOW_MS = 4000;
|
|
47
|
+
// FEC packet payload: [filenum(1)][blockIndex u32][parityIndex(1)][parity bytes].
|
|
48
|
+
// k (data chunks in the block) and m are NOT sent — both ends share FEC_K/FEC_M
|
|
49
|
+
// constants and derive the last block's k from the file size — so the 6-byte
|
|
50
|
+
// header + a full 1367-byte parity shard fits the 1373-byte crypto limit exactly
|
|
51
|
+
// and the data chunk size (MAX_FILE_DATA_SIZE) stays unchanged (0.1.73-compatible).
|
|
52
|
+
function encodeFec(fileNumber, blockIndex, parityIndex, parity) {
|
|
53
|
+
return concatBytes([Uint8Array.of(fileNumber & 0xff), u32be(blockIndex), Uint8Array.of(parityIndex & 0xff), parity]);
|
|
54
|
+
}
|
|
55
|
+
function parseFec(p) {
|
|
56
|
+
if (p.length < 6)
|
|
57
|
+
return undefined;
|
|
58
|
+
return { fileNumber: p[0], blockIndex: readU32be(p, 1), parityIndex: p[5], parity: p.slice(6) };
|
|
59
|
+
}
|
|
60
|
+
// How many data chunks are in block b of a file with `totalChunks` chunks.
|
|
61
|
+
function fecBlockK(blockIndex, totalChunks) {
|
|
62
|
+
return Math.min(FEC_K, totalChunks - blockIndex * FEC_K);
|
|
63
|
+
}
|
|
34
64
|
export const FILECONTROL = { ACCEPT: 0, PAUSE: 1, KILL: 2, SEEK: 3, ACK: 4 };
|
|
35
65
|
export const FILEKIND = { DATA: 0, AVATAR: 1 };
|
|
36
66
|
// Reliability tuning. We RAMP an in-flight window (AIMD-lite) rather than pin it:
|
|
@@ -62,10 +92,14 @@ const CWND_MIN_CHUNKS = 8; // ~11 KB floor — keeps a slow/flaky path's buffer
|
|
|
62
92
|
// link keeps a big window (it drains quickly, so a big window adds little
|
|
63
93
|
// delay); a thin flaky one keeps a small window (a big window there would add
|
|
64
94
|
// seconds). Growing whenever the added delay is low means no self-limiting.
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
95
|
+
// The budget is generous on purpose: for a bulk transfer to a flaky peer,
|
|
96
|
+
// ~0.5s of added latency keeps ping well under a second (fine — the peer stays
|
|
97
|
+
// usable), and buys far more throughput on a jittery path than a tight 150ms
|
|
98
|
+
// bound, which collapsed the window to the floor (~17 KB/s observed on lili).
|
|
99
|
+
const MAX_QUEUE_MS = 350; // raw-queue ceiling before draining (bulk transfer tolerates ~0.35s added latency)
|
|
100
|
+
const QUEUE_LOW_MS = 150; // grow while the smoothed (jitter-free) queue is under this
|
|
101
|
+
const VEGAS_GROW = 1.25; // window ×this per RTT when under-filled
|
|
102
|
+
const VEGAS_SHRINK = 0.8; // window ×this per RTT when the queue is too deep
|
|
69
103
|
const WATCHDOG_MS = 150; // stall poll cadence / base no-progress patience
|
|
70
104
|
const WATCHDOG_MAX_MS = 8000; // cap the RTT-adaptive stall patience (must exceed a slow path's RTT so acks aren't mistaken for a stall)
|
|
71
105
|
const FAST_RT_MIN_MS = 40; // min gap between fast-retransmits of the same hole (≈1 RTT)
|
|
@@ -193,8 +227,8 @@ export class FileTransferManager {
|
|
|
193
227
|
fileNumber, fileId, name: opts.name, data, size: data.length,
|
|
194
228
|
acked: 0, nextSend: 0, pumping: false, started: false, req, startMs: nowMs(),
|
|
195
229
|
lastProgressAcked: 0, lastAckAdvanceMs: nowMs(), lastResendMs: 0,
|
|
196
|
-
cwnd: CWND_INIT_CHUNKS, stalls: 0,
|
|
197
|
-
minRttMs: Infinity, srttMs: 0, probeOffset: -1, probeSentMs: 0,
|
|
230
|
+
cwnd: CWND_INIT_CHUNKS, stalls: 0, parityHighBlock: -1, lastLossMs: 0,
|
|
231
|
+
minRttMs: Infinity, minSrttMs: Infinity, srttMs: 0, probeOffset: -1, probeSentMs: 0,
|
|
198
232
|
};
|
|
199
233
|
this.#map(this.#sending, friendId).set(fileNumber, st);
|
|
200
234
|
void this.send(friendId, PACKET_ID_FILE_SENDREQUEST, req).catch(() => undefined);
|
|
@@ -265,6 +299,10 @@ export class FileTransferManager {
|
|
|
265
299
|
this.#onData(friendId, payload);
|
|
266
300
|
return true;
|
|
267
301
|
}
|
|
302
|
+
if (packetId === PACKET_ID_FILE_FEC) {
|
|
303
|
+
this.#onFec(friendId, payload);
|
|
304
|
+
return true;
|
|
305
|
+
}
|
|
268
306
|
return false;
|
|
269
307
|
}
|
|
270
308
|
#onSendRequest(friendId, payload) {
|
|
@@ -280,7 +318,7 @@ export class FileTransferManager {
|
|
|
280
318
|
const st = {
|
|
281
319
|
fileNumber: r.fileNumber, fileId: r.fileId, name: r.filename, size,
|
|
282
320
|
buf: new Uint8Array(size), got: new Uint8Array(chunkCount(size)),
|
|
283
|
-
contiguous: 0, maxByte: 0, persisted: 0, partPath: this.#partPath(r.fileId), done: false, ackDirty: false,
|
|
321
|
+
contiguous: 0, maxByte: 0, persisted: 0, partPath: this.#partPath(r.fileId), fecParity: new Map(), done: false, ackDirty: false,
|
|
284
322
|
};
|
|
285
323
|
// 断点续传: if we persisted a contiguous prefix of this exact file (matched
|
|
286
324
|
// by content-hash fileId) before a drop/restart, load it and resume — our
|
|
@@ -381,20 +419,114 @@ export class FileTransferManager {
|
|
|
381
419
|
if (!st.got[idx]) {
|
|
382
420
|
st.buf.set(data, offset);
|
|
383
421
|
st.got[idx] = 1;
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
422
|
+
this.#advanceContiguous(friendId, st);
|
|
423
|
+
}
|
|
424
|
+
// This chunk may complete a FEC block enough to reconstruct its other holes.
|
|
425
|
+
this.#tryRecoverBlock(friendId, st, Math.floor(idx / FEC_K));
|
|
426
|
+
this.#finishOrAck(friendId, st);
|
|
427
|
+
}
|
|
428
|
+
// Advance the contiguous prefix over received/recovered chunks, emitting
|
|
429
|
+
// progress and flushing the resume file. Does not finish or ack.
|
|
430
|
+
#advanceContiguous(friendId, st) {
|
|
431
|
+
let i = Math.floor(st.contiguous / MAX_FILE_DATA_SIZE);
|
|
432
|
+
while (i < st.got.length && st.got[i])
|
|
433
|
+
i++;
|
|
434
|
+
const nc = Math.min(i * MAX_FILE_DATA_SIZE, st.size);
|
|
435
|
+
if (nc > st.contiguous) {
|
|
436
|
+
st.contiguous = nc;
|
|
389
437
|
this.emit("file-progress", { friendId, fileId: hex(st.fileId), received: st.contiguous, total: st.size });
|
|
390
438
|
this.#persistPartial(st);
|
|
391
439
|
}
|
|
440
|
+
}
|
|
441
|
+
#finishOrAck(friendId, st) {
|
|
442
|
+
if (st.done)
|
|
443
|
+
return;
|
|
392
444
|
if (st.contiguous >= st.size) {
|
|
393
445
|
this.#finishRecv(friendId, st);
|
|
394
446
|
return;
|
|
395
447
|
}
|
|
396
448
|
this.#scheduleAck(friendId, st);
|
|
397
449
|
}
|
|
450
|
+
// A parity shard arrived → store it and try to repair its block.
|
|
451
|
+
#onFec(friendId, payload) {
|
|
452
|
+
const f = parseFec(payload);
|
|
453
|
+
if (!f)
|
|
454
|
+
return;
|
|
455
|
+
const st = this.#map(this.#receiving, friendId).get(f.fileNumber);
|
|
456
|
+
if (!st || st.done)
|
|
457
|
+
return;
|
|
458
|
+
if (f.parity.length !== MAX_FILE_DATA_SIZE || f.parityIndex >= FEC_M)
|
|
459
|
+
return;
|
|
460
|
+
let arr = st.fecParity.get(f.blockIndex);
|
|
461
|
+
if (!arr) {
|
|
462
|
+
arr = new Array(FEC_M).fill(undefined);
|
|
463
|
+
st.fecParity.set(f.blockIndex, arr);
|
|
464
|
+
}
|
|
465
|
+
if (!arr[f.parityIndex])
|
|
466
|
+
arr[f.parityIndex] = f.parity;
|
|
467
|
+
this.#tryRecoverBlock(friendId, st, f.blockIndex);
|
|
468
|
+
this.#finishOrAck(friendId, st);
|
|
469
|
+
}
|
|
470
|
+
// If FEC block `b` has at least K of its K+M shards (data + parity), rebuild
|
|
471
|
+
// the missing data chunks LOCALLY — no retransmit, so the sender's window
|
|
472
|
+
// never stalls waiting for the lost chunk.
|
|
473
|
+
#tryRecoverBlock(friendId, st, b) {
|
|
474
|
+
if (st.done || b < 0)
|
|
475
|
+
return;
|
|
476
|
+
const totalChunks = chunkCount(st.size);
|
|
477
|
+
if (b * FEC_K >= totalChunks)
|
|
478
|
+
return;
|
|
479
|
+
const k = fecBlockK(b, totalChunks);
|
|
480
|
+
const base = b * FEC_K;
|
|
481
|
+
const missing = [];
|
|
482
|
+
for (let i = 0; i < k; i++)
|
|
483
|
+
if (!st.got[base + i])
|
|
484
|
+
missing.push(i);
|
|
485
|
+
if (missing.length === 0) {
|
|
486
|
+
st.fecParity.delete(b);
|
|
487
|
+
return;
|
|
488
|
+
} // block already whole
|
|
489
|
+
const parityArr = st.fecParity.get(b);
|
|
490
|
+
const parityPresent = parityArr ? parityArr.reduce((n, s) => n + (s ? 1 : 0), 0) : 0;
|
|
491
|
+
if ((k - missing.length) + parityPresent < k)
|
|
492
|
+
return; // not enough shards yet
|
|
493
|
+
if (missing.length > parityPresent)
|
|
494
|
+
return; // can't cover this many holes
|
|
495
|
+
// Assemble the K+M shard array (present data padded to a full chunk).
|
|
496
|
+
const shards = [];
|
|
497
|
+
for (let i = 0; i < k; i++) {
|
|
498
|
+
if (st.got[base + i]) {
|
|
499
|
+
const off = (base + i) * MAX_FILE_DATA_SIZE;
|
|
500
|
+
const chunk = st.buf.subarray(off, Math.min(off + MAX_FILE_DATA_SIZE, st.size));
|
|
501
|
+
if (chunk.length === MAX_FILE_DATA_SIZE)
|
|
502
|
+
shards.push(chunk);
|
|
503
|
+
else {
|
|
504
|
+
const padded = new Uint8Array(MAX_FILE_DATA_SIZE);
|
|
505
|
+
padded.set(chunk);
|
|
506
|
+
shards.push(padded);
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
else
|
|
510
|
+
shards.push(null);
|
|
511
|
+
}
|
|
512
|
+
for (let pi = 0; pi < FEC_M; pi++)
|
|
513
|
+
shards.push(parityArr?.[pi] ?? null);
|
|
514
|
+
const recovered = rsDecode(shards, k, FEC_M, MAX_FILE_DATA_SIZE);
|
|
515
|
+
if (!recovered)
|
|
516
|
+
return;
|
|
517
|
+
for (const i of missing) {
|
|
518
|
+
const off = (base + i) * MAX_FILE_DATA_SIZE;
|
|
519
|
+
const realLen = Math.min(MAX_FILE_DATA_SIZE, st.size - off);
|
|
520
|
+
if (realLen <= 0)
|
|
521
|
+
continue;
|
|
522
|
+
st.buf.set(recovered[i].subarray(0, realLen), off);
|
|
523
|
+
st.got[base + i] = 1;
|
|
524
|
+
if (off + realLen > st.maxByte)
|
|
525
|
+
st.maxByte = off + realLen;
|
|
526
|
+
}
|
|
527
|
+
st.fecParity.delete(b); // recovered — free the parity
|
|
528
|
+
this.#advanceContiguous(friendId, st);
|
|
529
|
+
}
|
|
398
530
|
// Flush the newly-contiguous bytes to the resume file, throttled so we do at
|
|
399
531
|
// most one append per PERSIST_INTERVAL_BYTES of progress (not per chunk).
|
|
400
532
|
#persistPartial(st, force = false) {
|
|
@@ -498,15 +630,23 @@ export class FileTransferManager {
|
|
|
498
630
|
st.srttMs = st.srttMs === 0 ? rtt : 0.85 * st.srttMs + 0.15 * rtt;
|
|
499
631
|
if (rtt < st.minRttMs)
|
|
500
632
|
st.minRttMs = rtt;
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
633
|
+
if (st.srttMs < st.minSrttMs)
|
|
634
|
+
st.minSrttMs = st.srttMs;
|
|
635
|
+
// Hybrid signal, so per-packet JITTER doesn't masquerade as a queue:
|
|
636
|
+
// • GROW on the SMOOTHED queue (srtt − minSrtt): jitter averages out,
|
|
637
|
+
// so a flaky path that isn't really queuing keeps growing instead
|
|
638
|
+
// of collapsing to the floor (~17 KB/s observed on lili).
|
|
639
|
+
// • SHRINK on the RAW queue (rtt − minRtt) against a higher ceiling:
|
|
640
|
+
// fast reaction to genuine bufferbloat, but the ceiling sits above
|
|
641
|
+
// the jitter band so a lone jittery sample doesn't trigger it.
|
|
642
|
+
const smoothQueue = st.minSrttMs !== Infinity ? st.srttMs - st.minSrttMs : 0;
|
|
643
|
+
const rawQueue = st.minRttMs !== Infinity ? rtt - st.minRttMs : 0;
|
|
644
|
+
if (rawQueue > MAX_QUEUE_MS) {
|
|
645
|
+
st.cwnd = Math.max(CWND_MIN_CHUNKS, Math.floor(st.cwnd * VEGAS_SHRINK)); // real queue too deep → drain
|
|
506
646
|
}
|
|
507
|
-
else if (
|
|
508
|
-
st.cwnd = Math.
|
|
509
|
-
} // else hold
|
|
647
|
+
else if (smoothQueue < QUEUE_LOW_MS) {
|
|
648
|
+
st.cwnd = Math.min(WINDOW_CHUNKS, Math.ceil(st.cwnd * VEGAS_GROW)); // spare capacity → grow
|
|
649
|
+
} // else hold
|
|
510
650
|
}
|
|
511
651
|
}
|
|
512
652
|
if (st.acked - st.lastProgressAcked >= 256 * 1024 || st.acked >= st.size) {
|
|
@@ -520,6 +660,7 @@ export class FileTransferManager {
|
|
|
520
660
|
// ~1 RTT so dup-ack bursts for one loss don't storm). No window backoff:
|
|
521
661
|
// file-path loss is treated as random, not congestion.
|
|
522
662
|
st.lastResendMs = nowMs();
|
|
663
|
+
st.lastLossMs = nowMs(); // the path is lossy → turn adaptive FEC on
|
|
523
664
|
if (st.nextSend > st.acked)
|
|
524
665
|
st.nextSend = st.acked;
|
|
525
666
|
}
|
|
@@ -604,11 +745,54 @@ export class FileTransferManager {
|
|
|
604
745
|
st.probeSentMs = nowMs();
|
|
605
746
|
}
|
|
606
747
|
}
|
|
748
|
+
// After sending data, emit FEC parity for any block whose data is now all
|
|
749
|
+
// sent, so the receiver can repair losses locally instead of stalling.
|
|
750
|
+
this.#emitParity(friendId, st);
|
|
607
751
|
}
|
|
608
752
|
finally {
|
|
609
753
|
st.pumping = false;
|
|
610
754
|
}
|
|
611
755
|
}
|
|
756
|
+
// Send M parity shards for each FEC block whose K data chunks have all been
|
|
757
|
+
// sent (once per block). Parity is computed from the source file, so no state
|
|
758
|
+
// is kept beyond the high-water block index.
|
|
759
|
+
#emitParity(friendId, st) {
|
|
760
|
+
const totalChunks = chunkCount(st.size);
|
|
761
|
+
const totalBlocks = Math.ceil(totalChunks / FEC_K);
|
|
762
|
+
// Adaptive: only protect blocks while the path is actually lossy. On a clean
|
|
763
|
+
// path this sends zero parity (no overhead); the counter still advances so
|
|
764
|
+
// we never revisit a block.
|
|
765
|
+
const fecOn = nowMs() - st.lastLossMs < FEC_ACTIVE_WINDOW_MS;
|
|
766
|
+
while (st.parityHighBlock + 1 < totalBlocks) {
|
|
767
|
+
const b = st.parityHighBlock + 1;
|
|
768
|
+
const kActual = fecBlockK(b, totalChunks);
|
|
769
|
+
const blockEndByte = Math.min((b * FEC_K + kActual) * MAX_FILE_DATA_SIZE, st.size);
|
|
770
|
+
if (st.nextSend < blockEndByte)
|
|
771
|
+
break; // block's data not all sent yet
|
|
772
|
+
if (!fecOn) {
|
|
773
|
+
st.parityHighBlock = b;
|
|
774
|
+
continue;
|
|
775
|
+
} // clean path → skip parity
|
|
776
|
+
// Build the k data shards (zero-padded to a full chunk) and encode M parity.
|
|
777
|
+
const shards = [];
|
|
778
|
+
for (let i = 0; i < kActual; i++) {
|
|
779
|
+
const off = (b * FEC_K + i) * MAX_FILE_DATA_SIZE;
|
|
780
|
+
const chunk = st.data.subarray(off, Math.min(off + MAX_FILE_DATA_SIZE, st.size));
|
|
781
|
+
if (chunk.length === MAX_FILE_DATA_SIZE)
|
|
782
|
+
shards.push(chunk);
|
|
783
|
+
else {
|
|
784
|
+
const padded = new Uint8Array(MAX_FILE_DATA_SIZE);
|
|
785
|
+
padded.set(chunk);
|
|
786
|
+
shards.push(padded);
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
const parity = rsEncode(shards, FEC_M);
|
|
790
|
+
for (let pi = 0; pi < parity.length; pi++) {
|
|
791
|
+
void this.send(friendId, PACKET_ID_FILE_FEC, encodeFec(st.fileNumber, b, pi, parity[pi])).catch(() => undefined);
|
|
792
|
+
}
|
|
793
|
+
st.parityHighBlock = b;
|
|
794
|
+
}
|
|
795
|
+
}
|
|
612
796
|
#completeSend(friendId, st) {
|
|
613
797
|
const durationMs = nowMs() - st.startMs;
|
|
614
798
|
const mbps = durationMs > 0 ? st.size / 1024 / 1024 / (durationMs / 1000) : 0;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/** Encode: given K equal-length data shards, return M parity shards. */
|
|
2
|
+
export declare function rsEncode(data: Uint8Array[], m: number): Uint8Array[];
|
|
3
|
+
/**
|
|
4
|
+
* Recover the K data shards from any K present shards (data and/or parity).
|
|
5
|
+
* `shards` is the full K+M array with missing entries as null; `len` is the
|
|
6
|
+
* shard length. Returns the K reconstructed data shards, or null if fewer than
|
|
7
|
+
* K shards are present (unrecoverable).
|
|
8
|
+
*/
|
|
9
|
+
export declare function rsDecode(shards: Array<Uint8Array | null>, k: number, m: number, len: number): Uint8Array[] | null;
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Reed-Solomon ERASURE coding over GF(256), for FEC on lossy paths.
|
|
3
|
+
//
|
|
4
|
+
// The file transfer runs over a best-effort channel that, on a GFW-grade path,
|
|
5
|
+
// drops 15-50% of packets. Retransmit-based recovery collapses throughput
|
|
6
|
+
// (measured: ~1% of the link's real 1.3 MB/s). FEC instead sends a few PARITY
|
|
7
|
+
// shards alongside each block of K data shards; the receiver reconstructs any
|
|
8
|
+
// missing data shards LOCALLY as long as it got at least K of the K+M shards —
|
|
9
|
+
// no round-trip, so the send window never stalls on loss.
|
|
10
|
+
//
|
|
11
|
+
// This is *erasure* coding, not error correction: the receiver knows exactly
|
|
12
|
+
// which shards are missing (each chunk carries its offset), so recovery is just
|
|
13
|
+
// solving a linear system — no syndromes or error locators. We use a systematic
|
|
14
|
+
// Cauchy-Reed-Solomon matrix: the first K rows are the identity (data shards
|
|
15
|
+
// pass through untouched), the last M rows are a Cauchy matrix (the parity).
|
|
16
|
+
//
|
|
17
|
+
// All shards in a block must be the same length; the caller zero-pads.
|
|
18
|
+
//
|
|
19
|
+
// ── GF(256) with primitive polynomial 0x11d (x^8+x^4+x^3+x^2+1) ──────────────
|
|
20
|
+
const EXP = new Uint8Array(512);
|
|
21
|
+
const LOG = new Uint8Array(256);
|
|
22
|
+
(function initTables() {
|
|
23
|
+
let x = 1;
|
|
24
|
+
for (let i = 0; i < 255; i++) {
|
|
25
|
+
EXP[i] = x;
|
|
26
|
+
LOG[x] = i;
|
|
27
|
+
x <<= 1;
|
|
28
|
+
if (x & 0x100)
|
|
29
|
+
x ^= 0x11d;
|
|
30
|
+
}
|
|
31
|
+
for (let i = 255; i < 512; i++)
|
|
32
|
+
EXP[i] = EXP[i - 255];
|
|
33
|
+
})();
|
|
34
|
+
function gmul(a, b) {
|
|
35
|
+
if (a === 0 || b === 0)
|
|
36
|
+
return 0;
|
|
37
|
+
return EXP[LOG[a] + LOG[b]];
|
|
38
|
+
}
|
|
39
|
+
function gdiv(a, b) {
|
|
40
|
+
if (b === 0)
|
|
41
|
+
throw new Error("gf div by zero");
|
|
42
|
+
if (a === 0)
|
|
43
|
+
return 0;
|
|
44
|
+
return EXP[LOG[a] + 255 - LOG[b]];
|
|
45
|
+
}
|
|
46
|
+
function ginv(a) {
|
|
47
|
+
if (a === 0)
|
|
48
|
+
throw new Error("gf inv of zero");
|
|
49
|
+
return EXP[255 - LOG[a]];
|
|
50
|
+
}
|
|
51
|
+
// Build the M×K Cauchy parity matrix. Cauchy element = 1 / (x_i XOR y_j) with
|
|
52
|
+
// disjoint x (parity index space) and y (data index space). Guaranteed
|
|
53
|
+
// invertible submatrices → any K of K+M shards recover the data.
|
|
54
|
+
function cauchyParityMatrix(k, m) {
|
|
55
|
+
const rows = [];
|
|
56
|
+
for (let i = 0; i < m; i++) {
|
|
57
|
+
const row = new Uint8Array(k);
|
|
58
|
+
const xi = k + i; // parity nodes live at k..k+m-1
|
|
59
|
+
for (let j = 0; j < k; j++) {
|
|
60
|
+
row[j] = ginv(xi ^ j); // xi ^ j is never 0 since xi >= k > j
|
|
61
|
+
}
|
|
62
|
+
rows.push(row);
|
|
63
|
+
}
|
|
64
|
+
return rows;
|
|
65
|
+
}
|
|
66
|
+
/** Encode: given K equal-length data shards, return M parity shards. */
|
|
67
|
+
export function rsEncode(data, m) {
|
|
68
|
+
const k = data.length;
|
|
69
|
+
if (k === 0 || m === 0)
|
|
70
|
+
return [];
|
|
71
|
+
const len = data[0].length;
|
|
72
|
+
const parity = cauchyParityMatrix(k, m);
|
|
73
|
+
const out = [];
|
|
74
|
+
for (let i = 0; i < m; i++) {
|
|
75
|
+
const shard = new Uint8Array(len);
|
|
76
|
+
const row = parity[i];
|
|
77
|
+
for (let j = 0; j < k; j++) {
|
|
78
|
+
const coeff = row[j];
|
|
79
|
+
if (coeff === 0)
|
|
80
|
+
continue;
|
|
81
|
+
const src = data[j];
|
|
82
|
+
const lc = LOG[coeff];
|
|
83
|
+
for (let b = 0; b < len; b++) {
|
|
84
|
+
const s = src[b];
|
|
85
|
+
if (s !== 0)
|
|
86
|
+
shard[b] ^= EXP[lc + LOG[s]];
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
out.push(shard);
|
|
90
|
+
}
|
|
91
|
+
return out;
|
|
92
|
+
}
|
|
93
|
+
// Invert a K×K matrix in GF(256) via Gauss-Jordan. Returns null if singular
|
|
94
|
+
// (shouldn't happen for a Cauchy-derived submatrix, but guard anyway).
|
|
95
|
+
function invert(matrix, k) {
|
|
96
|
+
// Augment [M | I].
|
|
97
|
+
const a = matrix.map((r) => Uint8Array.from(r));
|
|
98
|
+
const inv = Array.from({ length: k }, (_, i) => {
|
|
99
|
+
const r = new Uint8Array(k);
|
|
100
|
+
r[i] = 1;
|
|
101
|
+
return r;
|
|
102
|
+
});
|
|
103
|
+
for (let col = 0; col < k; col++) {
|
|
104
|
+
// Find a pivot.
|
|
105
|
+
let piv = col;
|
|
106
|
+
while (piv < k && a[piv][col] === 0)
|
|
107
|
+
piv++;
|
|
108
|
+
if (piv === k)
|
|
109
|
+
return null;
|
|
110
|
+
if (piv !== col) {
|
|
111
|
+
[a[piv], a[col]] = [a[col], a[piv]];
|
|
112
|
+
[inv[piv], inv[col]] = [inv[col], inv[piv]];
|
|
113
|
+
}
|
|
114
|
+
// Normalize the pivot row.
|
|
115
|
+
const pv = a[col][col];
|
|
116
|
+
const pvInv = ginv(pv);
|
|
117
|
+
for (let j = 0; j < k; j++) {
|
|
118
|
+
a[col][j] = gmul(a[col][j], pvInv);
|
|
119
|
+
inv[col][j] = gmul(inv[col][j], pvInv);
|
|
120
|
+
}
|
|
121
|
+
// Eliminate the column from all other rows.
|
|
122
|
+
for (let row = 0; row < k; row++) {
|
|
123
|
+
if (row === col)
|
|
124
|
+
continue;
|
|
125
|
+
const f = a[row][col];
|
|
126
|
+
if (f === 0)
|
|
127
|
+
continue;
|
|
128
|
+
const lf = LOG[f];
|
|
129
|
+
for (let j = 0; j < k; j++) {
|
|
130
|
+
if (a[col][j] !== 0)
|
|
131
|
+
a[row][j] ^= EXP[lf + LOG[a[col][j]]];
|
|
132
|
+
if (inv[col][j] !== 0)
|
|
133
|
+
inv[row][j] ^= EXP[lf + LOG[inv[col][j]]];
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
return inv;
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Recover the K data shards from any K present shards (data and/or parity).
|
|
141
|
+
* `shards` is the full K+M array with missing entries as null; `len` is the
|
|
142
|
+
* shard length. Returns the K reconstructed data shards, or null if fewer than
|
|
143
|
+
* K shards are present (unrecoverable).
|
|
144
|
+
*/
|
|
145
|
+
export function rsDecode(shards, k, m, len) {
|
|
146
|
+
// Fast path: all K data shards present.
|
|
147
|
+
if (shards.slice(0, k).every((s) => s !== null))
|
|
148
|
+
return shards.slice(0, k);
|
|
149
|
+
// Collect the first K present shards and the rows of the full encoding matrix
|
|
150
|
+
// (identity for data rows, Cauchy for parity rows) that produced them.
|
|
151
|
+
const parity = cauchyParityMatrix(k, m);
|
|
152
|
+
const encRow = (idx) => {
|
|
153
|
+
if (idx < k) {
|
|
154
|
+
const r = new Uint8Array(k);
|
|
155
|
+
r[idx] = 1;
|
|
156
|
+
return r;
|
|
157
|
+
} // identity row
|
|
158
|
+
return parity[idx - k];
|
|
159
|
+
};
|
|
160
|
+
const subMatrix = [];
|
|
161
|
+
const present = [];
|
|
162
|
+
for (let idx = 0; idx < k + m && subMatrix.length < k; idx++) {
|
|
163
|
+
const s = shards[idx];
|
|
164
|
+
if (s) {
|
|
165
|
+
subMatrix.push(encRow(idx));
|
|
166
|
+
present.push(s);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
if (subMatrix.length < k)
|
|
170
|
+
return null; // not enough shards
|
|
171
|
+
const invMatrix = invert(subMatrix, k);
|
|
172
|
+
if (!invMatrix)
|
|
173
|
+
return null;
|
|
174
|
+
// data = inv(subMatrix) × present.
|
|
175
|
+
const data = [];
|
|
176
|
+
for (let i = 0; i < k; i++) {
|
|
177
|
+
if (shards[i]) {
|
|
178
|
+
data.push(shards[i]);
|
|
179
|
+
continue;
|
|
180
|
+
} // already have it
|
|
181
|
+
const shard = new Uint8Array(len);
|
|
182
|
+
const row = invMatrix[i];
|
|
183
|
+
for (let j = 0; j < k; j++) {
|
|
184
|
+
const coeff = row[j];
|
|
185
|
+
if (coeff === 0)
|
|
186
|
+
continue;
|
|
187
|
+
const src = present[j];
|
|
188
|
+
const lc = LOG[coeff];
|
|
189
|
+
for (let b = 0; b < len; b++) {
|
|
190
|
+
const sv = src[b];
|
|
191
|
+
if (sv !== 0)
|
|
192
|
+
shard[b] ^= EXP[lc + LOG[sv]];
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
data.push(shard);
|
|
196
|
+
}
|
|
197
|
+
return data;
|
|
198
|
+
}
|
package/dist/peer.js
CHANGED
|
@@ -9,7 +9,7 @@ import { PACKET_TYPE_MESSAGE, PACKET_TYPE_FRIEND_REQUEST, PACKET_TYPE_USERINFO,
|
|
|
9
9
|
import { NET_PACKET_ONION_ANNOUNCE_RESPONSE, NET_PACKET_ONION_DATA_RESPONSE, ONION_FRIEND_REQUEST_ID, createOnionAnnounceRequest, createOnionDataPacket, createOnionDataRequest, createOnionRequest0, openOnionAnnounceResponse, openOnionDataPacket, openOnionDataResponse } from "./compat/tox-onion.js";
|
|
10
10
|
import { CRYPTO_PACKET_DHTPK, CRYPTO_PACKET_FRIEND_REQ, NET_PACKET_CRYPTO, createToxDhtCryptoRequest, openToxDhtCryptoRequest } from "./compat/tox-dht-crypto.js";
|
|
11
11
|
import { NET_PACKET_PING_REQUEST, NET_PACKET_PING_RESPONSE, NET_PACKET_GET_NODES, NET_PACKET_SEND_NODES, encodeDhtRpc, decodeDhtRpc, buildPingPlain, parsePingPlain, buildGetNodesPlain, parseGetNodesPlain, buildSendNodesPlain, parseSendNodesNodeBytes, packUdpNodeV4 } from "./compat/dht-rpc.js";
|
|
12
|
-
import { FileTransferManager, PACKET_ID_FILE_SENDREQUEST, PACKET_ID_FILE_CONTROL, PACKET_ID_FILE_DATA } from "./compat/filetransfer.js";
|
|
12
|
+
import { FileTransferManager, PACKET_ID_FILE_SENDREQUEST, PACKET_ID_FILE_CONTROL, PACKET_ID_FILE_DATA, PACKET_ID_FILE_FEC } from "./compat/filetransfer.js";
|
|
13
13
|
import { NET_PACKET_CRYPTO_DATA, NET_PACKET_CRYPTO_HS, NET_PACKET_COOKIE_REQUEST, NET_PACKET_COOKIE_RESPONSE, createCookieRequest, createCookieResponse, createCryptoDataPacket, createCryptoHandshake, openCookieRequest, openCookieResponse, openCryptoDataPacket, openCryptoHandshake, incrementNonce } from "./compat/net-crypto.js";
|
|
14
14
|
import { LegacyExpressClient } from "./compat/express.js";
|
|
15
15
|
import { TcpRelayPool } from "./compat/tcp-relay-pool.js";
|
|
@@ -1649,7 +1649,8 @@ export class Peer {
|
|
|
1649
1649
|
const isSingleTransportKind = (this.#opts.bulkDataPacketId !== undefined && innerKind === this.#opts.bulkDataPacketId) ||
|
|
1650
1650
|
innerKind === PACKET_ID_FILE_SENDREQUEST ||
|
|
1651
1651
|
innerKind === PACKET_ID_FILE_CONTROL ||
|
|
1652
|
-
innerKind === PACKET_ID_FILE_DATA
|
|
1652
|
+
innerKind === PACKET_ID_FILE_DATA ||
|
|
1653
|
+
innerKind === PACKET_ID_FILE_FEC;
|
|
1653
1654
|
// Confirm the transport path + liveness for EVERY decrypted packet —
|
|
1654
1655
|
// INCLUDING a dual-send duplicate. Receiving over this transport proves
|
|
1655
1656
|
// it works, so this MUST run BEFORE the dedup drop below. Otherwise the
|
|
@@ -2922,7 +2923,7 @@ export class Peer {
|
|
|
2922
2923
|
// happens to finish before reordering bites. Pinning file transfer to a
|
|
2923
2924
|
// single transport keeps the chunks in send order. (Reliability across a
|
|
2924
2925
|
// lossy path still wants a reorder buffer + retransmit — tracked separately.)
|
|
2925
|
-
const isFileTransfer = kind === PACKET_ID_FILE_SENDREQUEST || kind === PACKET_ID_FILE_CONTROL || kind === PACKET_ID_FILE_DATA;
|
|
2926
|
+
const isFileTransfer = kind === PACKET_ID_FILE_SENDREQUEST || kind === PACKET_ID_FILE_CONTROL || kind === PACKET_ID_FILE_DATA || kind === PACKET_ID_FILE_FEC;
|
|
2926
2927
|
const isBulkData = (this.#opts.bulkDataPacketId !== undefined && kind === this.#opts.bulkDataPacketId) || isFileTransfer;
|
|
2927
2928
|
// File transfer rides the relay RELIABLY (never dropped under backpressure):
|
|
2928
2929
|
// a dropped IP/CCTV packet is fine (the stream moves on), but a dropped file
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@decentnetwork/peer",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.74",
|
|
4
4
|
"description": "Pure TypeScript port of Elastos Carrier (toxcore-derived) P2P messaging. DHT, onion routing, TCP relay, FlatBuffers app payloads, Express offline relay. Wire-compatible with iOS Beagle and the Carrier C SDK.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|