@peerbit/stream 5.0.17 → 5.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.d.ts +76 -2
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +400 -85
- package/dist/src/index.js.map +1 -1
- package/dist/src/routes.d.ts +31 -3
- package/dist/src/routes.d.ts.map +1 -1
- package/dist/src/routes.js.map +1 -1
- package/dist/src/rust-core.d.ts +561 -0
- package/dist/src/rust-core.d.ts.map +1 -0
- package/dist/src/rust-core.js +16 -0
- package/dist/src/rust-core.js.map +1 -0
- package/package.json +8 -11
- package/src/index.ts +552 -101
- package/src/routes.ts +43 -2
- package/src/rust-core.ts +696 -0
package/dist/src/index.js
CHANGED
|
@@ -17,11 +17,14 @@ import { computeSeekAckRouteUpdate, shouldAcknowledgeDataMessage, } from "./core
|
|
|
17
17
|
import { logger } from "./logger.js";
|
|
18
18
|
import { pushableLanes } from "./pushable-lanes.js";
|
|
19
19
|
import { MAX_ROUTE_DISTANCE, Routes } from "./routes.js";
|
|
20
|
+
import { resolveInjectedRustCore, } from "./rust-core.js";
|
|
20
21
|
import { BandwidthTracker } from "./stats.js";
|
|
21
22
|
import { waitForEvent } from "./wait-for-event.js";
|
|
22
23
|
export { logger };
|
|
23
24
|
const warn = logger.newScope("warn");
|
|
24
25
|
export { BandwidthTracker }; // might be useful for others
|
|
26
|
+
export { Routes, } from "./routes.js";
|
|
27
|
+
export { RUST_CORE_GLOBAL_KEY, } from "./rust-core.js";
|
|
25
28
|
const getErrorName = (e) => e?.name ?? e?.constructor?.name;
|
|
26
29
|
export const dontThrowIfDeliveryError = (e) => {
|
|
27
30
|
const errorName = getErrorName(e);
|
|
@@ -174,6 +177,7 @@ export class PeerStreams extends TypedEventEmitter {
|
|
|
174
177
|
seekedOnce;
|
|
175
178
|
usedBandWidthTracker;
|
|
176
179
|
outboundQueue;
|
|
180
|
+
lanesFactory;
|
|
177
181
|
// Unified outbound streams list (during grace may contain >1; after pruning length==1)
|
|
178
182
|
outboundStreams = [];
|
|
179
183
|
// Public debug exposure of current raw outbound streams (during grace may contain >1)
|
|
@@ -216,17 +220,28 @@ export class PeerStreams extends TypedEventEmitter {
|
|
|
216
220
|
const existing = this.outboundStreams.find((c) => c.raw === raw);
|
|
217
221
|
if (existing)
|
|
218
222
|
return existing;
|
|
219
|
-
const pushableInst =
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
223
|
+
const pushableInst = this.lanesFactory
|
|
224
|
+
? this.lanesFactory({
|
|
225
|
+
lanes: PRIORITY_LANES,
|
|
226
|
+
maxBufferedBytes: this.outboundQueue?.maxBufferedBytes,
|
|
227
|
+
onBufferSize: () => {
|
|
228
|
+
this.dispatchEvent(new CustomEvent("queue:outbound"));
|
|
229
|
+
},
|
|
230
|
+
onPush: (val) => {
|
|
231
|
+
candidate.bytesDelivered += val.byteLength;
|
|
232
|
+
},
|
|
233
|
+
})
|
|
234
|
+
: pushableLanes({
|
|
235
|
+
lanes: PRIORITY_LANES,
|
|
236
|
+
maxBufferedBytes: this.outboundQueue?.maxBufferedBytes,
|
|
237
|
+
overflow: "throw",
|
|
238
|
+
onBufferSize: () => {
|
|
239
|
+
this.dispatchEvent(new CustomEvent("queue:outbound"));
|
|
240
|
+
},
|
|
241
|
+
onPush: (val) => {
|
|
242
|
+
candidate.bytesDelivered += val.byteLength;
|
|
243
|
+
},
|
|
244
|
+
});
|
|
230
245
|
const candidate = {
|
|
231
246
|
raw,
|
|
232
247
|
pushable: pushableInst,
|
|
@@ -306,6 +321,7 @@ export class PeerStreams extends TypedEventEmitter {
|
|
|
306
321
|
this.usedBandWidthTracker = new BandwidthTracker(10);
|
|
307
322
|
this.usedBandWidthTracker.start();
|
|
308
323
|
this.outboundQueue = init.outboundQueue;
|
|
324
|
+
this.lanesFactory = init.lanesFactory;
|
|
309
325
|
}
|
|
310
326
|
/**
|
|
311
327
|
* Do we have a connection to read from?
|
|
@@ -370,6 +386,10 @@ export class PeerStreams extends TypedEventEmitter {
|
|
|
370
386
|
* Throws if there is no `stream` to write to available.
|
|
371
387
|
*/
|
|
372
388
|
write(data, priority) {
|
|
389
|
+
if (this.closed) {
|
|
390
|
+
logger.error(`Failed to send to stream ${this.peerId}: closed`);
|
|
391
|
+
throw new AbortError("Closed");
|
|
392
|
+
}
|
|
373
393
|
if (data.length > MAX_DATA_LENGTH_OUT) {
|
|
374
394
|
throw new Error(`Message too large (${data.length * 1e-6}) mb). Needs to be less than ${MAX_DATA_LENGTH_OUT * 1e-6} mb`);
|
|
375
395
|
}
|
|
@@ -442,11 +462,11 @@ export class PeerStreams extends TypedEventEmitter {
|
|
|
442
462
|
* in the shared `cleanup()` so nothing can dangle.
|
|
443
463
|
*/
|
|
444
464
|
async waitForWrite(bytes, priority = 0, signal) {
|
|
445
|
-
if (this.closed) {
|
|
446
|
-
logger.error(`Failed to send to stream ${this.peerId}: closed`);
|
|
447
|
-
return;
|
|
448
|
-
}
|
|
449
465
|
while (true) {
|
|
466
|
+
if (this.closed) {
|
|
467
|
+
logger.error(`Failed to send to stream ${this.peerId}: closed`);
|
|
468
|
+
throw new AbortError("Closed");
|
|
469
|
+
}
|
|
450
470
|
if (!this.isWritable) {
|
|
451
471
|
// Outbound stream negotiation can legitimately take several seconds in CI
|
|
452
472
|
// (identify/protocol discovery, resource contention, etc). Keep this fairly
|
|
@@ -484,6 +504,8 @@ export class PeerStreams extends TypedEventEmitter {
|
|
|
484
504
|
if (this.isWritable)
|
|
485
505
|
onOutbound();
|
|
486
506
|
});
|
|
507
|
+
// Re-enter through the lifecycle check before attempting the write.
|
|
508
|
+
continue;
|
|
487
509
|
}
|
|
488
510
|
try {
|
|
489
511
|
this.write(bytes, priority);
|
|
@@ -602,6 +624,34 @@ export class PeerStreams extends TypedEventEmitter {
|
|
|
602
624
|
}
|
|
603
625
|
this._pruneInboundInactive();
|
|
604
626
|
}
|
|
627
|
+
detachInboundStream(record, reason = new AbortError("Inbound stream ended"), options) {
|
|
628
|
+
const index = this.inboundStreams.indexOf(record);
|
|
629
|
+
if (index === -1)
|
|
630
|
+
return false;
|
|
631
|
+
this.inboundStreams.splice(index, 1);
|
|
632
|
+
try {
|
|
633
|
+
record.abortController.abort(reason);
|
|
634
|
+
}
|
|
635
|
+
catch { }
|
|
636
|
+
if (options?.closeRaw !== false) {
|
|
637
|
+
try {
|
|
638
|
+
record.raw.abort?.(reason);
|
|
639
|
+
}
|
|
640
|
+
catch { }
|
|
641
|
+
void Promise.resolve(record.raw.close?.()).catch(() => { });
|
|
642
|
+
}
|
|
643
|
+
if (this.rawInboundStream === record.raw) {
|
|
644
|
+
const next = this.inboundStreams[0];
|
|
645
|
+
this.rawInboundStream = next?.raw;
|
|
646
|
+
this.inboundStream = next?.iterable;
|
|
647
|
+
}
|
|
648
|
+
if (this.inboundStreams.length <= 1 && this._inboundPruneTimer) {
|
|
649
|
+
clearTimeout(this._inboundPruneTimer);
|
|
650
|
+
this._inboundPruneTimer = undefined;
|
|
651
|
+
}
|
|
652
|
+
this.dispatchEvent(new CustomEvent("stream:inbound"));
|
|
653
|
+
return true;
|
|
654
|
+
}
|
|
605
655
|
/**
|
|
606
656
|
* Attach a raw outbound stream and setup a write stream
|
|
607
657
|
*/
|
|
@@ -763,6 +813,10 @@ export class PeerStreams extends TypedEventEmitter {
|
|
|
763
813
|
this.inboundStreams = [];
|
|
764
814
|
}
|
|
765
815
|
}
|
|
816
|
+
const NATIVE_WIRE_RECORD_WORDS = 4;
|
|
817
|
+
const NATIVE_WIRE_FLAG_DECODE_OK = 0x01;
|
|
818
|
+
const NATIVE_WIRE_VERIFY_VERIFIED = 1;
|
|
819
|
+
const NATIVE_WIRE_VERIFY_UNSUPPORTED = 2;
|
|
766
820
|
const sharedRoutingByPrivateKey = new WeakMap();
|
|
767
821
|
export class DirectStream extends TypedEventEmitter {
|
|
768
822
|
components;
|
|
@@ -810,6 +864,32 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
810
864
|
totalOutboundQueueWaiters = new Set();
|
|
811
865
|
sharedRoutingKey;
|
|
812
866
|
sharedRoutingState;
|
|
867
|
+
nativeWire;
|
|
868
|
+
rustCore;
|
|
869
|
+
rustSeenCache;
|
|
870
|
+
pendingNativeWireFrames = [];
|
|
871
|
+
nativeWireFlushScheduled = false;
|
|
872
|
+
/**
|
|
873
|
+
* Always-on counters recording where per-frame wire work executed. Tests
|
|
874
|
+
* and benchmarks use them to prove the native path leaves no per-message
|
|
875
|
+
* JS decode/verify work on the hot path. `tsEnvelopeDecodes` counts the
|
|
876
|
+
* TS envelope object materializations (`Message.from`); on the native
|
|
877
|
+
* path that object only feeds the routing state machine and app-facing
|
|
878
|
+
* events, holding the payload as a zero-copy view, while decode
|
|
879
|
+
* validation and signature verification stay native.
|
|
880
|
+
*/
|
|
881
|
+
wireCounters = {
|
|
882
|
+
/** Frames decoded and signature-verified by the native wire module. */
|
|
883
|
+
nativeFrames: 0,
|
|
884
|
+
/** Frames where the native decode failed and the TS path took over. */
|
|
885
|
+
nativeFallbackFrames: 0,
|
|
886
|
+
/** Frames processed without a native wire module (pure TS path). */
|
|
887
|
+
tsFrames: 0,
|
|
888
|
+
/** TS-side signature verifications (not pre-seeded by native decode). */
|
|
889
|
+
tsSignatureVerifies: 0,
|
|
890
|
+
/** TS envelope object materializations (`Message.from`). */
|
|
891
|
+
tsEnvelopeDecodes: 0,
|
|
892
|
+
};
|
|
813
893
|
// for sequential creation of outbound streams
|
|
814
894
|
outboundInflightQueue;
|
|
815
895
|
seekTimeout;
|
|
@@ -820,7 +900,14 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
820
900
|
constructor(components, multicodecs, options) {
|
|
821
901
|
super();
|
|
822
902
|
this.components = components;
|
|
823
|
-
const { canRelayMessage = true, messageProcessingConcurrency = 10, maxInboundStreams, maxOutboundStreams, connectionManager, seekTimeout = SEEK_DELIVERY_TIMEOUT, routeMaxRetentionPeriod = ROUTE_MAX_RETANTION_PERIOD, routeCacheMaxFromEntries, routeCacheMaxTargetsPerFrom, routeCacheMaxRelaysPerTarget, sharedRouting = true, seenCacheMax = 1e6, seenCacheTtlMs = 10 * 60 * 1e3, inboundIdleTimeout, outboundQueue, } = options || {};
|
|
903
|
+
const { canRelayMessage = true, messageProcessingConcurrency = 10, maxInboundStreams, maxOutboundStreams, connectionManager, seekTimeout = SEEK_DELIVERY_TIMEOUT, routeMaxRetentionPeriod = ROUTE_MAX_RETANTION_PERIOD, routeCacheMaxFromEntries, routeCacheMaxTargetsPerFrom, routeCacheMaxRelaysPerTarget, sharedRouting = true, seenCacheMax = 1e6, seenCacheTtlMs = 10 * 60 * 1e3, inboundIdleTimeout, outboundQueue, nativeWire, rustCore, } = options || {};
|
|
904
|
+
this.rustCore =
|
|
905
|
+
rustCore === false ? undefined : (rustCore ?? resolveInjectedRustCore());
|
|
906
|
+
this.nativeWire = nativeWire ?? this.rustCore?.nativeWire;
|
|
907
|
+
this.rustSeenCache = this.rustCore?.createSeenCache({
|
|
908
|
+
max: Math.max(1, Math.floor(seenCacheMax)),
|
|
909
|
+
ttl: Math.max(1, Math.floor(seenCacheTtlMs)),
|
|
910
|
+
});
|
|
824
911
|
const signKey = getKeypairFromPrivateKey(components.privateKey);
|
|
825
912
|
this.seekTimeout = seekTimeout;
|
|
826
913
|
this.sign = (bytes) => signKey.sign(bytes, PreHash.SHA_256);
|
|
@@ -918,6 +1005,25 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
918
1005
|
})
|
|
919
1006
|
: undefined;
|
|
920
1007
|
}
|
|
1008
|
+
createRoutes(signal) {
|
|
1009
|
+
if (this.rustCore) {
|
|
1010
|
+
return this.rustCore.createRoutes({
|
|
1011
|
+
me: this.publicKeyHash,
|
|
1012
|
+
routeMaxRetentionPeriod: this.routeMaxRetentionPeriod,
|
|
1013
|
+
signal,
|
|
1014
|
+
maxFromEntries: this.routeCacheMaxFromEntries,
|
|
1015
|
+
maxTargetsPerFrom: this.routeCacheMaxTargetsPerFrom,
|
|
1016
|
+
maxRelaysPerTarget: this.routeCacheMaxRelaysPerTarget,
|
|
1017
|
+
});
|
|
1018
|
+
}
|
|
1019
|
+
return new Routes(this.publicKeyHash, {
|
|
1020
|
+
routeMaxRetentionPeriod: this.routeMaxRetentionPeriod,
|
|
1021
|
+
signal,
|
|
1022
|
+
maxFromEntries: this.routeCacheMaxFromEntries,
|
|
1023
|
+
maxTargetsPerFrom: this.routeCacheMaxTargetsPerFrom,
|
|
1024
|
+
maxRelaysPerTarget: this.routeCacheMaxRelaysPerTarget,
|
|
1025
|
+
});
|
|
1026
|
+
}
|
|
921
1027
|
pruneConnectionsToLimits() {
|
|
922
1028
|
if (this.pruneToLimitsInFlight) {
|
|
923
1029
|
return this.pruneToLimitsInFlight;
|
|
@@ -1019,13 +1125,7 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1019
1125
|
state = {
|
|
1020
1126
|
session: Date.now(),
|
|
1021
1127
|
controller,
|
|
1022
|
-
routes:
|
|
1023
|
-
routeMaxRetentionPeriod: this.routeMaxRetentionPeriod,
|
|
1024
|
-
signal: controller.signal,
|
|
1025
|
-
maxFromEntries: this.routeCacheMaxFromEntries,
|
|
1026
|
-
maxTargetsPerFrom: this.routeCacheMaxTargetsPerFrom,
|
|
1027
|
-
maxRelaysPerTarget: this.routeCacheMaxRelaysPerTarget,
|
|
1028
|
-
}),
|
|
1128
|
+
routes: this.createRoutes(controller.signal),
|
|
1029
1129
|
refs: 0,
|
|
1030
1130
|
};
|
|
1031
1131
|
sharedRoutingByPrivateKey.set(key, state);
|
|
@@ -1041,13 +1141,7 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1041
1141
|
}
|
|
1042
1142
|
else {
|
|
1043
1143
|
this.session = Date.now();
|
|
1044
|
-
this.routes =
|
|
1045
|
-
routeMaxRetentionPeriod: this.routeMaxRetentionPeriod,
|
|
1046
|
-
signal: this.closeController.signal,
|
|
1047
|
-
maxFromEntries: this.routeCacheMaxFromEntries,
|
|
1048
|
-
maxTargetsPerFrom: this.routeCacheMaxTargetsPerFrom,
|
|
1049
|
-
maxRelaysPerTarget: this.routeCacheMaxRelaysPerTarget,
|
|
1050
|
-
});
|
|
1144
|
+
this.routes = this.createRoutes(this.closeController.signal);
|
|
1051
1145
|
}
|
|
1052
1146
|
this.started = true;
|
|
1053
1147
|
this.stopping = false;
|
|
@@ -1188,6 +1282,11 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1188
1282
|
this.queue.clear();
|
|
1189
1283
|
this.peers.clear();
|
|
1190
1284
|
this.seenCache.clear();
|
|
1285
|
+
this.rustSeenCache?.clear();
|
|
1286
|
+
for (const pending of this.pendingNativeWireFrames) {
|
|
1287
|
+
pending.done.resolve();
|
|
1288
|
+
}
|
|
1289
|
+
this.pendingNativeWireFrames = [];
|
|
1191
1290
|
// When routing is shared across co-located protocols, only clear once the last
|
|
1192
1291
|
// instance stops. Otherwise we'd wipe routes still in use by other services.
|
|
1193
1292
|
if (!sharedState) {
|
|
@@ -1267,16 +1366,22 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1267
1366
|
return;
|
|
1268
1367
|
}
|
|
1269
1368
|
try {
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
this.
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1369
|
+
const streamSignal = anySignal([
|
|
1370
|
+
this.closeController.signal,
|
|
1371
|
+
...(opts?.signal ? [opts.signal] : []),
|
|
1372
|
+
]);
|
|
1373
|
+
try {
|
|
1374
|
+
stream = await connection.newStream(this.multicodecs, {
|
|
1375
|
+
// TODO this property seems necessary, together with waitFor isReadable when making sure two peers are conencted before talking.
|
|
1376
|
+
// research whether we can do without this so we can push data without beeing able to send
|
|
1377
|
+
// more info here https://github.com/libp2p/js-libp2p/issues/2321
|
|
1378
|
+
negotiateFully: true,
|
|
1379
|
+
signal: streamSignal,
|
|
1380
|
+
});
|
|
1381
|
+
}
|
|
1382
|
+
finally {
|
|
1383
|
+
streamSignal.clear?.();
|
|
1384
|
+
}
|
|
1280
1385
|
if (stream.protocol == null) {
|
|
1281
1386
|
stream.abort(new Error("Stream was not multiplexed"));
|
|
1282
1387
|
return;
|
|
@@ -1458,6 +1563,9 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1458
1563
|
protocol,
|
|
1459
1564
|
connId,
|
|
1460
1565
|
outboundQueue: this.outboundQueueOptions,
|
|
1566
|
+
lanesFactory: this.rustCore
|
|
1567
|
+
? (init) => this.rustCore.createLanes(init)
|
|
1568
|
+
: undefined,
|
|
1461
1569
|
});
|
|
1462
1570
|
this.peers.set(publicKeyHash, peerStreams);
|
|
1463
1571
|
this.updateSession(publicKey, -1);
|
|
@@ -1507,6 +1615,7 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1507
1615
|
* Responsible for processing each RPC message received by other peers.
|
|
1508
1616
|
*/
|
|
1509
1617
|
async processMessages(peerId, record, peerStreams) {
|
|
1618
|
+
let failed = false;
|
|
1510
1619
|
try {
|
|
1511
1620
|
for await (const data of record.iterable) {
|
|
1512
1621
|
const now = Date.now();
|
|
@@ -1516,6 +1625,7 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1516
1625
|
}
|
|
1517
1626
|
}
|
|
1518
1627
|
catch (err) {
|
|
1628
|
+
failed = true;
|
|
1519
1629
|
if (err?.code === "ERR_STREAM_RESET") {
|
|
1520
1630
|
// only send stream reset messages to info
|
|
1521
1631
|
logger("Failed processing messages to id: " +
|
|
@@ -1531,6 +1641,12 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1531
1641
|
}
|
|
1532
1642
|
this.onPeerDisconnected(peerStreams.peerId);
|
|
1533
1643
|
}
|
|
1644
|
+
finally {
|
|
1645
|
+
const removed = peerStreams.detachInboundStream(record, new AbortError("Inbound stream reader ended"), { closeRaw: failed });
|
|
1646
|
+
if (removed && !failed && !peerStreams.isReadable) {
|
|
1647
|
+
void this.onPeerDisconnected(peerStreams.peerId).catch(logError);
|
|
1648
|
+
}
|
|
1649
|
+
}
|
|
1534
1650
|
}
|
|
1535
1651
|
/**
|
|
1536
1652
|
* Handles an rpc request from a peer
|
|
@@ -1538,10 +1654,31 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1538
1654
|
async processRpc(from, peerStreams, message) {
|
|
1539
1655
|
// logger.trace("rpc from " + from + ", " + this.peerIdStr);
|
|
1540
1656
|
if (message.length > 0) {
|
|
1657
|
+
if (this.nativeWire) {
|
|
1658
|
+
// Batch frames arriving within the same tick so decode +
|
|
1659
|
+
// signature verification happen in one native call. The
|
|
1660
|
+
// returned promise still resolves only after this frame's
|
|
1661
|
+
// processMessage turn completes (the TS-path contract).
|
|
1662
|
+
const done = pDefer();
|
|
1663
|
+
this.pendingNativeWireFrames.push({
|
|
1664
|
+
from,
|
|
1665
|
+
peerStreams,
|
|
1666
|
+
bytes: message,
|
|
1667
|
+
done,
|
|
1668
|
+
});
|
|
1669
|
+
if (!this.nativeWireFlushScheduled) {
|
|
1670
|
+
this.nativeWireFlushScheduled = true;
|
|
1671
|
+
queueMicrotask(() => this.flushNativeWireFrames());
|
|
1672
|
+
}
|
|
1673
|
+
await done.promise;
|
|
1674
|
+
return true;
|
|
1675
|
+
}
|
|
1676
|
+
this.wireCounters.tsFrames++;
|
|
1541
1677
|
let decodedMessage;
|
|
1542
1678
|
let priority = 0;
|
|
1543
1679
|
try {
|
|
1544
1680
|
decodedMessage = Message.from(message);
|
|
1681
|
+
this.wireCounters.tsEnvelopeDecodes++;
|
|
1545
1682
|
priority = decodedMessage.header.priority ?? 0;
|
|
1546
1683
|
}
|
|
1547
1684
|
catch {
|
|
@@ -1562,7 +1699,115 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1562
1699
|
}
|
|
1563
1700
|
return true;
|
|
1564
1701
|
}
|
|
1565
|
-
|
|
1702
|
+
/**
|
|
1703
|
+
* Drain the frames collected by processRpc() through the native wire
|
|
1704
|
+
* module: one batched decode+verify call, then the usual per-message
|
|
1705
|
+
* pipeline with the TS message object pre-seeded with the native
|
|
1706
|
+
* verification result. Any native failure falls back to the pure TS
|
|
1707
|
+
* path for that frame, so semantics never diverge.
|
|
1708
|
+
*/
|
|
1709
|
+
flushNativeWireFrames() {
|
|
1710
|
+
this.nativeWireFlushScheduled = false;
|
|
1711
|
+
const pending = this.pendingNativeWireFrames;
|
|
1712
|
+
if (pending.length === 0) {
|
|
1713
|
+
return;
|
|
1714
|
+
}
|
|
1715
|
+
this.pendingNativeWireFrames = [];
|
|
1716
|
+
let records;
|
|
1717
|
+
if (this.nativeWire) {
|
|
1718
|
+
try {
|
|
1719
|
+
records = this.nativeWire.decodeAndVerifyBatch(pending.map((frame) => frame.bytes.subarray()), Date.now());
|
|
1720
|
+
if (records.length !== pending.length * NATIVE_WIRE_RECORD_WORDS) {
|
|
1721
|
+
records = undefined;
|
|
1722
|
+
}
|
|
1723
|
+
}
|
|
1724
|
+
catch (error) {
|
|
1725
|
+
logger("Native wire batch decode failed, falling back to TS path: " +
|
|
1726
|
+
error?.message);
|
|
1727
|
+
records = undefined;
|
|
1728
|
+
}
|
|
1729
|
+
}
|
|
1730
|
+
for (let i = 0; i < pending.length; i++) {
|
|
1731
|
+
const { from, peerStreams, bytes, done } = pending[i];
|
|
1732
|
+
const base = i * NATIVE_WIRE_RECORD_WORDS;
|
|
1733
|
+
const word0 = records ? records[base] : 0;
|
|
1734
|
+
const decodeOk = records != null && (word0 & NATIVE_WIRE_FLAG_DECODE_OK) !== 0;
|
|
1735
|
+
const verifyStatus = (word0 >>> 16) & 0xff;
|
|
1736
|
+
if (decodeOk && verifyStatus !== NATIVE_WIRE_VERIFY_UNSUPPORTED) {
|
|
1737
|
+
this.wireCounters.nativeFrames++;
|
|
1738
|
+
}
|
|
1739
|
+
else {
|
|
1740
|
+
this.wireCounters.nativeFallbackFrames++;
|
|
1741
|
+
}
|
|
1742
|
+
let decodedMessage;
|
|
1743
|
+
let priority = 0;
|
|
1744
|
+
try {
|
|
1745
|
+
decodedMessage = Message.from(bytes);
|
|
1746
|
+
this.wireCounters.tsEnvelopeDecodes++;
|
|
1747
|
+
priority = decodedMessage.header.priority ?? 0;
|
|
1748
|
+
}
|
|
1749
|
+
catch {
|
|
1750
|
+
// Best-effort peek, same as the TS path: processMessage()
|
|
1751
|
+
// performs the authoritative decode and logs invalid frames.
|
|
1752
|
+
}
|
|
1753
|
+
if (decodedMessage &&
|
|
1754
|
+
decodeOk &&
|
|
1755
|
+
verifyStatus !== NATIVE_WIRE_VERIFY_UNSUPPORTED) {
|
|
1756
|
+
// Seed the memoized verification result; verify(true) in the
|
|
1757
|
+
// dispatch path (verifyAndProcess) short-circuits on it.
|
|
1758
|
+
decodedMessage._verified =
|
|
1759
|
+
verifyStatus === NATIVE_WIRE_VERIFY_VERIFIED;
|
|
1760
|
+
}
|
|
1761
|
+
this.queue
|
|
1762
|
+
.add(async () => {
|
|
1763
|
+
try {
|
|
1764
|
+
await this.processMessage(from, peerStreams, bytes, decodedMessage);
|
|
1765
|
+
}
|
|
1766
|
+
catch (err) {
|
|
1767
|
+
logger.error(err);
|
|
1768
|
+
}
|
|
1769
|
+
}, { priority })
|
|
1770
|
+
.catch(logError)
|
|
1771
|
+
.finally(() => done.resolve());
|
|
1772
|
+
}
|
|
1773
|
+
}
|
|
1774
|
+
/**
|
|
1775
|
+
* Native decode+verify for an envelope frame that arrived outside the
|
|
1776
|
+
* inbound stream path (e.g. nested inside another protocol's payload).
|
|
1777
|
+
* Seeds the memoized verification result so `verify(true)`
|
|
1778
|
+
* short-circuits; on any native failure the TS verification path stays
|
|
1779
|
+
* authoritative.
|
|
1780
|
+
*/
|
|
1781
|
+
seedNativeWireVerification(message, frame) {
|
|
1782
|
+
if (!this.nativeWire || message._verified != null) {
|
|
1783
|
+
return;
|
|
1784
|
+
}
|
|
1785
|
+
try {
|
|
1786
|
+
const records = this.nativeWire.decodeAndVerifyBatch([frame instanceof Uint8Array ? frame : frame.subarray()], Date.now());
|
|
1787
|
+
if (records.length !== NATIVE_WIRE_RECORD_WORDS) {
|
|
1788
|
+
return;
|
|
1789
|
+
}
|
|
1790
|
+
const word0 = records[0];
|
|
1791
|
+
if ((word0 & NATIVE_WIRE_FLAG_DECODE_OK) === 0) {
|
|
1792
|
+
this.wireCounters.nativeFallbackFrames++;
|
|
1793
|
+
return;
|
|
1794
|
+
}
|
|
1795
|
+
const verifyStatus = (word0 >>> 16) & 0xff;
|
|
1796
|
+
if (verifyStatus === NATIVE_WIRE_VERIFY_UNSUPPORTED) {
|
|
1797
|
+
this.wireCounters.nativeFallbackFrames++;
|
|
1798
|
+
return;
|
|
1799
|
+
}
|
|
1800
|
+
this.wireCounters.nativeFrames++;
|
|
1801
|
+
message._verified = verifyStatus === NATIVE_WIRE_VERIFY_VERIFIED;
|
|
1802
|
+
}
|
|
1803
|
+
catch {
|
|
1804
|
+
// TS verification stays authoritative
|
|
1805
|
+
}
|
|
1806
|
+
}
|
|
1807
|
+
async modifySeenCache(message, getIdFn = getMsgId, seenKeyKind = 0) {
|
|
1808
|
+
if (this.rustSeenCache) {
|
|
1809
|
+
return this.rustSeenCache.modify(message instanceof Uint8Array ? message : message.subarray(), seenKeyKind);
|
|
1810
|
+
}
|
|
1566
1811
|
const msgId = await getIdFn(message);
|
|
1567
1812
|
const seen = this.seenCache.get(msgId);
|
|
1568
1813
|
this.seenCache.add(msgId, seen ? seen + 1 : 1);
|
|
@@ -1578,7 +1823,10 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1578
1823
|
// Ensure the message is valid before processing it
|
|
1579
1824
|
let message = decodedMessage;
|
|
1580
1825
|
try {
|
|
1581
|
-
message
|
|
1826
|
+
if (message == null) {
|
|
1827
|
+
message = Message.from(msg);
|
|
1828
|
+
this.wireCounters.tsEnvelopeDecodes++;
|
|
1829
|
+
}
|
|
1582
1830
|
}
|
|
1583
1831
|
catch (error) {
|
|
1584
1832
|
warn(error, "Failed to decode message frame from", from.hashcode());
|
|
@@ -1608,6 +1856,17 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1608
1856
|
}
|
|
1609
1857
|
}
|
|
1610
1858
|
shouldIgnore(message, seenBefore) {
|
|
1859
|
+
if (this.rustCore) {
|
|
1860
|
+
const mode = message.header.mode;
|
|
1861
|
+
return this.rustCore.decisions.shouldIgnoreData({
|
|
1862
|
+
seenBefore,
|
|
1863
|
+
acknowledgedMode: isAcknowledgedDeliveryMode(mode),
|
|
1864
|
+
redundancy: isAcknowledgedDeliveryMode(mode) ? mode.redundancy : 0,
|
|
1865
|
+
hops: getDeliveryHopTrace(mode),
|
|
1866
|
+
me: this.publicKeyHash,
|
|
1867
|
+
signedBySelf: message.header.signatures?.publicKeys.some((x) => x.equals(this.publicKey)) ?? false,
|
|
1868
|
+
});
|
|
1869
|
+
}
|
|
1611
1870
|
if (isAcknowledgedDeliveryMode(message.header.mode)) {
|
|
1612
1871
|
if (hasDeliveryHop(message.header.mode, this.publicKeyHash)) {
|
|
1613
1872
|
return true;
|
|
@@ -1670,6 +1929,9 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1670
1929
|
}
|
|
1671
1930
|
}
|
|
1672
1931
|
async verifyAndProcess(message) {
|
|
1932
|
+
if (message._verified == null) {
|
|
1933
|
+
this.wireCounters.tsSignatureVerifies++;
|
|
1934
|
+
}
|
|
1673
1935
|
const verified = await message.verify(true);
|
|
1674
1936
|
if (!verified) {
|
|
1675
1937
|
return false;
|
|
@@ -1689,11 +1951,18 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1689
1951
|
const isRecipient = message.header.mode instanceof AcknowledgeAnyWhere
|
|
1690
1952
|
? true
|
|
1691
1953
|
: message.header.mode.to.includes(this.publicKeyHash);
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1954
|
+
const shouldAcknowledge = this.rustCore
|
|
1955
|
+
? this.rustCore.decisions.shouldAcknowledge({
|
|
1956
|
+
isRecipient,
|
|
1957
|
+
seenBefore,
|
|
1958
|
+
redundancy: message.header.mode.redundancy,
|
|
1959
|
+
})
|
|
1960
|
+
: shouldAcknowledgeDataMessage({
|
|
1961
|
+
isRecipient,
|
|
1962
|
+
seenBefore,
|
|
1963
|
+
redundancy: message.header.mode.redundancy,
|
|
1964
|
+
});
|
|
1965
|
+
if (!shouldAcknowledge) {
|
|
1697
1966
|
return;
|
|
1698
1967
|
}
|
|
1699
1968
|
const signers = [...getDeliveryHopTrace(message.header.mode)];
|
|
@@ -1728,7 +1997,7 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1728
1997
|
async onAck(publicKey, peerStream, messageBytes, message) {
|
|
1729
1998
|
const seenBefore = await this.modifySeenCache(messageBytes instanceof Uint8Array
|
|
1730
1999
|
? messageBytes
|
|
1731
|
-
: messageBytes.subarray(), (bytes) => sha256Base64(bytes instanceof Uint8Array ? bytes : bytes.subarray()));
|
|
2000
|
+
: messageBytes.subarray(), (bytes) => sha256Base64(bytes instanceof Uint8Array ? bytes : bytes.subarray()), 1);
|
|
1732
2001
|
if (seenBefore > 0) {
|
|
1733
2002
|
logger.trace("Received message already seen of type: " + message.constructor.name);
|
|
1734
2003
|
return false;
|
|
@@ -1738,8 +2007,17 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1738
2007
|
return false;
|
|
1739
2008
|
}
|
|
1740
2009
|
const messageIdString = toBase64(message.messageIdToAcknowledge);
|
|
1741
|
-
|
|
1742
|
-
|
|
2010
|
+
let myIndex;
|
|
2011
|
+
let next;
|
|
2012
|
+
if (this.rustCore) {
|
|
2013
|
+
const hop = this.rustCore.decisions.ackNextHop(message.header.mode.trace, this.publicKeyHash);
|
|
2014
|
+
myIndex = hop.myIndex;
|
|
2015
|
+
next = hop.next;
|
|
2016
|
+
}
|
|
2017
|
+
else {
|
|
2018
|
+
myIndex = message.header.mode.trace.findIndex((x) => x === this.publicKeyHash);
|
|
2019
|
+
next = message.header.mode.trace[myIndex - 1];
|
|
2020
|
+
}
|
|
1743
2021
|
const nextStream = next ? this.peers.get(next) : undefined;
|
|
1744
2022
|
this._ackCallbacks
|
|
1745
2023
|
.get(messageIdString)
|
|
@@ -2141,15 +2419,25 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
2141
2419
|
if (message.header.mode instanceof AcknowledgeDelivery ||
|
|
2142
2420
|
message.header.mode instanceof AcknowledgeAnyWhere) {
|
|
2143
2421
|
const upstreamHash = messageFrom?.publicKey.hashcode();
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
|
|
2151
|
-
|
|
2152
|
-
|
|
2422
|
+
// Route "distance" is based on recipient-seen order (0 = fastest). This is relied upon by
|
|
2423
|
+
// `Routes.getFanout(...)` which uses `distance < redundancy` to select redundant next-hops.
|
|
2424
|
+
const routeUpdate = this.rustCore
|
|
2425
|
+
? {
|
|
2426
|
+
...this.rustCore.decisions.seekAckRouteUpdate({
|
|
2427
|
+
current: this.publicKeyHash,
|
|
2428
|
+
upstream: upstreamHash,
|
|
2429
|
+
downstream: messageThrough.publicKey.hashcode(),
|
|
2430
|
+
}),
|
|
2431
|
+
target: messageTargetHash,
|
|
2432
|
+
distance: seenCounter,
|
|
2433
|
+
}
|
|
2434
|
+
: computeSeekAckRouteUpdate({
|
|
2435
|
+
current: this.publicKeyHash,
|
|
2436
|
+
upstream: upstreamHash,
|
|
2437
|
+
downstream: messageThrough.publicKey.hashcode(),
|
|
2438
|
+
target: messageTargetHash,
|
|
2439
|
+
distance: seenCounter,
|
|
2440
|
+
});
|
|
2153
2441
|
this.addRouteConnection(routeUpdate.from, routeUpdate.neighbour, messageTarget, routeUpdate.distance, session, Number(ack.header.session));
|
|
2154
2442
|
}
|
|
2155
2443
|
if (messageToSet.has(messageTargetHash)) {
|
|
@@ -2251,14 +2539,26 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
2251
2539
|
if (!isRelayed &&
|
|
2252
2540
|
message.header.mode instanceof AcknowledgeDelivery &&
|
|
2253
2541
|
usedNeighbours.size < message.header.mode.redundancy) {
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2542
|
+
if (this.rustCore) {
|
|
2543
|
+
const probes = this.rustCore.decisions.selectRedundancyProbes([...this.peers.keys()], [...usedNeighbours], message.header.mode.redundancy);
|
|
2544
|
+
for (const neighbour of probes) {
|
|
2545
|
+
const stream = this.peers.get(neighbour);
|
|
2546
|
+
if (!stream)
|
|
2547
|
+
continue;
|
|
2548
|
+
usedNeighbours.add(neighbour);
|
|
2549
|
+
promises.push(this.waitForPeerWrite(stream, bytes, message.header.priority, signal));
|
|
2550
|
+
}
|
|
2551
|
+
}
|
|
2552
|
+
else {
|
|
2553
|
+
for (const [neighbour, stream] of this.peers) {
|
|
2554
|
+
if (usedNeighbours.size >= message.header.mode.redundancy) {
|
|
2555
|
+
break;
|
|
2556
|
+
}
|
|
2557
|
+
if (usedNeighbours.has(neighbour))
|
|
2558
|
+
continue;
|
|
2559
|
+
usedNeighbours.add(neighbour);
|
|
2560
|
+
promises.push(this.waitForPeerWrite(stream, bytes, message.header.priority, signal));
|
|
2257
2561
|
}
|
|
2258
|
-
if (usedNeighbours.has(neighbour))
|
|
2259
|
-
continue;
|
|
2260
|
-
usedNeighbours.add(neighbour);
|
|
2261
|
-
promises.push(this.waitForPeerWrite(stream, bytes, message.header.priority, signal));
|
|
2262
2562
|
}
|
|
2263
2563
|
}
|
|
2264
2564
|
await Promise.all(promises);
|
|
@@ -2272,17 +2572,22 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
2272
2572
|
if (isRelayed && message.header.mode instanceof SilentDelivery) {
|
|
2273
2573
|
const promises = [];
|
|
2274
2574
|
const originalTo = message.header.mode.to;
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
|
|
2279
|
-
|
|
2575
|
+
const recipients = this.rustCore
|
|
2576
|
+
? this.rustCore.decisions.filterSilentRelayRecipients(originalTo, this.publicKeyHash, from.hashcode(), [...this.peers.keys()], getDeliveryHopTrace(message.header.mode))
|
|
2577
|
+
: undefined;
|
|
2578
|
+
for (const recipient of recipients ?? originalTo) {
|
|
2579
|
+
if (!recipients) {
|
|
2580
|
+
if (recipient === this.publicKeyHash)
|
|
2581
|
+
continue;
|
|
2582
|
+
if (recipient === from.hashcode())
|
|
2583
|
+
continue; // never send back to previous hop
|
|
2584
|
+
if (hasDeliveryHop(message.header.mode, recipient)) {
|
|
2585
|
+
continue; // recipient already signed/seen this message
|
|
2586
|
+
}
|
|
2587
|
+
}
|
|
2280
2588
|
const stream = this.peers.get(recipient);
|
|
2281
2589
|
if (!stream)
|
|
2282
2590
|
continue;
|
|
2283
|
-
if (hasDeliveryHop(message.header.mode, recipient)) {
|
|
2284
|
-
continue; // recipient already signed/seen this message
|
|
2285
|
-
}
|
|
2286
2591
|
message.header.mode.to = [recipient];
|
|
2287
2592
|
promises.push(this.waitForPeerWrite(stream, message.bytes(), message.header.priority, signal));
|
|
2288
2593
|
}
|
|
@@ -2306,19 +2611,29 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
2306
2611
|
}
|
|
2307
2612
|
let sentOnce = false;
|
|
2308
2613
|
const promises = [];
|
|
2309
|
-
|
|
2310
|
-
const
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2614
|
+
if (this.rustCore) {
|
|
2615
|
+
const candidates = [...peers.values()];
|
|
2616
|
+
const kept = this.rustCore.decisions.filterFloodTargets(candidates.map((stream) => stream.publicKey.hashcode()), from.hashcode(), message.header.signatures?.publicKeys.map((x) => x.hashcode()) ?? [], getDeliveryHopTrace(message.header.mode));
|
|
2617
|
+
for (const index of kept) {
|
|
2618
|
+
sentOnce = true;
|
|
2619
|
+
promises.push(this.waitForPeerWrite(candidates[index], bytes, message.header.priority, signal));
|
|
2314
2620
|
}
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2621
|
+
}
|
|
2622
|
+
else {
|
|
2623
|
+
for (const stream of peers.values()) {
|
|
2624
|
+
const id = stream;
|
|
2625
|
+
// Dont sent back to the sender
|
|
2626
|
+
if (id.publicKey.equals(from)) {
|
|
2627
|
+
continue;
|
|
2628
|
+
}
|
|
2629
|
+
// Dont send message back to any peer already present in the path.
|
|
2630
|
+
if (message.header.signatures?.publicKeys.find((x) => x.equals(id.publicKey)) ||
|
|
2631
|
+
hasDeliveryHop(message.header.mode, id.publicKey.hashcode())) {
|
|
2632
|
+
continue;
|
|
2633
|
+
}
|
|
2634
|
+
sentOnce = true;
|
|
2635
|
+
promises.push(this.waitForPeerWrite(id, bytes, message.header.priority, signal));
|
|
2319
2636
|
}
|
|
2320
|
-
sentOnce = true;
|
|
2321
|
-
promises.push(this.waitForPeerWrite(id, bytes, message.header.priority, signal));
|
|
2322
2637
|
}
|
|
2323
2638
|
await Promise.all(promises);
|
|
2324
2639
|
startDeliveryTimeout?.();
|