@peerbit/stream 5.0.17 → 5.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.d.ts +76 -2
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +390 -81
- package/dist/src/index.js.map +1 -1
- package/dist/src/routes.d.ts +31 -3
- package/dist/src/routes.d.ts.map +1 -1
- package/dist/src/routes.js.map +1 -1
- package/dist/src/rust-core.d.ts +561 -0
- package/dist/src/rust-core.d.ts.map +1 -0
- package/dist/src/rust-core.js +16 -0
- package/dist/src/rust-core.js.map +1 -0
- package/package.json +5 -5
- package/src/index.ts +539 -96
- package/src/routes.ts +43 -2
- package/src/rust-core.ts +696 -0
package/dist/src/index.js
CHANGED
|
@@ -17,11 +17,14 @@ import { computeSeekAckRouteUpdate, shouldAcknowledgeDataMessage, } from "./core
|
|
|
17
17
|
import { logger } from "./logger.js";
|
|
18
18
|
import { pushableLanes } from "./pushable-lanes.js";
|
|
19
19
|
import { MAX_ROUTE_DISTANCE, Routes } from "./routes.js";
|
|
20
|
+
import { resolveInjectedRustCore, } from "./rust-core.js";
|
|
20
21
|
import { BandwidthTracker } from "./stats.js";
|
|
21
22
|
import { waitForEvent } from "./wait-for-event.js";
|
|
22
23
|
export { logger };
|
|
23
24
|
const warn = logger.newScope("warn");
|
|
24
25
|
export { BandwidthTracker }; // might be useful for others
|
|
26
|
+
export { Routes, } from "./routes.js";
|
|
27
|
+
export { RUST_CORE_GLOBAL_KEY, } from "./rust-core.js";
|
|
25
28
|
const getErrorName = (e) => e?.name ?? e?.constructor?.name;
|
|
26
29
|
export const dontThrowIfDeliveryError = (e) => {
|
|
27
30
|
const errorName = getErrorName(e);
|
|
@@ -174,6 +177,7 @@ export class PeerStreams extends TypedEventEmitter {
|
|
|
174
177
|
seekedOnce;
|
|
175
178
|
usedBandWidthTracker;
|
|
176
179
|
outboundQueue;
|
|
180
|
+
lanesFactory;
|
|
177
181
|
// Unified outbound streams list (during grace may contain >1; after pruning length==1)
|
|
178
182
|
outboundStreams = [];
|
|
179
183
|
// Public debug exposure of current raw outbound streams (during grace may contain >1)
|
|
@@ -216,17 +220,28 @@ export class PeerStreams extends TypedEventEmitter {
|
|
|
216
220
|
const existing = this.outboundStreams.find((c) => c.raw === raw);
|
|
217
221
|
if (existing)
|
|
218
222
|
return existing;
|
|
219
|
-
const pushableInst =
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
223
|
+
const pushableInst = this.lanesFactory
|
|
224
|
+
? this.lanesFactory({
|
|
225
|
+
lanes: PRIORITY_LANES,
|
|
226
|
+
maxBufferedBytes: this.outboundQueue?.maxBufferedBytes,
|
|
227
|
+
onBufferSize: () => {
|
|
228
|
+
this.dispatchEvent(new CustomEvent("queue:outbound"));
|
|
229
|
+
},
|
|
230
|
+
onPush: (val) => {
|
|
231
|
+
candidate.bytesDelivered += val.byteLength;
|
|
232
|
+
},
|
|
233
|
+
})
|
|
234
|
+
: pushableLanes({
|
|
235
|
+
lanes: PRIORITY_LANES,
|
|
236
|
+
maxBufferedBytes: this.outboundQueue?.maxBufferedBytes,
|
|
237
|
+
overflow: "throw",
|
|
238
|
+
onBufferSize: () => {
|
|
239
|
+
this.dispatchEvent(new CustomEvent("queue:outbound"));
|
|
240
|
+
},
|
|
241
|
+
onPush: (val) => {
|
|
242
|
+
candidate.bytesDelivered += val.byteLength;
|
|
243
|
+
},
|
|
244
|
+
});
|
|
230
245
|
const candidate = {
|
|
231
246
|
raw,
|
|
232
247
|
pushable: pushableInst,
|
|
@@ -306,6 +321,7 @@ export class PeerStreams extends TypedEventEmitter {
|
|
|
306
321
|
this.usedBandWidthTracker = new BandwidthTracker(10);
|
|
307
322
|
this.usedBandWidthTracker.start();
|
|
308
323
|
this.outboundQueue = init.outboundQueue;
|
|
324
|
+
this.lanesFactory = init.lanesFactory;
|
|
309
325
|
}
|
|
310
326
|
/**
|
|
311
327
|
* Do we have a connection to read from?
|
|
@@ -602,6 +618,34 @@ export class PeerStreams extends TypedEventEmitter {
|
|
|
602
618
|
}
|
|
603
619
|
this._pruneInboundInactive();
|
|
604
620
|
}
|
|
621
|
+
detachInboundStream(record, reason = new AbortError("Inbound stream ended"), options) {
|
|
622
|
+
const index = this.inboundStreams.indexOf(record);
|
|
623
|
+
if (index === -1)
|
|
624
|
+
return false;
|
|
625
|
+
this.inboundStreams.splice(index, 1);
|
|
626
|
+
try {
|
|
627
|
+
record.abortController.abort(reason);
|
|
628
|
+
}
|
|
629
|
+
catch { }
|
|
630
|
+
if (options?.closeRaw !== false) {
|
|
631
|
+
try {
|
|
632
|
+
record.raw.abort?.(reason);
|
|
633
|
+
}
|
|
634
|
+
catch { }
|
|
635
|
+
void Promise.resolve(record.raw.close?.()).catch(() => { });
|
|
636
|
+
}
|
|
637
|
+
if (this.rawInboundStream === record.raw) {
|
|
638
|
+
const next = this.inboundStreams[0];
|
|
639
|
+
this.rawInboundStream = next?.raw;
|
|
640
|
+
this.inboundStream = next?.iterable;
|
|
641
|
+
}
|
|
642
|
+
if (this.inboundStreams.length <= 1 && this._inboundPruneTimer) {
|
|
643
|
+
clearTimeout(this._inboundPruneTimer);
|
|
644
|
+
this._inboundPruneTimer = undefined;
|
|
645
|
+
}
|
|
646
|
+
this.dispatchEvent(new CustomEvent("stream:inbound"));
|
|
647
|
+
return true;
|
|
648
|
+
}
|
|
605
649
|
/**
|
|
606
650
|
* Attach a raw outbound stream and setup a write stream
|
|
607
651
|
*/
|
|
@@ -763,6 +807,10 @@ export class PeerStreams extends TypedEventEmitter {
|
|
|
763
807
|
this.inboundStreams = [];
|
|
764
808
|
}
|
|
765
809
|
}
|
|
810
|
+
const NATIVE_WIRE_RECORD_WORDS = 4;
|
|
811
|
+
const NATIVE_WIRE_FLAG_DECODE_OK = 0x01;
|
|
812
|
+
const NATIVE_WIRE_VERIFY_VERIFIED = 1;
|
|
813
|
+
const NATIVE_WIRE_VERIFY_UNSUPPORTED = 2;
|
|
766
814
|
const sharedRoutingByPrivateKey = new WeakMap();
|
|
767
815
|
export class DirectStream extends TypedEventEmitter {
|
|
768
816
|
components;
|
|
@@ -810,6 +858,32 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
810
858
|
totalOutboundQueueWaiters = new Set();
|
|
811
859
|
sharedRoutingKey;
|
|
812
860
|
sharedRoutingState;
|
|
861
|
+
nativeWire;
|
|
862
|
+
rustCore;
|
|
863
|
+
rustSeenCache;
|
|
864
|
+
pendingNativeWireFrames = [];
|
|
865
|
+
nativeWireFlushScheduled = false;
|
|
866
|
+
/**
|
|
867
|
+
* Always-on counters recording where per-frame wire work executed. Tests
|
|
868
|
+
* and benchmarks use them to prove the native path leaves no per-message
|
|
869
|
+
* JS decode/verify work on the hot path. `tsEnvelopeDecodes` counts the
|
|
870
|
+
* TS envelope object materializations (`Message.from`); on the native
|
|
871
|
+
* path that object only feeds the routing state machine and app-facing
|
|
872
|
+
* events, holding the payload as a zero-copy view, while decode
|
|
873
|
+
* validation and signature verification stay native.
|
|
874
|
+
*/
|
|
875
|
+
wireCounters = {
|
|
876
|
+
/** Frames decoded and signature-verified by the native wire module. */
|
|
877
|
+
nativeFrames: 0,
|
|
878
|
+
/** Frames where the native decode failed and the TS path took over. */
|
|
879
|
+
nativeFallbackFrames: 0,
|
|
880
|
+
/** Frames processed without a native wire module (pure TS path). */
|
|
881
|
+
tsFrames: 0,
|
|
882
|
+
/** TS-side signature verifications (not pre-seeded by native decode). */
|
|
883
|
+
tsSignatureVerifies: 0,
|
|
884
|
+
/** TS envelope object materializations (`Message.from`). */
|
|
885
|
+
tsEnvelopeDecodes: 0,
|
|
886
|
+
};
|
|
813
887
|
// for sequential creation of outbound streams
|
|
814
888
|
outboundInflightQueue;
|
|
815
889
|
seekTimeout;
|
|
@@ -820,7 +894,14 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
820
894
|
constructor(components, multicodecs, options) {
|
|
821
895
|
super();
|
|
822
896
|
this.components = components;
|
|
823
|
-
const { canRelayMessage = true, messageProcessingConcurrency = 10, maxInboundStreams, maxOutboundStreams, connectionManager, seekTimeout = SEEK_DELIVERY_TIMEOUT, routeMaxRetentionPeriod = ROUTE_MAX_RETANTION_PERIOD, routeCacheMaxFromEntries, routeCacheMaxTargetsPerFrom, routeCacheMaxRelaysPerTarget, sharedRouting = true, seenCacheMax = 1e6, seenCacheTtlMs = 10 * 60 * 1e3, inboundIdleTimeout, outboundQueue, } = options || {};
|
|
897
|
+
const { canRelayMessage = true, messageProcessingConcurrency = 10, maxInboundStreams, maxOutboundStreams, connectionManager, seekTimeout = SEEK_DELIVERY_TIMEOUT, routeMaxRetentionPeriod = ROUTE_MAX_RETANTION_PERIOD, routeCacheMaxFromEntries, routeCacheMaxTargetsPerFrom, routeCacheMaxRelaysPerTarget, sharedRouting = true, seenCacheMax = 1e6, seenCacheTtlMs = 10 * 60 * 1e3, inboundIdleTimeout, outboundQueue, nativeWire, rustCore, } = options || {};
|
|
898
|
+
this.rustCore =
|
|
899
|
+
rustCore === false ? undefined : (rustCore ?? resolveInjectedRustCore());
|
|
900
|
+
this.nativeWire = nativeWire ?? this.rustCore?.nativeWire;
|
|
901
|
+
this.rustSeenCache = this.rustCore?.createSeenCache({
|
|
902
|
+
max: Math.max(1, Math.floor(seenCacheMax)),
|
|
903
|
+
ttl: Math.max(1, Math.floor(seenCacheTtlMs)),
|
|
904
|
+
});
|
|
824
905
|
const signKey = getKeypairFromPrivateKey(components.privateKey);
|
|
825
906
|
this.seekTimeout = seekTimeout;
|
|
826
907
|
this.sign = (bytes) => signKey.sign(bytes, PreHash.SHA_256);
|
|
@@ -918,6 +999,25 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
918
999
|
})
|
|
919
1000
|
: undefined;
|
|
920
1001
|
}
|
|
1002
|
+
createRoutes(signal) {
|
|
1003
|
+
if (this.rustCore) {
|
|
1004
|
+
return this.rustCore.createRoutes({
|
|
1005
|
+
me: this.publicKeyHash,
|
|
1006
|
+
routeMaxRetentionPeriod: this.routeMaxRetentionPeriod,
|
|
1007
|
+
signal,
|
|
1008
|
+
maxFromEntries: this.routeCacheMaxFromEntries,
|
|
1009
|
+
maxTargetsPerFrom: this.routeCacheMaxTargetsPerFrom,
|
|
1010
|
+
maxRelaysPerTarget: this.routeCacheMaxRelaysPerTarget,
|
|
1011
|
+
});
|
|
1012
|
+
}
|
|
1013
|
+
return new Routes(this.publicKeyHash, {
|
|
1014
|
+
routeMaxRetentionPeriod: this.routeMaxRetentionPeriod,
|
|
1015
|
+
signal,
|
|
1016
|
+
maxFromEntries: this.routeCacheMaxFromEntries,
|
|
1017
|
+
maxTargetsPerFrom: this.routeCacheMaxTargetsPerFrom,
|
|
1018
|
+
maxRelaysPerTarget: this.routeCacheMaxRelaysPerTarget,
|
|
1019
|
+
});
|
|
1020
|
+
}
|
|
921
1021
|
pruneConnectionsToLimits() {
|
|
922
1022
|
if (this.pruneToLimitsInFlight) {
|
|
923
1023
|
return this.pruneToLimitsInFlight;
|
|
@@ -1019,13 +1119,7 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1019
1119
|
state = {
|
|
1020
1120
|
session: Date.now(),
|
|
1021
1121
|
controller,
|
|
1022
|
-
routes:
|
|
1023
|
-
routeMaxRetentionPeriod: this.routeMaxRetentionPeriod,
|
|
1024
|
-
signal: controller.signal,
|
|
1025
|
-
maxFromEntries: this.routeCacheMaxFromEntries,
|
|
1026
|
-
maxTargetsPerFrom: this.routeCacheMaxTargetsPerFrom,
|
|
1027
|
-
maxRelaysPerTarget: this.routeCacheMaxRelaysPerTarget,
|
|
1028
|
-
}),
|
|
1122
|
+
routes: this.createRoutes(controller.signal),
|
|
1029
1123
|
refs: 0,
|
|
1030
1124
|
};
|
|
1031
1125
|
sharedRoutingByPrivateKey.set(key, state);
|
|
@@ -1041,13 +1135,7 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1041
1135
|
}
|
|
1042
1136
|
else {
|
|
1043
1137
|
this.session = Date.now();
|
|
1044
|
-
this.routes =
|
|
1045
|
-
routeMaxRetentionPeriod: this.routeMaxRetentionPeriod,
|
|
1046
|
-
signal: this.closeController.signal,
|
|
1047
|
-
maxFromEntries: this.routeCacheMaxFromEntries,
|
|
1048
|
-
maxTargetsPerFrom: this.routeCacheMaxTargetsPerFrom,
|
|
1049
|
-
maxRelaysPerTarget: this.routeCacheMaxRelaysPerTarget,
|
|
1050
|
-
});
|
|
1138
|
+
this.routes = this.createRoutes(this.closeController.signal);
|
|
1051
1139
|
}
|
|
1052
1140
|
this.started = true;
|
|
1053
1141
|
this.stopping = false;
|
|
@@ -1188,6 +1276,11 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1188
1276
|
this.queue.clear();
|
|
1189
1277
|
this.peers.clear();
|
|
1190
1278
|
this.seenCache.clear();
|
|
1279
|
+
this.rustSeenCache?.clear();
|
|
1280
|
+
for (const pending of this.pendingNativeWireFrames) {
|
|
1281
|
+
pending.done.resolve();
|
|
1282
|
+
}
|
|
1283
|
+
this.pendingNativeWireFrames = [];
|
|
1191
1284
|
// When routing is shared across co-located protocols, only clear once the last
|
|
1192
1285
|
// instance stops. Otherwise we'd wipe routes still in use by other services.
|
|
1193
1286
|
if (!sharedState) {
|
|
@@ -1267,16 +1360,22 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1267
1360
|
return;
|
|
1268
1361
|
}
|
|
1269
1362
|
try {
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
this.
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1363
|
+
const streamSignal = anySignal([
|
|
1364
|
+
this.closeController.signal,
|
|
1365
|
+
...(opts?.signal ? [opts.signal] : []),
|
|
1366
|
+
]);
|
|
1367
|
+
try {
|
|
1368
|
+
stream = await connection.newStream(this.multicodecs, {
|
|
1369
|
+
// TODO this property seems necessary, together with waitFor isReadable when making sure two peers are conencted before talking.
|
|
1370
|
+
// research whether we can do without this so we can push data without beeing able to send
|
|
1371
|
+
// more info here https://github.com/libp2p/js-libp2p/issues/2321
|
|
1372
|
+
negotiateFully: true,
|
|
1373
|
+
signal: streamSignal,
|
|
1374
|
+
});
|
|
1375
|
+
}
|
|
1376
|
+
finally {
|
|
1377
|
+
streamSignal.clear?.();
|
|
1378
|
+
}
|
|
1280
1379
|
if (stream.protocol == null) {
|
|
1281
1380
|
stream.abort(new Error("Stream was not multiplexed"));
|
|
1282
1381
|
return;
|
|
@@ -1458,6 +1557,9 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1458
1557
|
protocol,
|
|
1459
1558
|
connId,
|
|
1460
1559
|
outboundQueue: this.outboundQueueOptions,
|
|
1560
|
+
lanesFactory: this.rustCore
|
|
1561
|
+
? (init) => this.rustCore.createLanes(init)
|
|
1562
|
+
: undefined,
|
|
1461
1563
|
});
|
|
1462
1564
|
this.peers.set(publicKeyHash, peerStreams);
|
|
1463
1565
|
this.updateSession(publicKey, -1);
|
|
@@ -1507,6 +1609,7 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1507
1609
|
* Responsible for processing each RPC message received by other peers.
|
|
1508
1610
|
*/
|
|
1509
1611
|
async processMessages(peerId, record, peerStreams) {
|
|
1612
|
+
let failed = false;
|
|
1510
1613
|
try {
|
|
1511
1614
|
for await (const data of record.iterable) {
|
|
1512
1615
|
const now = Date.now();
|
|
@@ -1516,6 +1619,7 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1516
1619
|
}
|
|
1517
1620
|
}
|
|
1518
1621
|
catch (err) {
|
|
1622
|
+
failed = true;
|
|
1519
1623
|
if (err?.code === "ERR_STREAM_RESET") {
|
|
1520
1624
|
// only send stream reset messages to info
|
|
1521
1625
|
logger("Failed processing messages to id: " +
|
|
@@ -1531,6 +1635,12 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1531
1635
|
}
|
|
1532
1636
|
this.onPeerDisconnected(peerStreams.peerId);
|
|
1533
1637
|
}
|
|
1638
|
+
finally {
|
|
1639
|
+
const removed = peerStreams.detachInboundStream(record, new AbortError("Inbound stream reader ended"), { closeRaw: failed });
|
|
1640
|
+
if (removed && !failed && !peerStreams.isReadable) {
|
|
1641
|
+
void this.onPeerDisconnected(peerStreams.peerId).catch(logError);
|
|
1642
|
+
}
|
|
1643
|
+
}
|
|
1534
1644
|
}
|
|
1535
1645
|
/**
|
|
1536
1646
|
* Handles an rpc request from a peer
|
|
@@ -1538,10 +1648,31 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1538
1648
|
async processRpc(from, peerStreams, message) {
|
|
1539
1649
|
// logger.trace("rpc from " + from + ", " + this.peerIdStr);
|
|
1540
1650
|
if (message.length > 0) {
|
|
1651
|
+
if (this.nativeWire) {
|
|
1652
|
+
// Batch frames arriving within the same tick so decode +
|
|
1653
|
+
// signature verification happen in one native call. The
|
|
1654
|
+
// returned promise still resolves only after this frame's
|
|
1655
|
+
// processMessage turn completes (the TS-path contract).
|
|
1656
|
+
const done = pDefer();
|
|
1657
|
+
this.pendingNativeWireFrames.push({
|
|
1658
|
+
from,
|
|
1659
|
+
peerStreams,
|
|
1660
|
+
bytes: message,
|
|
1661
|
+
done,
|
|
1662
|
+
});
|
|
1663
|
+
if (!this.nativeWireFlushScheduled) {
|
|
1664
|
+
this.nativeWireFlushScheduled = true;
|
|
1665
|
+
queueMicrotask(() => this.flushNativeWireFrames());
|
|
1666
|
+
}
|
|
1667
|
+
await done.promise;
|
|
1668
|
+
return true;
|
|
1669
|
+
}
|
|
1670
|
+
this.wireCounters.tsFrames++;
|
|
1541
1671
|
let decodedMessage;
|
|
1542
1672
|
let priority = 0;
|
|
1543
1673
|
try {
|
|
1544
1674
|
decodedMessage = Message.from(message);
|
|
1675
|
+
this.wireCounters.tsEnvelopeDecodes++;
|
|
1545
1676
|
priority = decodedMessage.header.priority ?? 0;
|
|
1546
1677
|
}
|
|
1547
1678
|
catch {
|
|
@@ -1562,7 +1693,115 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1562
1693
|
}
|
|
1563
1694
|
return true;
|
|
1564
1695
|
}
|
|
1565
|
-
|
|
1696
|
+
/**
|
|
1697
|
+
* Drain the frames collected by processRpc() through the native wire
|
|
1698
|
+
* module: one batched decode+verify call, then the usual per-message
|
|
1699
|
+
* pipeline with the TS message object pre-seeded with the native
|
|
1700
|
+
* verification result. Any native failure falls back to the pure TS
|
|
1701
|
+
* path for that frame, so semantics never diverge.
|
|
1702
|
+
*/
|
|
1703
|
+
flushNativeWireFrames() {
|
|
1704
|
+
this.nativeWireFlushScheduled = false;
|
|
1705
|
+
const pending = this.pendingNativeWireFrames;
|
|
1706
|
+
if (pending.length === 0) {
|
|
1707
|
+
return;
|
|
1708
|
+
}
|
|
1709
|
+
this.pendingNativeWireFrames = [];
|
|
1710
|
+
let records;
|
|
1711
|
+
if (this.nativeWire) {
|
|
1712
|
+
try {
|
|
1713
|
+
records = this.nativeWire.decodeAndVerifyBatch(pending.map((frame) => frame.bytes.subarray()), Date.now());
|
|
1714
|
+
if (records.length !== pending.length * NATIVE_WIRE_RECORD_WORDS) {
|
|
1715
|
+
records = undefined;
|
|
1716
|
+
}
|
|
1717
|
+
}
|
|
1718
|
+
catch (error) {
|
|
1719
|
+
logger("Native wire batch decode failed, falling back to TS path: " +
|
|
1720
|
+
error?.message);
|
|
1721
|
+
records = undefined;
|
|
1722
|
+
}
|
|
1723
|
+
}
|
|
1724
|
+
for (let i = 0; i < pending.length; i++) {
|
|
1725
|
+
const { from, peerStreams, bytes, done } = pending[i];
|
|
1726
|
+
const base = i * NATIVE_WIRE_RECORD_WORDS;
|
|
1727
|
+
const word0 = records ? records[base] : 0;
|
|
1728
|
+
const decodeOk = records != null && (word0 & NATIVE_WIRE_FLAG_DECODE_OK) !== 0;
|
|
1729
|
+
const verifyStatus = (word0 >>> 16) & 0xff;
|
|
1730
|
+
if (decodeOk && verifyStatus !== NATIVE_WIRE_VERIFY_UNSUPPORTED) {
|
|
1731
|
+
this.wireCounters.nativeFrames++;
|
|
1732
|
+
}
|
|
1733
|
+
else {
|
|
1734
|
+
this.wireCounters.nativeFallbackFrames++;
|
|
1735
|
+
}
|
|
1736
|
+
let decodedMessage;
|
|
1737
|
+
let priority = 0;
|
|
1738
|
+
try {
|
|
1739
|
+
decodedMessage = Message.from(bytes);
|
|
1740
|
+
this.wireCounters.tsEnvelopeDecodes++;
|
|
1741
|
+
priority = decodedMessage.header.priority ?? 0;
|
|
1742
|
+
}
|
|
1743
|
+
catch {
|
|
1744
|
+
// Best-effort peek, same as the TS path: processMessage()
|
|
1745
|
+
// performs the authoritative decode and logs invalid frames.
|
|
1746
|
+
}
|
|
1747
|
+
if (decodedMessage &&
|
|
1748
|
+
decodeOk &&
|
|
1749
|
+
verifyStatus !== NATIVE_WIRE_VERIFY_UNSUPPORTED) {
|
|
1750
|
+
// Seed the memoized verification result; verify(true) in the
|
|
1751
|
+
// dispatch path (verifyAndProcess) short-circuits on it.
|
|
1752
|
+
decodedMessage._verified =
|
|
1753
|
+
verifyStatus === NATIVE_WIRE_VERIFY_VERIFIED;
|
|
1754
|
+
}
|
|
1755
|
+
this.queue
|
|
1756
|
+
.add(async () => {
|
|
1757
|
+
try {
|
|
1758
|
+
await this.processMessage(from, peerStreams, bytes, decodedMessage);
|
|
1759
|
+
}
|
|
1760
|
+
catch (err) {
|
|
1761
|
+
logger.error(err);
|
|
1762
|
+
}
|
|
1763
|
+
}, { priority })
|
|
1764
|
+
.catch(logError)
|
|
1765
|
+
.finally(() => done.resolve());
|
|
1766
|
+
}
|
|
1767
|
+
}
|
|
1768
|
+
/**
|
|
1769
|
+
* Native decode+verify for an envelope frame that arrived outside the
|
|
1770
|
+
* inbound stream path (e.g. nested inside another protocol's payload).
|
|
1771
|
+
* Seeds the memoized verification result so `verify(true)`
|
|
1772
|
+
* short-circuits; on any native failure the TS verification path stays
|
|
1773
|
+
* authoritative.
|
|
1774
|
+
*/
|
|
1775
|
+
seedNativeWireVerification(message, frame) {
|
|
1776
|
+
if (!this.nativeWire || message._verified != null) {
|
|
1777
|
+
return;
|
|
1778
|
+
}
|
|
1779
|
+
try {
|
|
1780
|
+
const records = this.nativeWire.decodeAndVerifyBatch([frame instanceof Uint8Array ? frame : frame.subarray()], Date.now());
|
|
1781
|
+
if (records.length !== NATIVE_WIRE_RECORD_WORDS) {
|
|
1782
|
+
return;
|
|
1783
|
+
}
|
|
1784
|
+
const word0 = records[0];
|
|
1785
|
+
if ((word0 & NATIVE_WIRE_FLAG_DECODE_OK) === 0) {
|
|
1786
|
+
this.wireCounters.nativeFallbackFrames++;
|
|
1787
|
+
return;
|
|
1788
|
+
}
|
|
1789
|
+
const verifyStatus = (word0 >>> 16) & 0xff;
|
|
1790
|
+
if (verifyStatus === NATIVE_WIRE_VERIFY_UNSUPPORTED) {
|
|
1791
|
+
this.wireCounters.nativeFallbackFrames++;
|
|
1792
|
+
return;
|
|
1793
|
+
}
|
|
1794
|
+
this.wireCounters.nativeFrames++;
|
|
1795
|
+
message._verified = verifyStatus === NATIVE_WIRE_VERIFY_VERIFIED;
|
|
1796
|
+
}
|
|
1797
|
+
catch {
|
|
1798
|
+
// TS verification stays authoritative
|
|
1799
|
+
}
|
|
1800
|
+
}
|
|
1801
|
+
async modifySeenCache(message, getIdFn = getMsgId, seenKeyKind = 0) {
|
|
1802
|
+
if (this.rustSeenCache) {
|
|
1803
|
+
return this.rustSeenCache.modify(message instanceof Uint8Array ? message : message.subarray(), seenKeyKind);
|
|
1804
|
+
}
|
|
1566
1805
|
const msgId = await getIdFn(message);
|
|
1567
1806
|
const seen = this.seenCache.get(msgId);
|
|
1568
1807
|
this.seenCache.add(msgId, seen ? seen + 1 : 1);
|
|
@@ -1578,7 +1817,10 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1578
1817
|
// Ensure the message is valid before processing it
|
|
1579
1818
|
let message = decodedMessage;
|
|
1580
1819
|
try {
|
|
1581
|
-
message
|
|
1820
|
+
if (message == null) {
|
|
1821
|
+
message = Message.from(msg);
|
|
1822
|
+
this.wireCounters.tsEnvelopeDecodes++;
|
|
1823
|
+
}
|
|
1582
1824
|
}
|
|
1583
1825
|
catch (error) {
|
|
1584
1826
|
warn(error, "Failed to decode message frame from", from.hashcode());
|
|
@@ -1608,6 +1850,17 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1608
1850
|
}
|
|
1609
1851
|
}
|
|
1610
1852
|
shouldIgnore(message, seenBefore) {
|
|
1853
|
+
if (this.rustCore) {
|
|
1854
|
+
const mode = message.header.mode;
|
|
1855
|
+
return this.rustCore.decisions.shouldIgnoreData({
|
|
1856
|
+
seenBefore,
|
|
1857
|
+
acknowledgedMode: isAcknowledgedDeliveryMode(mode),
|
|
1858
|
+
redundancy: isAcknowledgedDeliveryMode(mode) ? mode.redundancy : 0,
|
|
1859
|
+
hops: getDeliveryHopTrace(mode),
|
|
1860
|
+
me: this.publicKeyHash,
|
|
1861
|
+
signedBySelf: message.header.signatures?.publicKeys.some((x) => x.equals(this.publicKey)) ?? false,
|
|
1862
|
+
});
|
|
1863
|
+
}
|
|
1611
1864
|
if (isAcknowledgedDeliveryMode(message.header.mode)) {
|
|
1612
1865
|
if (hasDeliveryHop(message.header.mode, this.publicKeyHash)) {
|
|
1613
1866
|
return true;
|
|
@@ -1670,6 +1923,9 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1670
1923
|
}
|
|
1671
1924
|
}
|
|
1672
1925
|
async verifyAndProcess(message) {
|
|
1926
|
+
if (message._verified == null) {
|
|
1927
|
+
this.wireCounters.tsSignatureVerifies++;
|
|
1928
|
+
}
|
|
1673
1929
|
const verified = await message.verify(true);
|
|
1674
1930
|
if (!verified) {
|
|
1675
1931
|
return false;
|
|
@@ -1689,11 +1945,18 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1689
1945
|
const isRecipient = message.header.mode instanceof AcknowledgeAnyWhere
|
|
1690
1946
|
? true
|
|
1691
1947
|
: message.header.mode.to.includes(this.publicKeyHash);
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1948
|
+
const shouldAcknowledge = this.rustCore
|
|
1949
|
+
? this.rustCore.decisions.shouldAcknowledge({
|
|
1950
|
+
isRecipient,
|
|
1951
|
+
seenBefore,
|
|
1952
|
+
redundancy: message.header.mode.redundancy,
|
|
1953
|
+
})
|
|
1954
|
+
: shouldAcknowledgeDataMessage({
|
|
1955
|
+
isRecipient,
|
|
1956
|
+
seenBefore,
|
|
1957
|
+
redundancy: message.header.mode.redundancy,
|
|
1958
|
+
});
|
|
1959
|
+
if (!shouldAcknowledge) {
|
|
1697
1960
|
return;
|
|
1698
1961
|
}
|
|
1699
1962
|
const signers = [...getDeliveryHopTrace(message.header.mode)];
|
|
@@ -1728,7 +1991,7 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1728
1991
|
async onAck(publicKey, peerStream, messageBytes, message) {
|
|
1729
1992
|
const seenBefore = await this.modifySeenCache(messageBytes instanceof Uint8Array
|
|
1730
1993
|
? messageBytes
|
|
1731
|
-
: messageBytes.subarray(), (bytes) => sha256Base64(bytes instanceof Uint8Array ? bytes : bytes.subarray()));
|
|
1994
|
+
: messageBytes.subarray(), (bytes) => sha256Base64(bytes instanceof Uint8Array ? bytes : bytes.subarray()), 1);
|
|
1732
1995
|
if (seenBefore > 0) {
|
|
1733
1996
|
logger.trace("Received message already seen of type: " + message.constructor.name);
|
|
1734
1997
|
return false;
|
|
@@ -1738,8 +2001,17 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
1738
2001
|
return false;
|
|
1739
2002
|
}
|
|
1740
2003
|
const messageIdString = toBase64(message.messageIdToAcknowledge);
|
|
1741
|
-
|
|
1742
|
-
|
|
2004
|
+
let myIndex;
|
|
2005
|
+
let next;
|
|
2006
|
+
if (this.rustCore) {
|
|
2007
|
+
const hop = this.rustCore.decisions.ackNextHop(message.header.mode.trace, this.publicKeyHash);
|
|
2008
|
+
myIndex = hop.myIndex;
|
|
2009
|
+
next = hop.next;
|
|
2010
|
+
}
|
|
2011
|
+
else {
|
|
2012
|
+
myIndex = message.header.mode.trace.findIndex((x) => x === this.publicKeyHash);
|
|
2013
|
+
next = message.header.mode.trace[myIndex - 1];
|
|
2014
|
+
}
|
|
1743
2015
|
const nextStream = next ? this.peers.get(next) : undefined;
|
|
1744
2016
|
this._ackCallbacks
|
|
1745
2017
|
.get(messageIdString)
|
|
@@ -2141,15 +2413,25 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
2141
2413
|
if (message.header.mode instanceof AcknowledgeDelivery ||
|
|
2142
2414
|
message.header.mode instanceof AcknowledgeAnyWhere) {
|
|
2143
2415
|
const upstreamHash = messageFrom?.publicKey.hashcode();
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
|
|
2151
|
-
|
|
2152
|
-
|
|
2416
|
+
// Route "distance" is based on recipient-seen order (0 = fastest). This is relied upon by
|
|
2417
|
+
// `Routes.getFanout(...)` which uses `distance < redundancy` to select redundant next-hops.
|
|
2418
|
+
const routeUpdate = this.rustCore
|
|
2419
|
+
? {
|
|
2420
|
+
...this.rustCore.decisions.seekAckRouteUpdate({
|
|
2421
|
+
current: this.publicKeyHash,
|
|
2422
|
+
upstream: upstreamHash,
|
|
2423
|
+
downstream: messageThrough.publicKey.hashcode(),
|
|
2424
|
+
}),
|
|
2425
|
+
target: messageTargetHash,
|
|
2426
|
+
distance: seenCounter,
|
|
2427
|
+
}
|
|
2428
|
+
: computeSeekAckRouteUpdate({
|
|
2429
|
+
current: this.publicKeyHash,
|
|
2430
|
+
upstream: upstreamHash,
|
|
2431
|
+
downstream: messageThrough.publicKey.hashcode(),
|
|
2432
|
+
target: messageTargetHash,
|
|
2433
|
+
distance: seenCounter,
|
|
2434
|
+
});
|
|
2153
2435
|
this.addRouteConnection(routeUpdate.from, routeUpdate.neighbour, messageTarget, routeUpdate.distance, session, Number(ack.header.session));
|
|
2154
2436
|
}
|
|
2155
2437
|
if (messageToSet.has(messageTargetHash)) {
|
|
@@ -2251,14 +2533,26 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
2251
2533
|
if (!isRelayed &&
|
|
2252
2534
|
message.header.mode instanceof AcknowledgeDelivery &&
|
|
2253
2535
|
usedNeighbours.size < message.header.mode.redundancy) {
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2536
|
+
if (this.rustCore) {
|
|
2537
|
+
const probes = this.rustCore.decisions.selectRedundancyProbes([...this.peers.keys()], [...usedNeighbours], message.header.mode.redundancy);
|
|
2538
|
+
for (const neighbour of probes) {
|
|
2539
|
+
const stream = this.peers.get(neighbour);
|
|
2540
|
+
if (!stream)
|
|
2541
|
+
continue;
|
|
2542
|
+
usedNeighbours.add(neighbour);
|
|
2543
|
+
promises.push(this.waitForPeerWrite(stream, bytes, message.header.priority, signal));
|
|
2544
|
+
}
|
|
2545
|
+
}
|
|
2546
|
+
else {
|
|
2547
|
+
for (const [neighbour, stream] of this.peers) {
|
|
2548
|
+
if (usedNeighbours.size >= message.header.mode.redundancy) {
|
|
2549
|
+
break;
|
|
2550
|
+
}
|
|
2551
|
+
if (usedNeighbours.has(neighbour))
|
|
2552
|
+
continue;
|
|
2553
|
+
usedNeighbours.add(neighbour);
|
|
2554
|
+
promises.push(this.waitForPeerWrite(stream, bytes, message.header.priority, signal));
|
|
2257
2555
|
}
|
|
2258
|
-
if (usedNeighbours.has(neighbour))
|
|
2259
|
-
continue;
|
|
2260
|
-
usedNeighbours.add(neighbour);
|
|
2261
|
-
promises.push(this.waitForPeerWrite(stream, bytes, message.header.priority, signal));
|
|
2262
2556
|
}
|
|
2263
2557
|
}
|
|
2264
2558
|
await Promise.all(promises);
|
|
@@ -2272,17 +2566,22 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
2272
2566
|
if (isRelayed && message.header.mode instanceof SilentDelivery) {
|
|
2273
2567
|
const promises = [];
|
|
2274
2568
|
const originalTo = message.header.mode.to;
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
|
|
2279
|
-
|
|
2569
|
+
const recipients = this.rustCore
|
|
2570
|
+
? this.rustCore.decisions.filterSilentRelayRecipients(originalTo, this.publicKeyHash, from.hashcode(), [...this.peers.keys()], getDeliveryHopTrace(message.header.mode))
|
|
2571
|
+
: undefined;
|
|
2572
|
+
for (const recipient of recipients ?? originalTo) {
|
|
2573
|
+
if (!recipients) {
|
|
2574
|
+
if (recipient === this.publicKeyHash)
|
|
2575
|
+
continue;
|
|
2576
|
+
if (recipient === from.hashcode())
|
|
2577
|
+
continue; // never send back to previous hop
|
|
2578
|
+
if (hasDeliveryHop(message.header.mode, recipient)) {
|
|
2579
|
+
continue; // recipient already signed/seen this message
|
|
2580
|
+
}
|
|
2581
|
+
}
|
|
2280
2582
|
const stream = this.peers.get(recipient);
|
|
2281
2583
|
if (!stream)
|
|
2282
2584
|
continue;
|
|
2283
|
-
if (hasDeliveryHop(message.header.mode, recipient)) {
|
|
2284
|
-
continue; // recipient already signed/seen this message
|
|
2285
|
-
}
|
|
2286
2585
|
message.header.mode.to = [recipient];
|
|
2287
2586
|
promises.push(this.waitForPeerWrite(stream, message.bytes(), message.header.priority, signal));
|
|
2288
2587
|
}
|
|
@@ -2306,19 +2605,29 @@ export class DirectStream extends TypedEventEmitter {
|
|
|
2306
2605
|
}
|
|
2307
2606
|
let sentOnce = false;
|
|
2308
2607
|
const promises = [];
|
|
2309
|
-
|
|
2310
|
-
const
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2608
|
+
if (this.rustCore) {
|
|
2609
|
+
const candidates = [...peers.values()];
|
|
2610
|
+
const kept = this.rustCore.decisions.filterFloodTargets(candidates.map((stream) => stream.publicKey.hashcode()), from.hashcode(), message.header.signatures?.publicKeys.map((x) => x.hashcode()) ?? [], getDeliveryHopTrace(message.header.mode));
|
|
2611
|
+
for (const index of kept) {
|
|
2612
|
+
sentOnce = true;
|
|
2613
|
+
promises.push(this.waitForPeerWrite(candidates[index], bytes, message.header.priority, signal));
|
|
2314
2614
|
}
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2615
|
+
}
|
|
2616
|
+
else {
|
|
2617
|
+
for (const stream of peers.values()) {
|
|
2618
|
+
const id = stream;
|
|
2619
|
+
// Dont sent back to the sender
|
|
2620
|
+
if (id.publicKey.equals(from)) {
|
|
2621
|
+
continue;
|
|
2622
|
+
}
|
|
2623
|
+
// Dont send message back to any peer already present in the path.
|
|
2624
|
+
if (message.header.signatures?.publicKeys.find((x) => x.equals(id.publicKey)) ||
|
|
2625
|
+
hasDeliveryHop(message.header.mode, id.publicKey.hashcode())) {
|
|
2626
|
+
continue;
|
|
2627
|
+
}
|
|
2628
|
+
sentOnce = true;
|
|
2629
|
+
promises.push(this.waitForPeerWrite(id, bytes, message.header.priority, signal));
|
|
2319
2630
|
}
|
|
2320
|
-
sentOnce = true;
|
|
2321
|
-
promises.push(this.waitForPeerWrite(id, bytes, message.header.priority, signal));
|
|
2322
2631
|
}
|
|
2323
2632
|
await Promise.all(promises);
|
|
2324
2633
|
startDeliveryTimeout?.();
|