@peerbit/stream 4.5.3 → 4.6.0-000e3f1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/benchmark/directstream-sim.d.ts +12 -0
- package/dist/benchmark/directstream-sim.d.ts.map +1 -0
- package/dist/benchmark/directstream-sim.js +299 -0
- package/dist/benchmark/directstream-sim.js.map +1 -0
- package/dist/benchmark/index.d.ts +10 -0
- package/dist/benchmark/index.d.ts.map +1 -0
- package/dist/benchmark/index.js +48 -0
- package/dist/benchmark/index.js.map +1 -0
- package/dist/benchmark/topology-sim.d.ts +12 -0
- package/dist/benchmark/topology-sim.d.ts.map +1 -0
- package/dist/benchmark/topology-sim.js +410 -0
- package/dist/benchmark/topology-sim.js.map +1 -0
- package/dist/benchmark/transfer.js +2 -2
- package/dist/benchmark/transfer.js.map +1 -1
- package/dist/src/core/seek-routing.d.ts +39 -0
- package/dist/src/core/seek-routing.d.ts.map +1 -0
- package/dist/src/core/seek-routing.js +33 -0
- package/dist/src/core/seek-routing.js.map +1 -0
- package/dist/src/index.d.ts +36 -4
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +528 -310
- package/dist/src/index.js.map +1 -1
- package/dist/src/routes.d.ts +11 -0
- package/dist/src/routes.d.ts.map +1 -1
- package/dist/src/routes.js +105 -16
- package/dist/src/routes.js.map +1 -1
- package/dist/src/wait-for-event.d.ts.map +1 -1
- package/dist/src/wait-for-event.js +34 -10
- package/dist/src/wait-for-event.js.map +1 -1
- package/package.json +9 -9
- package/src/core/seek-routing.ts +75 -0
- package/src/index.ts +803 -519
- package/src/routes.ts +121 -19
- package/src/wait-for-event.ts +46 -15
package/src/index.ts
CHANGED
|
@@ -29,6 +29,7 @@ import {
|
|
|
29
29
|
import type { SignatureWithKey } from "@peerbit/crypto";
|
|
30
30
|
import {
|
|
31
31
|
ACK,
|
|
32
|
+
AcknowledgeAnyWhere,
|
|
32
33
|
AcknowledgeDelivery,
|
|
33
34
|
AnyWhere,
|
|
34
35
|
DataMessage,
|
|
@@ -39,7 +40,6 @@ import {
|
|
|
39
40
|
MessageHeader,
|
|
40
41
|
MultiAddrinfo,
|
|
41
42
|
NotStartedError,
|
|
42
|
-
SeekDelivery,
|
|
43
43
|
SilentDelivery,
|
|
44
44
|
TracedDelivery,
|
|
45
45
|
coercePeerRefsToHashes,
|
|
@@ -69,6 +69,10 @@ import { type Pushable, pushable } from "it-pushable";
|
|
|
69
69
|
import pDefer, { type DeferredPromise } from "p-defer";
|
|
70
70
|
import Queue from "p-queue";
|
|
71
71
|
import { Uint8ArrayList } from "uint8arraylist";
|
|
72
|
+
import {
|
|
73
|
+
computeSeekAckRouteUpdate,
|
|
74
|
+
shouldAcknowledgeDataMessage,
|
|
75
|
+
} from "./core/seek-routing.js";
|
|
72
76
|
import { logger } from "./logger.js";
|
|
73
77
|
import { type PushableLanes, pushableLanes } from "./pushable-lanes.js";
|
|
74
78
|
import { MAX_ROUTE_DISTANCE, Routes } from "./routes.js";
|
|
@@ -167,15 +171,19 @@ const DEFAULT_MAX_CONNECTIONS = 300;
|
|
|
167
171
|
|
|
168
172
|
const DEFAULT_PRUNED_CONNNECTIONS_TIMEOUT = 30 * 1000;
|
|
169
173
|
|
|
170
|
-
const ROUTE_UPDATE_DELAY_FACTOR = 3e4;
|
|
171
|
-
|
|
172
174
|
const DEFAULT_CREATE_OUTBOUND_STREAM_TIMEOUT = 30_000;
|
|
173
175
|
|
|
176
|
+
const PRIORITY_LANES = 4;
|
|
177
|
+
|
|
174
178
|
const getLaneFromPriority = (priority: number) => {
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
+
// Higher priority numbers should be drained first.
|
|
180
|
+
// Lane 0 is the fastest/highest-priority lane.
|
|
181
|
+
const maxLane = PRIORITY_LANES - 1;
|
|
182
|
+
if (!Number.isFinite(priority)) {
|
|
183
|
+
return maxLane;
|
|
184
|
+
}
|
|
185
|
+
const clampedPriority = Math.max(0, Math.min(maxLane, Math.floor(priority)));
|
|
186
|
+
return maxLane - clampedPriority;
|
|
179
187
|
};
|
|
180
188
|
interface OutboundCandidate {
|
|
181
189
|
raw: Stream;
|
|
@@ -195,7 +203,7 @@ export interface InboundStreamRecord {
|
|
|
195
203
|
}
|
|
196
204
|
// Hook for tests to override queued length measurement (peerStreams, default impl)
|
|
197
205
|
export let measureOutboundQueuedBytes: (
|
|
198
|
-
ps: PeerStreams, // return queued bytes for active outbound (
|
|
206
|
+
ps: PeerStreams, // return queued bytes for active outbound (all lanes) or 0 if none
|
|
199
207
|
) => number = (ps: PeerStreams) => {
|
|
200
208
|
const active = ps._getActiveOutboundPushable();
|
|
201
209
|
if (!active) return 0;
|
|
@@ -203,8 +211,7 @@ export let measureOutboundQueuedBytes: (
|
|
|
203
211
|
// @ts-ignore - optional test helper
|
|
204
212
|
if (typeof active.getReadableLength === "function") {
|
|
205
213
|
try {
|
|
206
|
-
|
|
207
|
-
return active.getReadableLength(0) || 0;
|
|
214
|
+
return active.getReadableLength() || 0;
|
|
208
215
|
} catch {
|
|
209
216
|
// ignore
|
|
210
217
|
}
|
|
@@ -256,6 +263,19 @@ export class PeerStreams extends TypedEventEmitter<PeerStreamEvents> {
|
|
|
256
263
|
public _getActiveOutboundPushable(): PushableLanes<Uint8Array> | undefined {
|
|
257
264
|
return this.outboundStreams[0]?.pushable;
|
|
258
265
|
}
|
|
266
|
+
public getOutboundQueuedBytes(): number {
|
|
267
|
+
return this._getActiveOutboundPushable()?.readableLength ?? 0;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
public getOutboundQueuedBytesByLane(): number[] {
|
|
271
|
+
const p = this._getActiveOutboundPushable();
|
|
272
|
+
if (!p) return Array(PRIORITY_LANES).fill(0);
|
|
273
|
+
const out: number[] = [];
|
|
274
|
+
for (let lane = 0; lane < PRIORITY_LANES; lane++) {
|
|
275
|
+
out.push(p.getReadableLength(lane));
|
|
276
|
+
}
|
|
277
|
+
return out;
|
|
278
|
+
}
|
|
259
279
|
public _getOutboundCount() {
|
|
260
280
|
return this.outboundStreams.length;
|
|
261
281
|
}
|
|
@@ -284,7 +304,7 @@ export class PeerStreams extends TypedEventEmitter<PeerStreamEvents> {
|
|
|
284
304
|
const existing = this.outboundStreams.find((c) => c.raw === raw);
|
|
285
305
|
if (existing) return existing;
|
|
286
306
|
const pushableInst = pushableLanes<Uint8Array>({
|
|
287
|
-
lanes:
|
|
307
|
+
lanes: PRIORITY_LANES,
|
|
288
308
|
onPush: (val: Uint8Array) => {
|
|
289
309
|
candidate.bytesDelivered += val.length || val.byteLength || 0;
|
|
290
310
|
},
|
|
@@ -428,12 +448,7 @@ export class PeerStreams extends TypedEventEmitter<PeerStreamEvents> {
|
|
|
428
448
|
}
|
|
429
449
|
|
|
430
450
|
try {
|
|
431
|
-
c.pushable.push(
|
|
432
|
-
payload,
|
|
433
|
-
c.pushable.getReadableLength(0) === 0
|
|
434
|
-
? 0
|
|
435
|
-
: getLaneFromPriority(priority),
|
|
436
|
-
);
|
|
451
|
+
c.pushable.push(payload, getLaneFromPriority(priority));
|
|
437
452
|
successes++;
|
|
438
453
|
} catch (e) {
|
|
439
454
|
failures.push(e);
|
|
@@ -495,7 +510,10 @@ export class PeerStreams extends TypedEventEmitter<PeerStreamEvents> {
|
|
|
495
510
|
return;
|
|
496
511
|
}
|
|
497
512
|
|
|
498
|
-
|
|
513
|
+
// Outbound stream negotiation can legitimately take several seconds in CI
|
|
514
|
+
// (identify/protocol discovery, resource contention, etc). Keep this fairly
|
|
515
|
+
// generous so control-plane messages (joins/subscriptions) don't flap.
|
|
516
|
+
const timeoutMs = 10_000;
|
|
499
517
|
|
|
500
518
|
await new Promise<void>((resolve, reject) => {
|
|
501
519
|
const onOutbound = () => {
|
|
@@ -780,21 +798,29 @@ export class PeerStreams extends TypedEventEmitter<PeerStreamEvents> {
|
|
|
780
798
|
this.outboundAbortController.abort();
|
|
781
799
|
}
|
|
782
800
|
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
801
|
+
// End inbound streams
|
|
802
|
+
if (this.inboundStreams.length) {
|
|
803
|
+
for (const inbound of this.inboundStreams) {
|
|
804
|
+
try {
|
|
805
|
+
inbound.abortController.abort();
|
|
806
|
+
} catch {
|
|
807
|
+
logger.error("Failed to abort inbound stream");
|
|
808
|
+
}
|
|
809
|
+
try {
|
|
810
|
+
// Best-effort shutdown: on some transports (notably websockets),
|
|
811
|
+
// awaiting a graceful close can hang indefinitely if the remote is
|
|
812
|
+
// concurrently stopping. Abort immediately and do not await close.
|
|
813
|
+
try {
|
|
814
|
+
inbound.raw.abort?.(new AbortError("Closed"));
|
|
815
|
+
} catch {
|
|
816
|
+
// ignore
|
|
817
|
+
}
|
|
818
|
+
void Promise.resolve(inbound.raw.close?.()).catch(() => {});
|
|
819
|
+
} catch {
|
|
820
|
+
logger.error("Failed to close inbound stream");
|
|
821
|
+
}
|
|
795
822
|
}
|
|
796
823
|
}
|
|
797
|
-
}
|
|
798
824
|
|
|
799
825
|
this.usedBandWidthTracker.stop();
|
|
800
826
|
|
|
@@ -838,9 +864,27 @@ export type DirectStreamOptions = {
|
|
|
838
864
|
maxOutboundStreams?: number;
|
|
839
865
|
inboundIdleTimeout?: number; // override PeerStreams.INBOUND_IDLE_MS
|
|
840
866
|
connectionManager?: ConnectionManagerArguments;
|
|
841
|
-
routeSeekInterval?: number;
|
|
842
867
|
seekTimeout?: number;
|
|
843
868
|
routeMaxRetentionPeriod?: number;
|
|
869
|
+
/**
|
|
870
|
+
* Best-effort bounds for the per-process route cache. These exist to prevent
|
|
871
|
+
* unbounded memory growth in large networks/simulations.
|
|
872
|
+
*/
|
|
873
|
+
routeCacheMaxFromEntries?: number;
|
|
874
|
+
routeCacheMaxTargetsPerFrom?: number;
|
|
875
|
+
routeCacheMaxRelaysPerTarget?: number;
|
|
876
|
+
/**
|
|
877
|
+
* Share node-level routing/session state across DirectStream instances created
|
|
878
|
+
* from the same libp2p private key.
|
|
879
|
+
*
|
|
880
|
+
* This reduces duplicated topology knowledge when multiple protocols run on
|
|
881
|
+
* the same node (e.g. pubsub + fanout overlays).
|
|
882
|
+
*
|
|
883
|
+
* Defaults to `true`.
|
|
884
|
+
*/
|
|
885
|
+
sharedRouting?: boolean;
|
|
886
|
+
seenCacheMax?: number;
|
|
887
|
+
seenCacheTtlMs?: number;
|
|
844
888
|
};
|
|
845
889
|
|
|
846
890
|
type ConnectionManagerLike = {
|
|
@@ -870,16 +914,25 @@ export interface DirectStreamComponents {
|
|
|
870
914
|
privateKey: PrivateKey;
|
|
871
915
|
}
|
|
872
916
|
|
|
917
|
+
type SharedRoutingState = {
|
|
918
|
+
session: number;
|
|
919
|
+
routes: Routes;
|
|
920
|
+
controller: AbortController;
|
|
921
|
+
refs: number;
|
|
922
|
+
};
|
|
923
|
+
|
|
924
|
+
const sharedRoutingByPrivateKey = new WeakMap<PrivateKey, SharedRoutingState>();
|
|
925
|
+
|
|
873
926
|
export type PublishOptions = (WithMode | WithTo) &
|
|
874
927
|
PriorityOptions &
|
|
875
928
|
WithExtraSigners;
|
|
876
929
|
|
|
877
930
|
export abstract class DirectStream<
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
{
|
|
931
|
+
Events extends { [s: string]: any } = StreamEvents,
|
|
932
|
+
>
|
|
933
|
+
extends TypedEventEmitter<Events>
|
|
934
|
+
implements WaitForPeer, PublicKeyFromHashResolver
|
|
935
|
+
{
|
|
883
936
|
public peerId: PeerId;
|
|
884
937
|
public publicKey: PublicSignKey;
|
|
885
938
|
public publicKeyHash: string;
|
|
@@ -905,26 +958,35 @@ export abstract class DirectStream<
|
|
|
905
958
|
public multicodecs: string[];
|
|
906
959
|
public seenCache: Cache<number>;
|
|
907
960
|
private _registrarTopologyIds: string[] | undefined;
|
|
961
|
+
private _peerConnectListener?: (ev: any) => void;
|
|
962
|
+
private _peerDisconnectListener?: (ev: any) => void;
|
|
908
963
|
private readonly maxInboundStreams?: number;
|
|
909
964
|
private readonly maxOutboundStreams?: number;
|
|
910
965
|
connectionManagerOptions: ConnectionManagerOptions;
|
|
911
966
|
private recentDials?: Cache<string>;
|
|
912
967
|
private healthChecks: Map<string, ReturnType<typeof setTimeout>>;
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
968
|
+
private pruneConnectionsTimeout: ReturnType<typeof setInterval>;
|
|
969
|
+
private prunedConnectionsCache?: Cache<string>;
|
|
970
|
+
private pruneToLimitsInFlight?: Promise<void>;
|
|
971
|
+
private _startInFlight?: Promise<void>;
|
|
972
|
+
private routeMaxRetentionPeriod: number;
|
|
973
|
+
private routeCacheMaxFromEntries?: number;
|
|
974
|
+
private routeCacheMaxTargetsPerFrom?: number;
|
|
975
|
+
private routeCacheMaxRelaysPerTarget?: number;
|
|
976
|
+
private readonly sharedRouting: boolean;
|
|
977
|
+
private sharedRoutingKey?: PrivateKey;
|
|
978
|
+
private sharedRoutingState?: SharedRoutingState;
|
|
916
979
|
|
|
917
980
|
// for sequential creation of outbound streams
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
981
|
+
public outboundInflightQueue: Pushable<{
|
|
982
|
+
connection: Connection;
|
|
983
|
+
peerId: PeerId;
|
|
984
|
+
}>;
|
|
922
985
|
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
_outboundPump: ReturnType<typeof pipe> | undefined;
|
|
986
|
+
seekTimeout: number;
|
|
987
|
+
closeController: AbortController;
|
|
988
|
+
session: number;
|
|
989
|
+
_outboundPump: ReturnType<typeof pipe> | undefined;
|
|
928
990
|
|
|
929
991
|
private _ackCallbacks: Map<
|
|
930
992
|
string,
|
|
@@ -948,14 +1010,19 @@ export abstract class DirectStream<
|
|
|
948
1010
|
const {
|
|
949
1011
|
canRelayMessage = true,
|
|
950
1012
|
messageProcessingConcurrency = 10,
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
1013
|
+
maxInboundStreams,
|
|
1014
|
+
maxOutboundStreams,
|
|
1015
|
+
connectionManager,
|
|
1016
|
+
seekTimeout = SEEK_DELIVERY_TIMEOUT,
|
|
1017
|
+
routeMaxRetentionPeriod = ROUTE_MAX_RETANTION_PERIOD,
|
|
1018
|
+
routeCacheMaxFromEntries,
|
|
1019
|
+
routeCacheMaxTargetsPerFrom,
|
|
1020
|
+
routeCacheMaxRelaysPerTarget,
|
|
1021
|
+
sharedRouting = true,
|
|
1022
|
+
seenCacheMax = 1e6,
|
|
1023
|
+
seenCacheTtlMs = 10 * 60 * 1e3,
|
|
1024
|
+
inboundIdleTimeout,
|
|
1025
|
+
} = options || {};
|
|
959
1026
|
|
|
960
1027
|
const signKey = getKeypairFromPrivateKey(components.privateKey);
|
|
961
1028
|
this.seekTimeout = seekTimeout;
|
|
@@ -969,19 +1036,25 @@ export abstract class DirectStream<
|
|
|
969
1036
|
this.publicKeyHash = signKey.publicKey.hashcode();
|
|
970
1037
|
this.multicodecs = multicodecs;
|
|
971
1038
|
this.started = false;
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
1039
|
+
this.peers = new Map<string, PeerStreams>();
|
|
1040
|
+
this.canRelayMessage = canRelayMessage;
|
|
1041
|
+
this.healthChecks = new Map();
|
|
1042
|
+
this.queue = new Queue({ concurrency: messageProcessingConcurrency });
|
|
1043
|
+
this.maxInboundStreams = maxInboundStreams;
|
|
1044
|
+
this.maxOutboundStreams = maxOutboundStreams;
|
|
1045
|
+
this.seenCache = new Cache({
|
|
1046
|
+
max: Math.max(1, Math.floor(seenCacheMax)),
|
|
1047
|
+
ttl: Math.max(1, Math.floor(seenCacheTtlMs)),
|
|
1048
|
+
});
|
|
1049
|
+
this.routeMaxRetentionPeriod = routeMaxRetentionPeriod;
|
|
1050
|
+
this.routeCacheMaxFromEntries = routeCacheMaxFromEntries;
|
|
1051
|
+
this.routeCacheMaxTargetsPerFrom = routeCacheMaxTargetsPerFrom;
|
|
1052
|
+
this.routeCacheMaxRelaysPerTarget = routeCacheMaxRelaysPerTarget;
|
|
1053
|
+
this.sharedRouting = sharedRouting !== false;
|
|
1054
|
+
this.peerKeyHashToPublicKey = new Map();
|
|
1055
|
+
this._onIncomingStream = this._onIncomingStream.bind(this);
|
|
1056
|
+
this.onPeerConnected = this.onPeerConnected.bind(this);
|
|
1057
|
+
this.onPeerDisconnected = this.onPeerDisconnected.bind(this);
|
|
985
1058
|
|
|
986
1059
|
this._ackCallbacks = new Map();
|
|
987
1060
|
|
|
@@ -1022,23 +1095,60 @@ export abstract class DirectStream<
|
|
|
1022
1095
|
})
|
|
1023
1096
|
: undefined;
|
|
1024
1097
|
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1098
|
+
this.prunedConnectionsCache = this.connectionManagerOptions.pruner
|
|
1099
|
+
? new Cache({
|
|
1100
|
+
max: 1e6,
|
|
1101
|
+
ttl: this.connectionManagerOptions.pruner.connectionTimeout,
|
|
1102
|
+
})
|
|
1103
|
+
: undefined;
|
|
1104
|
+
}
|
|
1032
1105
|
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1106
|
+
private pruneConnectionsToLimits(): Promise<void> {
|
|
1107
|
+
if (this.pruneToLimitsInFlight) {
|
|
1108
|
+
return this.pruneToLimitsInFlight;
|
|
1109
|
+
}
|
|
1110
|
+
this.pruneToLimitsInFlight = (async () => {
|
|
1111
|
+
// Respect minConnections as a hard floor.
|
|
1112
|
+
const maxConnections = Math.max(
|
|
1113
|
+
this.connectionManagerOptions.minConnections,
|
|
1114
|
+
this.connectionManagerOptions.maxConnections,
|
|
1115
|
+
);
|
|
1116
|
+
if (this.peers.size <= maxConnections) {
|
|
1117
|
+
return;
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
// Prune in batches so we can quickly recover from join storms without waiting
|
|
1121
|
+
// for repeated pruner ticks. Bound work per run to avoid starving the event loop.
|
|
1122
|
+
const maxPrunes = Math.min(256, Math.max(1, this.peers.size - maxConnections));
|
|
1123
|
+
for (let i = 0; i < maxPrunes; i++) {
|
|
1124
|
+
if (this.peers.size <= maxConnections) break;
|
|
1125
|
+
const before = this.peers.size;
|
|
1126
|
+
await this.pruneConnections();
|
|
1127
|
+
if (this.peers.size >= before) break; // nothing prunable
|
|
1128
|
+
}
|
|
1129
|
+
})().finally(() => {
|
|
1130
|
+
this.pruneToLimitsInFlight = undefined;
|
|
1131
|
+
});
|
|
1132
|
+
return this.pruneToLimitsInFlight;
|
|
1036
1133
|
}
|
|
1037
1134
|
|
|
1038
|
-
|
|
1039
|
-
|
|
1135
|
+
async start() {
|
|
1136
|
+
if (this.started) return;
|
|
1137
|
+
if (this._startInFlight) return this._startInFlight;
|
|
1138
|
+
this._startInFlight = this._startImpl().finally(() => {
|
|
1139
|
+
this._startInFlight = undefined;
|
|
1140
|
+
});
|
|
1141
|
+
return this._startInFlight;
|
|
1142
|
+
}
|
|
1040
1143
|
|
|
1041
|
-
|
|
1144
|
+
private async _startImpl() {
|
|
1145
|
+
if (this.started) {
|
|
1146
|
+
return;
|
|
1147
|
+
}
|
|
1148
|
+
|
|
1149
|
+
await ready;
|
|
1150
|
+
|
|
1151
|
+
this.closeController = new AbortController();
|
|
1042
1152
|
|
|
1043
1153
|
this.outboundInflightQueue = pushable({ objectMode: true });
|
|
1044
1154
|
|
|
@@ -1116,10 +1226,47 @@ export abstract class DirectStream<
|
|
|
1116
1226
|
this.outboundInflightQueue.return();
|
|
1117
1227
|
});
|
|
1118
1228
|
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1229
|
+
if (this.sharedRouting) {
|
|
1230
|
+
const key = this.components.privateKey;
|
|
1231
|
+
this.sharedRoutingKey = key;
|
|
1232
|
+
let state = sharedRoutingByPrivateKey.get(key);
|
|
1233
|
+
if (!state) {
|
|
1234
|
+
const controller = new AbortController();
|
|
1235
|
+
state = {
|
|
1236
|
+
session: Date.now(),
|
|
1237
|
+
controller,
|
|
1238
|
+
routes: new Routes(this.publicKeyHash, {
|
|
1239
|
+
routeMaxRetentionPeriod: this.routeMaxRetentionPeriod,
|
|
1240
|
+
signal: controller.signal,
|
|
1241
|
+
maxFromEntries: this.routeCacheMaxFromEntries,
|
|
1242
|
+
maxTargetsPerFrom: this.routeCacheMaxTargetsPerFrom,
|
|
1243
|
+
maxRelaysPerTarget: this.routeCacheMaxRelaysPerTarget,
|
|
1244
|
+
}),
|
|
1245
|
+
refs: 0,
|
|
1246
|
+
};
|
|
1247
|
+
sharedRoutingByPrivateKey.set(key, state);
|
|
1248
|
+
} else {
|
|
1249
|
+
// Best-effort: prefer the strictest cleanup policy among co-located protocols.
|
|
1250
|
+
state.routes.routeMaxRetentionPeriod = Math.min(
|
|
1251
|
+
state.routes.routeMaxRetentionPeriod,
|
|
1252
|
+
this.routeMaxRetentionPeriod,
|
|
1253
|
+
);
|
|
1254
|
+
}
|
|
1255
|
+
|
|
1256
|
+
state.refs += 1;
|
|
1257
|
+
this.sharedRoutingState = state;
|
|
1258
|
+
this.session = state.session;
|
|
1259
|
+
this.routes = state.routes;
|
|
1260
|
+
} else {
|
|
1261
|
+
this.session = Date.now();
|
|
1262
|
+
this.routes = new Routes(this.publicKeyHash, {
|
|
1263
|
+
routeMaxRetentionPeriod: this.routeMaxRetentionPeriod,
|
|
1264
|
+
signal: this.closeController.signal,
|
|
1265
|
+
maxFromEntries: this.routeCacheMaxFromEntries,
|
|
1266
|
+
maxTargetsPerFrom: this.routeCacheMaxTargetsPerFrom,
|
|
1267
|
+
maxRelaysPerTarget: this.routeCacheMaxRelaysPerTarget,
|
|
1268
|
+
});
|
|
1269
|
+
}
|
|
1123
1270
|
|
|
1124
1271
|
this.started = true;
|
|
1125
1272
|
this.stopping = false;
|
|
@@ -1149,6 +1296,46 @@ export abstract class DirectStream<
|
|
|
1149
1296
|
),
|
|
1150
1297
|
);
|
|
1151
1298
|
|
|
1299
|
+
// Best-effort fallback: topology callbacks can depend on identify/protocol
|
|
1300
|
+
// discovery. Some test environments connect peers without triggering the
|
|
1301
|
+
// per-protocol topology immediately. Listening to peer connection events
|
|
1302
|
+
// ensures we still attempt to open outbound streams opportunistically.
|
|
1303
|
+
this._peerConnectListener = (ev: any) => {
|
|
1304
|
+
if (this.stopping || !this.started) return;
|
|
1305
|
+
const peerId = (ev as any)?.detail ?? (ev as any)?.peerId;
|
|
1306
|
+
if (!peerId) return;
|
|
1307
|
+
const conns = this.components.connectionManager.getConnections(peerId);
|
|
1308
|
+
if (!conns || conns.length === 0) return;
|
|
1309
|
+
let conn = conns[0]!;
|
|
1310
|
+
for (const c of conns) {
|
|
1311
|
+
if (!isWebsocketConnection(c)) {
|
|
1312
|
+
conn = c;
|
|
1313
|
+
break;
|
|
1314
|
+
}
|
|
1315
|
+
}
|
|
1316
|
+
void this.onPeerConnected(peerId, conn);
|
|
1317
|
+
};
|
|
1318
|
+
this._peerDisconnectListener = (ev: any) => {
|
|
1319
|
+
if (this.stopping || !this.started) return;
|
|
1320
|
+
const peerId = (ev as any)?.detail ?? (ev as any)?.peerId;
|
|
1321
|
+
if (!peerId) return;
|
|
1322
|
+
const conns = this.components.connectionManager.getConnections(peerId);
|
|
1323
|
+
const conn = conns && conns.length > 0 ? conns[0] : undefined;
|
|
1324
|
+
void this.onPeerDisconnected(peerId, conn);
|
|
1325
|
+
};
|
|
1326
|
+
try {
|
|
1327
|
+
this.components.events.addEventListener(
|
|
1328
|
+
"peer:connect",
|
|
1329
|
+
this._peerConnectListener as any,
|
|
1330
|
+
);
|
|
1331
|
+
this.components.events.addEventListener(
|
|
1332
|
+
"peer:disconnect",
|
|
1333
|
+
this._peerDisconnectListener as any,
|
|
1334
|
+
);
|
|
1335
|
+
} catch {
|
|
1336
|
+
// ignore unsupported event targets
|
|
1337
|
+
}
|
|
1338
|
+
|
|
1152
1339
|
// All existing connections are like new ones for us. To deduplication on remotes so we only resuse one connection for this protocol (we could be connected with many connections)
|
|
1153
1340
|
const peerToConnections: Map<string, Connection[]> = new Map();
|
|
1154
1341
|
const connections = this.components.connectionManager.getConnections();
|
|
@@ -1198,7 +1385,28 @@ export abstract class DirectStream<
|
|
|
1198
1385
|
return;
|
|
1199
1386
|
}
|
|
1200
1387
|
|
|
1388
|
+
const sharedState = this.sharedRoutingState;
|
|
1389
|
+
const sharedKey = this.sharedRoutingKey;
|
|
1390
|
+
|
|
1201
1391
|
clearTimeout(this.pruneConnectionsTimeout);
|
|
1392
|
+
try {
|
|
1393
|
+
if (this._peerConnectListener) {
|
|
1394
|
+
this.components.events.removeEventListener(
|
|
1395
|
+
"peer:connect",
|
|
1396
|
+
this._peerConnectListener as any,
|
|
1397
|
+
);
|
|
1398
|
+
}
|
|
1399
|
+
if (this._peerDisconnectListener) {
|
|
1400
|
+
this.components.events.removeEventListener(
|
|
1401
|
+
"peer:disconnect",
|
|
1402
|
+
this._peerDisconnectListener as any,
|
|
1403
|
+
);
|
|
1404
|
+
}
|
|
1405
|
+
} catch {
|
|
1406
|
+
// ignore unsupported event targets
|
|
1407
|
+
}
|
|
1408
|
+
this._peerConnectListener = undefined;
|
|
1409
|
+
this._peerDisconnectListener = undefined;
|
|
1202
1410
|
|
|
1203
1411
|
await Promise.all(
|
|
1204
1412
|
this.multicodecs.map((x) => this.components.registrar.unhandle(x)),
|
|
@@ -1232,7 +1440,11 @@ export abstract class DirectStream<
|
|
|
1232
1440
|
this.queue.clear();
|
|
1233
1441
|
this.peers.clear();
|
|
1234
1442
|
this.seenCache.clear();
|
|
1235
|
-
|
|
1443
|
+
// When routing is shared across co-located protocols, only clear once the last
|
|
1444
|
+
// instance stops. Otherwise we'd wipe routes still in use by other services.
|
|
1445
|
+
if (!sharedState) {
|
|
1446
|
+
this.routes.clear();
|
|
1447
|
+
}
|
|
1236
1448
|
this.peerKeyHashToPublicKey.clear();
|
|
1237
1449
|
|
|
1238
1450
|
for (const [_k, v] of this._ackCallbacks) {
|
|
@@ -1240,6 +1452,24 @@ export abstract class DirectStream<
|
|
|
1240
1452
|
}
|
|
1241
1453
|
|
|
1242
1454
|
this._ackCallbacks.clear();
|
|
1455
|
+
this.sharedRoutingState = undefined;
|
|
1456
|
+
this.sharedRoutingKey = undefined;
|
|
1457
|
+
if (sharedState && sharedKey) {
|
|
1458
|
+
sharedState.refs = Math.max(0, sharedState.refs - 1);
|
|
1459
|
+
if (sharedState.refs === 0) {
|
|
1460
|
+
try {
|
|
1461
|
+
sharedState.routes.clear();
|
|
1462
|
+
} catch {
|
|
1463
|
+
// ignore
|
|
1464
|
+
}
|
|
1465
|
+
try {
|
|
1466
|
+
sharedState.controller.abort();
|
|
1467
|
+
} catch {
|
|
1468
|
+
// ignore
|
|
1469
|
+
}
|
|
1470
|
+
sharedRoutingByPrivateKey.delete(sharedKey);
|
|
1471
|
+
}
|
|
1472
|
+
}
|
|
1243
1473
|
logger.trace("stopped");
|
|
1244
1474
|
this.stopping = false;
|
|
1245
1475
|
}
|
|
@@ -1436,7 +1666,8 @@ export abstract class DirectStream<
|
|
|
1436
1666
|
).catch(dontThrowIfDeliveryError);
|
|
1437
1667
|
}
|
|
1438
1668
|
|
|
1439
|
-
|
|
1669
|
+
// Best-effort liveness probe; never let background probe failures crash callers.
|
|
1670
|
+
void this.checkIsAlive([peerKeyHash]).catch(() => false);
|
|
1440
1671
|
}
|
|
1441
1672
|
|
|
1442
1673
|
logger.trace("connection ended:" + peerKey.toString());
|
|
@@ -1461,27 +1692,31 @@ export abstract class DirectStream<
|
|
|
1461
1692
|
distance: number,
|
|
1462
1693
|
session: number,
|
|
1463
1694
|
remoteSession: number,
|
|
1464
|
-
|
|
1465
|
-
|
|
1695
|
+
) {
|
|
1696
|
+
const targetHash = typeof target === "string" ? target : target.hashcode();
|
|
1697
|
+
// Best-effort: keep a hash -> public key map for any routed targets so
|
|
1698
|
+
// peer:unreachable events can always carry a PublicSignKey when we have seen it.
|
|
1699
|
+
if (typeof target !== "string") {
|
|
1700
|
+
this.peerKeyHashToPublicKey.set(targetHash, target);
|
|
1701
|
+
}
|
|
1466
1702
|
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1703
|
+
const update = this.routes.add(
|
|
1704
|
+
from,
|
|
1705
|
+
neighbour,
|
|
1706
|
+
targetHash,
|
|
1471
1707
|
distance,
|
|
1472
1708
|
session,
|
|
1473
1709
|
remoteSession,
|
|
1474
1710
|
);
|
|
1475
1711
|
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1712
|
+
// second condition is that we don't want to emit 'reachable' events for routes where we act only as a relay
|
|
1713
|
+
// in this case, from is != this.publicKeyhash
|
|
1714
|
+
if (from === this.publicKeyHash) {
|
|
1715
|
+
if (update === "new") {
|
|
1716
|
+
this.onPeerReachable(target);
|
|
1717
|
+
}
|
|
1482
1718
|
}
|
|
1483
1719
|
}
|
|
1484
|
-
}
|
|
1485
1720
|
|
|
1486
1721
|
public onPeerReachable(publicKey: PublicSignKey) {
|
|
1487
1722
|
// override this fn
|
|
@@ -1490,16 +1725,19 @@ export abstract class DirectStream<
|
|
|
1490
1725
|
);
|
|
1491
1726
|
}
|
|
1492
1727
|
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
}
|
|
1501
|
-
|
|
1502
|
-
|
|
1728
|
+
public onPeerUnreachable(hash: string) {
|
|
1729
|
+
// override this fns
|
|
1730
|
+
const key = this.peerKeyHashToPublicKey.get(hash);
|
|
1731
|
+
if (!key) {
|
|
1732
|
+
// Best-effort: we may only have the hash (no public key) for some routes.
|
|
1733
|
+
// Avoid crashing downstream listeners that assume `detail` is a PublicSignKey.
|
|
1734
|
+
return;
|
|
1735
|
+
}
|
|
1736
|
+
this.dispatchEvent(
|
|
1737
|
+
// TODO types
|
|
1738
|
+
new CustomEvent("peer:unreachable", { detail: key }),
|
|
1739
|
+
);
|
|
1740
|
+
}
|
|
1503
1741
|
|
|
1504
1742
|
public updateSession(key: PublicSignKey, session?: number) {
|
|
1505
1743
|
if (this.routes.updateSession(key.hashcode(), session)) {
|
|
@@ -1522,11 +1760,11 @@ export abstract class DirectStream<
|
|
|
1522
1760
|
/**
|
|
1523
1761
|
* Notifies the router that a peer has been connected
|
|
1524
1762
|
*/
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1763
|
+
addPeer(
|
|
1764
|
+
peerId: PeerId,
|
|
1765
|
+
publicKey: PublicSignKey,
|
|
1766
|
+
protocol: string,
|
|
1767
|
+
connId: string,
|
|
1530
1768
|
): PeerStreams {
|
|
1531
1769
|
const publicKeyHash = publicKey.hashcode();
|
|
1532
1770
|
|
|
@@ -1574,17 +1812,23 @@ export abstract class DirectStream<
|
|
|
1574
1812
|
{ once: true },
|
|
1575
1813
|
);
|
|
1576
1814
|
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1815
|
+
this.addRouteConnection(
|
|
1816
|
+
this.publicKeyHash,
|
|
1817
|
+
publicKey.hashcode(),
|
|
1818
|
+
publicKey,
|
|
1819
|
+
-1,
|
|
1820
|
+
+new Date(),
|
|
1821
|
+
-1,
|
|
1822
|
+
);
|
|
1585
1823
|
|
|
1586
|
-
|
|
1587
|
-
|
|
1824
|
+
// Enforce connection manager limits eagerly when new peers are added. Without this,
|
|
1825
|
+
// join storms can create large temporary peer sets and OOM in single-process sims.
|
|
1826
|
+
if (this.peers.size > this.connectionManagerOptions.maxConnections) {
|
|
1827
|
+
void this.pruneConnectionsToLimits().catch(() => {});
|
|
1828
|
+
}
|
|
1829
|
+
|
|
1830
|
+
return peerStreams;
|
|
1831
|
+
}
|
|
1588
1832
|
|
|
1589
1833
|
/**
|
|
1590
1834
|
* Notifies the router that a peer has been disconnected
|
|
@@ -1727,24 +1971,25 @@ export abstract class DirectStream<
|
|
|
1727
1971
|
}
|
|
1728
1972
|
|
|
1729
1973
|
public shouldIgnore(message: DataMessage, seenBefore: number) {
|
|
1730
|
-
const
|
|
1731
|
-
|
|
1732
|
-
|
|
1974
|
+
const signedBySelf =
|
|
1975
|
+
message.header.signatures?.publicKeys.some((x) =>
|
|
1976
|
+
x.equals(this.publicKey),
|
|
1977
|
+
) ?? false;
|
|
1733
1978
|
|
|
1734
|
-
if (
|
|
1979
|
+
if (signedBySelf) {
|
|
1735
1980
|
return true;
|
|
1736
1981
|
}
|
|
1737
1982
|
|
|
1983
|
+
// For acknowledged modes, allow limited duplicate forwarding so that we can
|
|
1984
|
+
// discover and maintain multiple candidate routes (distance=seenCounter).
|
|
1738
1985
|
if (
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
(message.header.mode instanceof SeekDelivery &&
|
|
1742
|
-
seenBefore >= message.header.mode.redundancy)
|
|
1986
|
+
message.header.mode instanceof AcknowledgeDelivery ||
|
|
1987
|
+
message.header.mode instanceof AcknowledgeAnyWhere
|
|
1743
1988
|
) {
|
|
1744
|
-
return
|
|
1989
|
+
return seenBefore >= message.header.mode.redundancy;
|
|
1745
1990
|
}
|
|
1746
1991
|
|
|
1747
|
-
return
|
|
1992
|
+
return seenBefore > 0;
|
|
1748
1993
|
}
|
|
1749
1994
|
|
|
1750
1995
|
public async onDataMessage(
|
|
@@ -1758,14 +2003,19 @@ export abstract class DirectStream<
|
|
|
1758
2003
|
}
|
|
1759
2004
|
|
|
1760
2005
|
let isForMe = false;
|
|
1761
|
-
if (
|
|
2006
|
+
if (
|
|
2007
|
+
message.header.mode instanceof AnyWhere ||
|
|
2008
|
+
message.header.mode instanceof AcknowledgeAnyWhere
|
|
2009
|
+
) {
|
|
1762
2010
|
isForMe = true;
|
|
1763
2011
|
} else {
|
|
1764
2012
|
const isFromSelf = this.publicKey.equals(from);
|
|
1765
|
-
if (
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
message.header.mode
|
|
2013
|
+
if (
|
|
2014
|
+
!isFromSelf &&
|
|
2015
|
+
(message.header.mode instanceof SilentDelivery ||
|
|
2016
|
+
message.header.mode instanceof AcknowledgeDelivery)
|
|
2017
|
+
) {
|
|
2018
|
+
isForMe = message.header.mode.to.includes(this.publicKeyHash);
|
|
1769
2019
|
}
|
|
1770
2020
|
}
|
|
1771
2021
|
|
|
@@ -1801,25 +2051,17 @@ export abstract class DirectStream<
|
|
|
1801
2051
|
}
|
|
1802
2052
|
}
|
|
1803
2053
|
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
) && x !== peerStream,
|
|
1814
|
-
);
|
|
1815
|
-
if (to.length > 0) {
|
|
1816
|
-
this.relayMessage(from, message, to);
|
|
1817
|
-
}
|
|
1818
|
-
}
|
|
1819
|
-
} else {
|
|
2054
|
+
// Forward
|
|
2055
|
+
const shouldForward =
|
|
2056
|
+
seenBefore === 0 ||
|
|
2057
|
+
((message.header.mode instanceof AcknowledgeDelivery ||
|
|
2058
|
+
message.header.mode instanceof AcknowledgeAnyWhere) &&
|
|
2059
|
+
seenBefore < message.header.mode.redundancy);
|
|
2060
|
+
|
|
2061
|
+
if (shouldForward) {
|
|
2062
|
+
// DONT await this since it might introduce a dead-lock
|
|
1820
2063
|
this.relayMessage(from, message);
|
|
1821
2064
|
}
|
|
1822
|
-
}
|
|
1823
2065
|
}
|
|
1824
2066
|
|
|
1825
2067
|
public async verifyAndProcess(message: Message<any>) {
|
|
@@ -1843,14 +2085,20 @@ export abstract class DirectStream<
|
|
|
1843
2085
|
seenBefore: number,
|
|
1844
2086
|
) {
|
|
1845
2087
|
if (
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
seenBefore < message.header.mode.redundancy
|
|
2088
|
+
message.header.mode instanceof AcknowledgeDelivery ||
|
|
2089
|
+
message.header.mode instanceof AcknowledgeAnyWhere
|
|
1849
2090
|
) {
|
|
1850
|
-
const
|
|
1851
|
-
message.header.mode
|
|
1852
|
-
|
|
1853
|
-
|
|
2091
|
+
const isRecipient =
|
|
2092
|
+
message.header.mode instanceof AcknowledgeAnyWhere
|
|
2093
|
+
? true
|
|
2094
|
+
: message.header.mode.to.includes(this.publicKeyHash);
|
|
2095
|
+
if (
|
|
2096
|
+
!shouldAcknowledgeDataMessage({
|
|
2097
|
+
isRecipient,
|
|
2098
|
+
seenBefore,
|
|
2099
|
+
redundancy: message.header.mode.redundancy,
|
|
2100
|
+
})
|
|
2101
|
+
) {
|
|
1854
2102
|
return;
|
|
1855
2103
|
}
|
|
1856
2104
|
const signers = message.header.signatures!.publicKeys.map((x) =>
|
|
@@ -1867,14 +2115,18 @@ export abstract class DirectStream<
|
|
|
1867
2115
|
mode: new TracedDelivery(signers),
|
|
1868
2116
|
session: this.session,
|
|
1869
2117
|
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
2118
|
+
// include our origin for route-learning/dialer hints (best-effort privacy/anti-spam control):
|
|
2119
|
+
// only include once (seenBefore=0) and only if we have not recently pruned
|
|
2120
|
+
// a connection to any signer in the path
|
|
2121
|
+
origin:
|
|
2122
|
+
(message.header.mode instanceof AcknowledgeAnyWhere ||
|
|
2123
|
+
message.header.mode instanceof AcknowledgeDelivery) &&
|
|
2124
|
+
seenBefore === 0 &&
|
|
2125
|
+
!message.header.signatures!.publicKeys.find((x) =>
|
|
2126
|
+
this.prunedConnectionsCache?.has(x.hashcode()),
|
|
2127
|
+
)
|
|
2128
|
+
? new MultiAddrinfo(
|
|
2129
|
+
this.components.addressManager
|
|
1878
2130
|
.getAddresses()
|
|
1879
2131
|
.map((x) => x.toString()),
|
|
1880
2132
|
)
|
|
@@ -2017,33 +2269,42 @@ export abstract class DirectStream<
|
|
|
2017
2269
|
for (const remote of remotes) {
|
|
2018
2270
|
this.invalidateSession(remote);
|
|
2019
2271
|
}
|
|
2020
|
-
|
|
2272
|
+
// Best-effort liveness probe; never let background probe failures crash callers.
|
|
2273
|
+
void this.checkIsAlive(remotes).catch(() => false);
|
|
2021
2274
|
}
|
|
2022
2275
|
private async checkIsAlive(remotes: string[]) {
|
|
2023
2276
|
if (this.peers.size === 0) {
|
|
2024
2277
|
return false;
|
|
2025
2278
|
}
|
|
2026
2279
|
if (remotes.length > 0) {
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2280
|
+
try {
|
|
2281
|
+
// Keepalive probes are best-effort, but must still require ACKs so we can
|
|
2282
|
+
// conclusively prune stale routes when peers actually went away.
|
|
2283
|
+
await this.publish(undefined, {
|
|
2284
|
+
mode: new AcknowledgeDelivery({
|
|
2285
|
+
to: remotes,
|
|
2286
|
+
redundancy: DEFAULT_SILENT_MESSAGE_REDUDANCY,
|
|
2287
|
+
}),
|
|
2288
|
+
});
|
|
2289
|
+
return true;
|
|
2290
|
+
} catch (e: any) {
|
|
2291
|
+
const errorName =
|
|
2292
|
+
e?.name ?? e?.constructor?.name ?? "UnknownError";
|
|
2293
|
+
if (
|
|
2294
|
+
e instanceof DeliveryError ||
|
|
2295
|
+
e instanceof NotStartedError ||
|
|
2296
|
+
e instanceof TimeoutError ||
|
|
2297
|
+
e instanceof AbortError ||
|
|
2298
|
+
errorName === "DeliveryError" ||
|
|
2299
|
+
errorName === "NotStartedError" ||
|
|
2300
|
+
errorName === "TimeoutError" ||
|
|
2301
|
+
errorName === "AbortError"
|
|
2302
|
+
) {
|
|
2303
|
+
return false;
|
|
2304
|
+
}
|
|
2305
|
+
warn(`checkIsAlive unexpected error: ${errorName}`);
|
|
2306
|
+
return false;
|
|
2307
|
+
}
|
|
2047
2308
|
}
|
|
2048
2309
|
return false;
|
|
2049
2310
|
}
|
|
@@ -2056,7 +2317,11 @@ export abstract class DirectStream<
|
|
|
2056
2317
|
) {
|
|
2057
2318
|
// dispatch the event if we are interested
|
|
2058
2319
|
|
|
2059
|
-
let mode:
|
|
2320
|
+
let mode:
|
|
2321
|
+
| SilentDelivery
|
|
2322
|
+
| AcknowledgeDelivery
|
|
2323
|
+
| AcknowledgeAnyWhere
|
|
2324
|
+
| AnyWhere = (
|
|
2060
2325
|
options as WithMode
|
|
2061
2326
|
).mode
|
|
2062
2327
|
? (options as WithMode).mode!
|
|
@@ -2067,8 +2332,7 @@ export abstract class DirectStream<
|
|
|
2067
2332
|
|
|
2068
2333
|
if (
|
|
2069
2334
|
mode instanceof AcknowledgeDelivery ||
|
|
2070
|
-
mode instanceof SilentDelivery
|
|
2071
|
-
mode instanceof SeekDelivery
|
|
2335
|
+
mode instanceof SilentDelivery
|
|
2072
2336
|
) {
|
|
2073
2337
|
if (mode.to) {
|
|
2074
2338
|
let preLength = mode.to.length;
|
|
@@ -2080,7 +2344,7 @@ export abstract class DirectStream<
|
|
|
2080
2344
|
);
|
|
2081
2345
|
}
|
|
2082
2346
|
|
|
2083
|
-
if (mode.to.length === 0
|
|
2347
|
+
if (mode.to.length === 0) {
|
|
2084
2348
|
throw new InvalidMessageError(
|
|
2085
2349
|
"Unexpected to deliver message with mode: " +
|
|
2086
2350
|
mode.constructor.name +
|
|
@@ -2091,28 +2355,6 @@ export abstract class DirectStream<
|
|
|
2091
2355
|
}
|
|
2092
2356
|
}
|
|
2093
2357
|
|
|
2094
|
-
if (mode instanceof AcknowledgeDelivery || mode instanceof SilentDelivery) {
|
|
2095
|
-
const now = +new Date();
|
|
2096
|
-
for (const hash of mode.to) {
|
|
2097
|
-
const neighbourRoutes = this.routes.routes
|
|
2098
|
-
.get(this.publicKeyHash)
|
|
2099
|
-
?.get(hash);
|
|
2100
|
-
|
|
2101
|
-
if (
|
|
2102
|
-
!neighbourRoutes ||
|
|
2103
|
-
now - neighbourRoutes.session >
|
|
2104
|
-
neighbourRoutes.list.length * this.routeSeekInterval ||
|
|
2105
|
-
!this.routes.isUpToDate(hash, neighbourRoutes)
|
|
2106
|
-
) {
|
|
2107
|
-
mode = new SeekDelivery({
|
|
2108
|
-
to: mode.to,
|
|
2109
|
-
redundancy: DEFAULT_SEEK_MESSAGE_REDUDANCY,
|
|
2110
|
-
});
|
|
2111
|
-
break;
|
|
2112
|
-
}
|
|
2113
|
-
}
|
|
2114
|
-
}
|
|
2115
|
-
|
|
2116
2358
|
const message = new DataMessage({
|
|
2117
2359
|
data: data instanceof Uint8ArrayList ? data.subarray() : data,
|
|
2118
2360
|
header: new MessageHeader({
|
|
@@ -2139,7 +2381,9 @@ export abstract class DirectStream<
|
|
|
2139
2381
|
async publish(
|
|
2140
2382
|
data: Uint8Array | Uint8ArrayList | undefined,
|
|
2141
2383
|
options: PublishOptions = {
|
|
2142
|
-
mode: new
|
|
2384
|
+
mode: new AcknowledgeAnyWhere({
|
|
2385
|
+
redundancy: DEFAULT_SEEK_MESSAGE_REDUDANCY,
|
|
2386
|
+
}),
|
|
2143
2387
|
},
|
|
2144
2388
|
): Promise<Uint8Array | undefined> {
|
|
2145
2389
|
if (!this.started) {
|
|
@@ -2184,7 +2428,7 @@ export abstract class DirectStream<
|
|
|
2184
2428
|
if (message instanceof DataMessage) {
|
|
2185
2429
|
if (
|
|
2186
2430
|
message.header.mode instanceof AcknowledgeDelivery ||
|
|
2187
|
-
message.header.mode instanceof
|
|
2431
|
+
message.header.mode instanceof AcknowledgeAnyWhere
|
|
2188
2432
|
) {
|
|
2189
2433
|
await message.sign(this.sign);
|
|
2190
2434
|
}
|
|
@@ -2212,6 +2456,37 @@ export abstract class DirectStream<
|
|
|
2212
2456
|
this.healthChecks.delete(to);
|
|
2213
2457
|
}
|
|
2214
2458
|
|
|
2459
|
+
private formatDeliveryDebugState(
|
|
2460
|
+
targets: Set<string>,
|
|
2461
|
+
fastestNodesReached: Map<string, number[]>,
|
|
2462
|
+
): string {
|
|
2463
|
+
const sampleSize = 16;
|
|
2464
|
+
const sampledTargets = [...targets].slice(0, sampleSize);
|
|
2465
|
+
const sampled = sampledTargets.map((target) => {
|
|
2466
|
+
const route = this.routes.findNeighbor(this.publicKeyHash, target);
|
|
2467
|
+
const routeSample =
|
|
2468
|
+
route?.list
|
|
2469
|
+
.slice(0, 3)
|
|
2470
|
+
.map((r) => `${r.hash}:${r.distance}${r.expireAt ? ":old" : ""}`)
|
|
2471
|
+
.join(",") ?? "-";
|
|
2472
|
+
const reachable = this.routes.isReachable(this.publicKeyHash, target);
|
|
2473
|
+
const stream = this.peers.get(target);
|
|
2474
|
+
let openConnections = 0;
|
|
2475
|
+
if (stream) {
|
|
2476
|
+
try {
|
|
2477
|
+
openConnections = this.components.connectionManager.getConnections(
|
|
2478
|
+
stream.peerId,
|
|
2479
|
+
).length;
|
|
2480
|
+
} catch {
|
|
2481
|
+
openConnections = stream.isReadable || stream.isWritable ? 1 : 0;
|
|
2482
|
+
}
|
|
2483
|
+
}
|
|
2484
|
+
return `${target}{reachable=${reachable ? 1 : 0},ack=${fastestNodesReached.has(target) ? 1 : 0},stream=${stream ? 1 : 0},conns=${openConnections},routes=${routeSample}}`;
|
|
2485
|
+
});
|
|
2486
|
+
const more = targets.size - sampledTargets.length;
|
|
2487
|
+
return `deliveryState peers=${this.peers.size} routesLocal=${this.routes.count()} routesAll=${this.routes.countAll()} targets=[${sampled.join(";")}]${more > 0 ? ` (+${more} more)` : ""}`;
|
|
2488
|
+
}
|
|
2489
|
+
|
|
2215
2490
|
private async createDeliveryPromise(
|
|
2216
2491
|
from: PublicSignKey,
|
|
2217
2492
|
message: DataMessage | Goodbye,
|
|
@@ -2235,11 +2510,9 @@ export abstract class DirectStream<
|
|
|
2235
2510
|
|
|
2236
2511
|
const fastestNodesReached = new Map<string, number[]>();
|
|
2237
2512
|
const messageToSet: Set<string> = new Set();
|
|
2238
|
-
if (message.header.mode
|
|
2513
|
+
if (deliveryModeHasReceiver(message.header.mode)) {
|
|
2239
2514
|
for (const to of message.header.mode.to) {
|
|
2240
|
-
if (to === from.hashcode())
|
|
2241
|
-
continue;
|
|
2242
|
-
}
|
|
2515
|
+
if (to === from.hashcode()) continue;
|
|
2243
2516
|
messageToSet.add(to);
|
|
2244
2517
|
|
|
2245
2518
|
if (!relayed && !this.healthChecks.has(to)) {
|
|
@@ -2273,29 +2546,25 @@ export abstract class DirectStream<
|
|
|
2273
2546
|
const willGetAllAcknowledgements = !relayed; // Only the origin will get all acks
|
|
2274
2547
|
|
|
2275
2548
|
// Expected to receive at least 'filterMessageForSeenCounter' acknowledgements from each peer
|
|
2276
|
-
const filterMessageForSeenCounter = relayed
|
|
2277
|
-
? undefined
|
|
2278
|
-
: message.header.mode instanceof SeekDelivery
|
|
2279
|
-
? Math.min(this.peers.size, message.header.mode.redundancy)
|
|
2280
|
-
: 1; /* message.deliveryMode instanceof SeekDelivery ? Math.min(this.peers.size - (relayed ? 1 : 0), message.deliveryMode.redundancy) : 1 */
|
|
2549
|
+
const filterMessageForSeenCounter = relayed ? undefined : 1;
|
|
2281
2550
|
|
|
2282
2551
|
const uniqueAcks = new Set();
|
|
2283
2552
|
const session = +new Date();
|
|
2284
2553
|
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
2554
|
+
const onUnreachable =
|
|
2555
|
+
!relayed &&
|
|
2556
|
+
((ev: any) => {
|
|
2557
|
+
const target = ev.detail.hashcode();
|
|
2558
|
+
if (messageToSet.has(target)) {
|
|
2559
|
+
// A peer can transiently appear unreachable while routes are being updated.
|
|
2560
|
+
// Keep waiting for ACK/timeout instead of failing delivery immediately.
|
|
2561
|
+
logger.trace(
|
|
2562
|
+
"peer unreachable during delivery (msg=%s, target=%s); waiting for ACK/timeout",
|
|
2563
|
+
idString,
|
|
2564
|
+
target,
|
|
2565
|
+
);
|
|
2566
|
+
}
|
|
2567
|
+
});
|
|
2299
2568
|
|
|
2300
2569
|
onUnreachable && this.addEventListener("peer:unreachable", onUnreachable);
|
|
2301
2570
|
|
|
@@ -2308,8 +2577,8 @@ export abstract class DirectStream<
|
|
|
2308
2577
|
onAbort && signal?.removeEventListener("abort", onAbort);
|
|
2309
2578
|
};
|
|
2310
2579
|
|
|
2311
|
-
|
|
2312
|
-
|
|
2580
|
+
const timeout = setTimeout(async () => {
|
|
2581
|
+
clear();
|
|
2313
2582
|
|
|
2314
2583
|
let hasAll = true;
|
|
2315
2584
|
|
|
@@ -2327,22 +2596,31 @@ export abstract class DirectStream<
|
|
|
2327
2596
|
}
|
|
2328
2597
|
}
|
|
2329
2598
|
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
|
|
2599
|
+
if (!hasAll && willGetAllAcknowledgements) {
|
|
2600
|
+
const debugState = this.formatDeliveryDebugState(
|
|
2601
|
+
messageToSet,
|
|
2602
|
+
fastestNodesReached,
|
|
2603
|
+
);
|
|
2604
|
+
const msgMeta = `msgType=${message.constructor.name} dataBytes=${
|
|
2605
|
+
message instanceof DataMessage
|
|
2606
|
+
? (message.data?.byteLength ?? 0)
|
|
2607
|
+
: 0
|
|
2608
|
+
} relayed=${relayed ? 1 : 0}`;
|
|
2609
|
+
deliveryDeferredPromise.reject(
|
|
2610
|
+
new DeliveryError(
|
|
2611
|
+
`Failed to get message ${idString} ${filterMessageForSeenCounter} ${[
|
|
2612
|
+
...messageToSet,
|
|
2613
|
+
]} delivery acknowledges from all nodes (${
|
|
2614
|
+
fastestNodesReached.size
|
|
2615
|
+
}/${messageToSet.size}). Mode: ${
|
|
2616
|
+
message.header.mode.constructor.name
|
|
2617
|
+
}. Redundancy: ${(message.header.mode as any)["redundancy"]}. ${msgMeta}. ${debugState}`,
|
|
2618
|
+
),
|
|
2619
|
+
);
|
|
2342
2620
|
} else {
|
|
2343
2621
|
deliveryDeferredPromise.resolve();
|
|
2344
2622
|
}
|
|
2345
|
-
|
|
2623
|
+
}, this.seekTimeout);
|
|
2346
2624
|
|
|
2347
2625
|
if (signal) {
|
|
2348
2626
|
onAbort = () => {
|
|
@@ -2356,22 +2634,33 @@ export abstract class DirectStream<
|
|
|
2356
2634
|
}
|
|
2357
2635
|
}
|
|
2358
2636
|
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
2637
|
+
const checkDone = () => {
|
|
2638
|
+
// This if clause should never enter for relayed connections, since we don't
|
|
2639
|
+
// know how many ACKs we will get
|
|
2640
|
+
if (
|
|
2641
|
+
filterMessageForSeenCounter != null &&
|
|
2642
|
+
uniqueAcks.size >= messageToSet.size * filterMessageForSeenCounter
|
|
2643
|
+
) {
|
|
2644
|
+
const shouldKeepCallbackForRouteLearning =
|
|
2645
|
+
!relayed &&
|
|
2646
|
+
message.header.mode instanceof AcknowledgeDelivery &&
|
|
2647
|
+
message.header.mode.redundancy > 1;
|
|
2648
|
+
if (haveReceivers && !shouldKeepCallbackForRouteLearning) {
|
|
2649
|
+
// If we have an explicit recipient list we can clear the ACK callback once we
|
|
2650
|
+
// got the expected acknowledgements.
|
|
2651
|
+
clear();
|
|
2652
|
+
} else if (haveReceivers && shouldKeepCallbackForRouteLearning) {
|
|
2653
|
+
// Resolve delivery early, but keep ACK callbacks alive until timeout so we can
|
|
2654
|
+
// learn additional redundant routes (seenCounter=1..redundancy-1).
|
|
2655
|
+
onUnreachable &&
|
|
2656
|
+
this.removeEventListener("peer:unreachable", onUnreachable);
|
|
2657
|
+
onAbort && signal?.removeEventListener("abort", onAbort);
|
|
2658
|
+
onAbort = undefined;
|
|
2659
|
+
}
|
|
2371
2660
|
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2661
|
+
deliveryDeferredPromise.resolve();
|
|
2662
|
+
return true;
|
|
2663
|
+
}
|
|
2375
2664
|
return false;
|
|
2376
2665
|
};
|
|
2377
2666
|
|
|
@@ -2385,26 +2674,39 @@ export abstract class DirectStream<
|
|
|
2385
2674
|
// remove the automatic removal of route timeout since we have observed lifesigns of a peer
|
|
2386
2675
|
this.clearHealthcheckTimer(messageTargetHash);
|
|
2387
2676
|
|
|
2388
|
-
//
|
|
2389
|
-
//
|
|
2390
|
-
|
|
2677
|
+
// If the target is not inside the original message `to`, we can still add the target to our routes.
|
|
2678
|
+
// This matters because relays may modify `to`, and because "ack-anywhere" style probes intentionally
|
|
2679
|
+
// do not provide an explicit recipient list.
|
|
2680
|
+
if (
|
|
2681
|
+
message.header.mode instanceof AcknowledgeDelivery ||
|
|
2682
|
+
message.header.mode instanceof AcknowledgeAnyWhere
|
|
2683
|
+
) {
|
|
2684
|
+
const upstreamHash = messageFrom?.publicKey.hashcode();
|
|
2685
|
+
|
|
2686
|
+
const routeUpdate = computeSeekAckRouteUpdate({
|
|
2687
|
+
current: this.publicKeyHash,
|
|
2688
|
+
upstream: upstreamHash,
|
|
2689
|
+
downstream: messageThrough.publicKey.hashcode(),
|
|
2690
|
+
target: messageTargetHash,
|
|
2691
|
+
// Route "distance" is based on recipient-seen order (0 = fastest). This is relied upon by
|
|
2692
|
+
// `Routes.getFanout(...)` which uses `distance < redundancy` to select redundant next-hops.
|
|
2693
|
+
distance: seenCounter,
|
|
2694
|
+
});
|
|
2695
|
+
|
|
2391
2696
|
this.addRouteConnection(
|
|
2392
|
-
|
|
2393
|
-
|
|
2697
|
+
routeUpdate.from,
|
|
2698
|
+
routeUpdate.neighbour,
|
|
2394
2699
|
messageTarget,
|
|
2395
|
-
|
|
2700
|
+
routeUpdate.distance,
|
|
2396
2701
|
session,
|
|
2397
2702
|
Number(ack.header.session),
|
|
2398
|
-
);
|
|
2703
|
+
);
|
|
2399
2704
|
}
|
|
2400
2705
|
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
seenCounter < filterMessageForSeenCounter
|
|
2406
|
-
) {
|
|
2407
|
-
// TODO set limit correctly
|
|
2706
|
+
if (messageToSet.has(messageTargetHash)) {
|
|
2707
|
+
// Any ACK from the target should satisfy delivery semantics.
|
|
2708
|
+
// Relying on only seenCounter=0 can fail under churn if the first ACK
|
|
2709
|
+
// is lost but a later ACK (seenCounter>0) arrives.
|
|
2408
2710
|
if (seenCounter < MAX_ROUTE_DISTANCE) {
|
|
2409
2711
|
let arr = fastestNodesReached.get(messageTargetHash);
|
|
2410
2712
|
if (!arr) {
|
|
@@ -2413,10 +2715,9 @@ export abstract class DirectStream<
|
|
|
2413
2715
|
}
|
|
2414
2716
|
arr.push(seenCounter);
|
|
2415
2717
|
|
|
2416
|
-
uniqueAcks.add(messageTargetHash
|
|
2718
|
+
uniqueAcks.add(messageTargetHash);
|
|
2417
2719
|
}
|
|
2418
2720
|
}
|
|
2419
|
-
}
|
|
2420
2721
|
|
|
2421
2722
|
checkDone();
|
|
2422
2723
|
},
|
|
@@ -2440,7 +2741,9 @@ export abstract class DirectStream<
|
|
|
2440
2741
|
throw new NotStartedError();
|
|
2441
2742
|
}
|
|
2442
2743
|
|
|
2744
|
+
const isRelayed = relayed ?? from.hashcode() !== this.publicKeyHash;
|
|
2443
2745
|
let delivereyPromise: Promise<void> | undefined = undefined as any;
|
|
2746
|
+
let ackCallbackId: string | undefined;
|
|
2444
2747
|
|
|
2445
2748
|
if (
|
|
2446
2749
|
(!message.header.signatures ||
|
|
@@ -2455,123 +2758,182 @@ export abstract class DirectStream<
|
|
|
2455
2758
|
* Logic for handling acknowledge messages when we receive them (later)
|
|
2456
2759
|
*/
|
|
2457
2760
|
|
|
2458
|
-
if (
|
|
2459
|
-
message instanceof DataMessage &&
|
|
2460
|
-
message.header.mode instanceof SeekDelivery &&
|
|
2461
|
-
!relayed
|
|
2462
|
-
) {
|
|
2463
|
-
to = this.peers; // seek delivery will not work unless we try all possible paths
|
|
2464
|
-
}
|
|
2465
|
-
|
|
2466
|
-
if (message.header.mode instanceof AcknowledgeDelivery) {
|
|
2467
|
-
to = undefined;
|
|
2468
|
-
}
|
|
2469
|
-
|
|
2470
2761
|
if (
|
|
2471
2762
|
(message instanceof DataMessage || message instanceof Goodbye) &&
|
|
2472
|
-
(message.header.mode instanceof
|
|
2473
|
-
message.header.mode instanceof
|
|
2763
|
+
(message.header.mode instanceof AcknowledgeDelivery ||
|
|
2764
|
+
message.header.mode instanceof AcknowledgeAnyWhere)
|
|
2474
2765
|
) {
|
|
2475
2766
|
const deliveryDeferredPromise = await this.createDeliveryPromise(
|
|
2476
2767
|
from,
|
|
2477
2768
|
message,
|
|
2478
|
-
|
|
2769
|
+
isRelayed,
|
|
2479
2770
|
signal,
|
|
2480
2771
|
);
|
|
2481
2772
|
delivereyPromise = deliveryDeferredPromise.promise;
|
|
2773
|
+
ackCallbackId = toBase64(message.id);
|
|
2482
2774
|
}
|
|
2483
2775
|
|
|
2484
|
-
|
|
2776
|
+
try {
|
|
2777
|
+
const bytes = message.bytes();
|
|
2485
2778
|
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
|
|
2489
|
-
|
|
2779
|
+
if (!isRelayed) {
|
|
2780
|
+
const bytesArray = bytes instanceof Uint8Array ? bytes : bytes.subarray();
|
|
2781
|
+
await this.modifySeenCache(bytesArray);
|
|
2782
|
+
}
|
|
2490
2783
|
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
|
|
2784
|
+
/**
|
|
2785
|
+
* For non SEEKing message delivery modes, use routing
|
|
2786
|
+
*/
|
|
2494
2787
|
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2788
|
+
if (message instanceof DataMessage) {
|
|
2789
|
+
if (
|
|
2790
|
+
(message.header.mode instanceof AcknowledgeDelivery ||
|
|
2791
|
+
message.header.mode instanceof SilentDelivery) &&
|
|
2792
|
+
!to
|
|
2793
|
+
) {
|
|
2794
|
+
if (message.header.mode.to.length === 0) {
|
|
2795
|
+
// we definitely know that we should not forward the message anywhere
|
|
2796
|
+
return delivereyPromise;
|
|
2797
|
+
}
|
|
2504
2798
|
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
|
|
2799
|
+
const fanout = this.routes.getFanout(
|
|
2800
|
+
from.hashcode(),
|
|
2801
|
+
message.header.mode.to,
|
|
2802
|
+
message.header.mode.redundancy,
|
|
2803
|
+
);
|
|
2510
2804
|
|
|
2511
|
-
|
|
2512
|
-
|
|
2805
|
+
// If we have explicit routing information, send only along the chosen next-hops.
|
|
2806
|
+
// If `fanout` is empty (no route info yet), fall through to the flooding logic below
|
|
2807
|
+
// so acknowledged deliveries can discover/repair routes instead of timing out.
|
|
2808
|
+
if (fanout && fanout.size > 0) {
|
|
2513
2809
|
const promises: Promise<any>[] = [];
|
|
2810
|
+
const usedNeighbours = new Set<string>();
|
|
2811
|
+
const originalTo = message.header.mode.to;
|
|
2514
2812
|
for (const [neighbour, _distantPeers] of fanout) {
|
|
2515
2813
|
const stream = this.peers.get(neighbour);
|
|
2516
|
-
stream
|
|
2814
|
+
if (!stream) continue;
|
|
2815
|
+
if (message.header.mode instanceof SilentDelivery) {
|
|
2816
|
+
message.header.mode.to = [..._distantPeers.keys()];
|
|
2517
2817
|
promises.push(
|
|
2518
|
-
stream.waitForWrite(bytes, message.header.priority),
|
|
2818
|
+
stream.waitForWrite(message.bytes(), message.header.priority),
|
|
2519
2819
|
);
|
|
2820
|
+
} else {
|
|
2821
|
+
promises.push(stream.waitForWrite(bytes, message.header.priority));
|
|
2822
|
+
}
|
|
2823
|
+
usedNeighbours.add(neighbour);
|
|
2824
|
+
}
|
|
2825
|
+
if (message.header.mode instanceof SilentDelivery) {
|
|
2826
|
+
message.header.mode.to = originalTo;
|
|
2827
|
+
}
|
|
2828
|
+
|
|
2829
|
+
// If the sender requested redundancy but we don't yet have enough distinct
|
|
2830
|
+
// next-hops for the target(s), opportunistically probe additional neighbours.
|
|
2831
|
+
// This replaces the previous "greedy fanout" probing behavior without needing
|
|
2832
|
+
// a separate delivery mode.
|
|
2833
|
+
if (
|
|
2834
|
+
!isRelayed &&
|
|
2835
|
+
message.header.mode instanceof AcknowledgeDelivery &&
|
|
2836
|
+
usedNeighbours.size < message.header.mode.redundancy
|
|
2837
|
+
) {
|
|
2838
|
+
for (const [neighbour, stream] of this.peers) {
|
|
2839
|
+
if (usedNeighbours.size >= message.header.mode.redundancy) {
|
|
2840
|
+
break;
|
|
2841
|
+
}
|
|
2842
|
+
if (usedNeighbours.has(neighbour)) continue;
|
|
2843
|
+
usedNeighbours.add(neighbour);
|
|
2844
|
+
promises.push(stream.waitForWrite(bytes, message.header.priority));
|
|
2845
|
+
}
|
|
2520
2846
|
}
|
|
2847
|
+
|
|
2521
2848
|
await Promise.all(promises);
|
|
2522
|
-
return delivereyPromise;
|
|
2849
|
+
return delivereyPromise;
|
|
2523
2850
|
}
|
|
2524
2851
|
|
|
2525
|
-
|
|
2852
|
+
// If we don't have routing information:
|
|
2853
|
+
// - For acknowledged deliveries, fall through to flooding (route discovery / repair).
|
|
2854
|
+
// - For silent deliveries, relays should not flood (prevents unnecessary fanout); origin may still flood.
|
|
2855
|
+
// We still allow direct neighbour delivery to explicit recipients (if connected).
|
|
2856
|
+
if (isRelayed && message.header.mode instanceof SilentDelivery) {
|
|
2857
|
+
const promises: Promise<any>[] = [];
|
|
2858
|
+
const originalTo = message.header.mode.to;
|
|
2859
|
+
for (const recipient of originalTo) {
|
|
2860
|
+
if (recipient === this.publicKeyHash) continue;
|
|
2861
|
+
if (recipient === from.hashcode()) continue; // never send back to previous hop
|
|
2862
|
+
const stream = this.peers.get(recipient);
|
|
2863
|
+
if (!stream) continue;
|
|
2864
|
+
if (
|
|
2865
|
+
message.header.signatures?.publicKeys.find(
|
|
2866
|
+
(x) => x.hashcode() === recipient,
|
|
2867
|
+
)
|
|
2868
|
+
) {
|
|
2869
|
+
continue; // recipient already signed/seen this message
|
|
2870
|
+
}
|
|
2871
|
+
message.header.mode.to = [recipient];
|
|
2872
|
+
promises.push(
|
|
2873
|
+
stream.waitForWrite(message.bytes(), message.header.priority),
|
|
2874
|
+
);
|
|
2875
|
+
}
|
|
2876
|
+
message.header.mode.to = originalTo;
|
|
2877
|
+
if (promises.length > 0) {
|
|
2878
|
+
await Promise.all(promises);
|
|
2879
|
+
}
|
|
2880
|
+
return delivereyPromise;
|
|
2881
|
+
}
|
|
2526
2882
|
}
|
|
2883
|
+
}
|
|
2527
2884
|
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
|
|
2885
|
+
// We fail to send the message directly, instead fallback to floodsub
|
|
2886
|
+
const peers: PeerStreams[] | Map<string, PeerStreams> = to || this.peers;
|
|
2887
|
+
if (
|
|
2888
|
+
peers == null ||
|
|
2889
|
+
(Array.isArray(peers) && peers.length === 0) ||
|
|
2890
|
+
(peers instanceof Map && peers.size === 0)
|
|
2891
|
+
) {
|
|
2892
|
+
logger.trace("No peers to send to");
|
|
2893
|
+
return delivereyPromise;
|
|
2894
|
+
}
|
|
2534
2895
|
|
|
2535
|
-
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
(Array.isArray(peers) && peers.length === 0) ||
|
|
2540
|
-
(peers instanceof Map && peers.size === 0)
|
|
2541
|
-
) {
|
|
2542
|
-
logger.trace("No peers to send to");
|
|
2543
|
-
return delivereyPromise;
|
|
2544
|
-
}
|
|
2896
|
+
let sentOnce = false;
|
|
2897
|
+
const promises: Promise<any>[] = [];
|
|
2898
|
+
for (const stream of peers.values()) {
|
|
2899
|
+
const id = stream as PeerStreams;
|
|
2545
2900
|
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
|
|
2901
|
+
// Dont sent back to the sender
|
|
2902
|
+
if (id.publicKey.equals(from)) {
|
|
2903
|
+
continue;
|
|
2904
|
+
}
|
|
2905
|
+
// Dont send message back to any of the signers (they have already seen the message)
|
|
2906
|
+
if (
|
|
2907
|
+
message.header.signatures?.publicKeys.find((x) =>
|
|
2908
|
+
x.equals(id.publicKey),
|
|
2909
|
+
)
|
|
2910
|
+
) {
|
|
2911
|
+
continue;
|
|
2912
|
+
}
|
|
2550
2913
|
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
continue;
|
|
2554
|
-
}
|
|
2555
|
-
// Dont send message back to any of the signers (they have already seen the message)
|
|
2556
|
-
if (
|
|
2557
|
-
message.header.signatures?.publicKeys.find((x) =>
|
|
2558
|
-
x.equals(id.publicKey),
|
|
2559
|
-
)
|
|
2560
|
-
) {
|
|
2561
|
-
continue;
|
|
2914
|
+
sentOnce = true;
|
|
2915
|
+
promises.push(id.waitForWrite(bytes, message.header.priority));
|
|
2562
2916
|
}
|
|
2917
|
+
await Promise.all(promises);
|
|
2563
2918
|
|
|
2564
|
-
sentOnce
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
|
|
2919
|
+
if (!sentOnce) {
|
|
2920
|
+
// If the caller provided an explicit peer list, treat "no valid receivers" as an error
|
|
2921
|
+
// even when forwarding. This catches programming mistakes early and matches test expectations.
|
|
2922
|
+
if (!isRelayed || to != null) {
|
|
2923
|
+
throw new DeliveryError("Message did not have any valid receivers");
|
|
2924
|
+
}
|
|
2925
|
+
}
|
|
2568
2926
|
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2927
|
+
return delivereyPromise;
|
|
2928
|
+
} catch (error) {
|
|
2929
|
+
// If message fanout/write fails before publishMessage returns its delivery
|
|
2930
|
+
// promise, clear any ACK callback to avoid late timer rejections leaking as
|
|
2931
|
+
// unhandled rejections in fire-and-forget call paths.
|
|
2932
|
+
if (ackCallbackId) {
|
|
2933
|
+
this._ackCallbacks.get(ackCallbackId)?.clear();
|
|
2572
2934
|
}
|
|
2935
|
+
throw error;
|
|
2573
2936
|
}
|
|
2574
|
-
return delivereyPromise;
|
|
2575
2937
|
}
|
|
2576
2938
|
|
|
2577
2939
|
async maybeConnectDirectly(toHash: string, origin: MultiAddrinfo) {
|
|
@@ -2694,8 +3056,11 @@ export abstract class DirectStream<
|
|
|
2694
3056
|
const wins = new Set<string>();
|
|
2695
3057
|
for (const h of admitted) if (reached(h, target)) wins.add(h);
|
|
2696
3058
|
|
|
2697
|
-
|
|
2698
|
-
|
|
3059
|
+
// Preserve input order in the returned list (important for deterministic callers/tests).
|
|
3060
|
+
const orderedWins = () => admitted.filter((h) => wins.has(h));
|
|
3061
|
+
|
|
3062
|
+
if (settle === "any" && wins.size > 0) return orderedWins();
|
|
3063
|
+
if (settle === "all" && wins.size === admitted.length) return orderedWins();
|
|
2699
3064
|
|
|
2700
3065
|
// Abort/timeout
|
|
2701
3066
|
const abortSignals = [this.closeController.signal];
|
|
@@ -2711,143 +3076,62 @@ export abstract class DirectStream<
|
|
|
2711
3076
|
return defer.resolve();
|
|
2712
3077
|
};
|
|
2713
3078
|
|
|
2714
|
-
try {
|
|
2715
|
-
await waitForEvent(this, eventsFor(target), check, {
|
|
2716
|
-
signals: abortSignals,
|
|
2717
|
-
timeout,
|
|
2718
|
-
});
|
|
2719
|
-
return [...wins];
|
|
2720
|
-
} catch (e) {
|
|
2721
|
-
const abortSignal = abortSignals.find((s) => s.aborted);
|
|
2722
|
-
if (abortSignal) {
|
|
2723
|
-
if (abortSignal.reason instanceof Error) {
|
|
2724
|
-
throw abortSignal.reason;
|
|
2725
|
-
}
|
|
2726
|
-
throw new AbortError(
|
|
2727
|
-
"Aborted waiting for peers: " + abortSignal.reason,
|
|
2728
|
-
);
|
|
2729
|
-
}
|
|
2730
|
-
if (e instanceof Error) {
|
|
2731
|
-
throw e;
|
|
2732
|
-
}
|
|
2733
|
-
if (settle === "all") throw new TimeoutError("Timeout waiting for peers");
|
|
2734
|
-
return [...wins]; // settle:any: return whatever successes we got
|
|
2735
|
-
}
|
|
2736
|
-
}
|
|
2737
|
-
|
|
2738
|
-
/* async waitFor(
|
|
2739
|
-
peer: PeerId | PublicSignKey | string,
|
|
2740
|
-
options?: {
|
|
2741
|
-
timeout?: number;
|
|
2742
|
-
signal?: AbortSignal;
|
|
2743
|
-
neighbour?: boolean;
|
|
2744
|
-
inflight?: boolean;
|
|
2745
|
-
},
|
|
2746
|
-
) {
|
|
2747
|
-
const hash =
|
|
2748
|
-
typeof peer === "string"
|
|
2749
|
-
? peer
|
|
2750
|
-
: (peer instanceof PublicSignKey
|
|
2751
|
-
? peer
|
|
2752
|
-
: getPublicKeyFromPeerId(peer)
|
|
2753
|
-
).hashcode();
|
|
2754
|
-
if (hash === this.publicKeyHash) {
|
|
2755
|
-
return; // TODO throw error instead?
|
|
2756
|
-
}
|
|
2757
|
-
|
|
2758
|
-
if (options?.inflight) {
|
|
2759
|
-
// if peer is not in active connections or dialQueue, return silenty
|
|
2760
|
-
if (
|
|
2761
|
-
!this.peers.has(hash) &&
|
|
2762
|
-
!this.components.connectionManager
|
|
2763
|
-
.getDialQueue()
|
|
2764
|
-
.some((x) => getPublicKeyFromPeerId(x.peerId).hashcode() === hash) &&
|
|
2765
|
-
!this.components.connectionManager
|
|
2766
|
-
.getConnections()
|
|
2767
|
-
.some((x) => getPublicKeyFromPeerId(x.remotePeer).hashcode() === hash)
|
|
2768
|
-
) {
|
|
2769
|
-
return;
|
|
2770
|
-
}
|
|
2771
|
-
}
|
|
2772
|
-
|
|
2773
|
-
const checkIsReachable = (deferred: DeferredPromise<void>) => {
|
|
2774
|
-
if (options?.neighbour && !this.peers.has(hash)) {
|
|
2775
|
-
return;
|
|
2776
|
-
}
|
|
2777
|
-
|
|
2778
|
-
if (!this.routes.isReachable(this.publicKeyHash, hash, 0)) {
|
|
2779
|
-
return;
|
|
2780
|
-
}
|
|
2781
|
-
|
|
2782
|
-
deferred.resolve();
|
|
2783
|
-
};
|
|
2784
|
-
const abortSignals = [this.closeController.signal];
|
|
2785
|
-
if (options?.signal) {
|
|
2786
|
-
abortSignals.push(options.signal);
|
|
2787
|
-
}
|
|
2788
|
-
|
|
2789
|
-
try {
|
|
2790
|
-
await waitForEvent(this, ["peer:reachable"], checkIsReachable, {
|
|
2791
|
-
signals: abortSignals,
|
|
2792
|
-
timeout: options?.timeout,
|
|
2793
|
-
});
|
|
2794
|
-
} catch (error) {
|
|
2795
|
-
throw new Error(
|
|
2796
|
-
"Stream to " +
|
|
2797
|
-
hash +
|
|
2798
|
-
" from " +
|
|
2799
|
-
this.publicKeyHash +
|
|
2800
|
-
" does not exist. Connection exist: " +
|
|
2801
|
-
this.peers.has(hash) +
|
|
2802
|
-
". Route exist: " +
|
|
2803
|
-
this.routes.isReachable(this.publicKeyHash, hash, 0),
|
|
2804
|
-
);
|
|
2805
|
-
}
|
|
2806
|
-
|
|
2807
|
-
if (options?.neighbour) {
|
|
2808
|
-
const stream = this.peers.get(hash)!;
|
|
2809
3079
|
try {
|
|
2810
|
-
|
|
2811
|
-
|
|
2812
|
-
|
|
3080
|
+
await waitForEvent(this, eventsFor(target), check, {
|
|
3081
|
+
signals: abortSignals,
|
|
3082
|
+
timeout,
|
|
3083
|
+
});
|
|
3084
|
+
return orderedWins();
|
|
3085
|
+
} catch (e) {
|
|
3086
|
+
const abortSignal = abortSignals.find((s) => s.aborted);
|
|
3087
|
+
if (abortSignal) {
|
|
3088
|
+
if (abortSignal.reason instanceof Error) {
|
|
3089
|
+
throw abortSignal.reason;
|
|
2813
3090
|
}
|
|
2814
|
-
|
|
2815
|
-
|
|
2816
|
-
|
|
2817
|
-
|
|
2818
|
-
|
|
2819
|
-
{
|
|
2820
|
-
|
|
2821
|
-
|
|
2822
|
-
|
|
2823
|
-
|
|
2824
|
-
|
|
2825
|
-
|
|
2826
|
-
|
|
2827
|
-
|
|
2828
|
-
|
|
2829
|
-
|
|
2830
|
-
|
|
2831
|
-
|
|
2832
|
-
|
|
3091
|
+
throw new AbortError(
|
|
3092
|
+
"Aborted waiting for peers: " + abortSignal.reason,
|
|
3093
|
+
);
|
|
3094
|
+
}
|
|
3095
|
+
if (e instanceof TimeoutError) {
|
|
3096
|
+
if (settle === "any") {
|
|
3097
|
+
if (wins.size > 0) return orderedWins();
|
|
3098
|
+
throw new TimeoutError(
|
|
3099
|
+
`Timeout waiting for peers (target=${target}, seek=${seek}, missing=${admitted.length}/${admitted.length})`,
|
|
3100
|
+
);
|
|
3101
|
+
}
|
|
3102
|
+
const missing = admitted.filter((h) => !wins.has(h));
|
|
3103
|
+
const preview = missing.slice(0, 5).join(", ");
|
|
3104
|
+
throw new TimeoutError(
|
|
3105
|
+
`Timeout waiting for peers (target=${target}, seek=${seek}, missing=${missing.length}/${admitted.length}${
|
|
3106
|
+
preview ? `, e.g. ${preview}` : ""
|
|
3107
|
+
})`,
|
|
3108
|
+
);
|
|
3109
|
+
}
|
|
3110
|
+
if (e instanceof Error) throw e;
|
|
3111
|
+
if (settle === "all") throw new TimeoutError("Timeout waiting for peers");
|
|
3112
|
+
return orderedWins(); // settle:any: return whatever successes we got
|
|
2833
3113
|
}
|
|
2834
3114
|
}
|
|
2835
|
-
} */
|
|
2836
3115
|
|
|
2837
|
-
|
|
2838
|
-
|
|
2839
|
-
|
|
3116
|
+
getPublicKey(hash: string): PublicSignKey | undefined {
|
|
3117
|
+
return this.peerKeyHashToPublicKey.get(hash);
|
|
3118
|
+
}
|
|
2840
3119
|
|
|
2841
3120
|
get pending(): boolean {
|
|
2842
3121
|
return this._ackCallbacks.size > 0;
|
|
2843
3122
|
}
|
|
2844
3123
|
|
|
2845
3124
|
// make this into a job? run every few ms
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
if (this.
|
|
2849
|
-
|
|
2850
|
-
|
|
3125
|
+
maybePruneConnections(): Promise<void> {
|
|
3126
|
+
// Hard cap on peer streams: treat as a primary pruning signal.
|
|
3127
|
+
if (this.peers.size > this.connectionManagerOptions.maxConnections) {
|
|
3128
|
+
return this.pruneConnectionsToLimits();
|
|
3129
|
+
}
|
|
3130
|
+
|
|
3131
|
+
if (this.connectionManagerOptions.pruner) {
|
|
3132
|
+
if (this.connectionManagerOptions.pruner.bandwidth != null) {
|
|
3133
|
+
let usedBandwidth = 0;
|
|
3134
|
+
for (const [_k, v] of this.peers) {
|
|
2851
3135
|
usedBandwidth += v.usedBandwidth;
|
|
2852
3136
|
}
|
|
2853
3137
|
usedBandwidth /= this.peers.size;
|
|
@@ -2856,17 +3140,17 @@ export abstract class DirectStream<
|
|
|
2856
3140
|
// prune
|
|
2857
3141
|
return this.pruneConnections();
|
|
2858
3142
|
}
|
|
2859
|
-
|
|
2860
|
-
|
|
2861
|
-
|
|
2862
|
-
|
|
2863
|
-
|
|
3143
|
+
} else if (this.connectionManagerOptions.pruner.maxBuffer != null) {
|
|
3144
|
+
const queuedBytes = this.getQueuedBytes();
|
|
3145
|
+
if (queuedBytes > this.connectionManagerOptions.pruner.maxBuffer) {
|
|
3146
|
+
// prune
|
|
3147
|
+
return this.pruneConnections();
|
|
3148
|
+
}
|
|
2864
3149
|
}
|
|
2865
3150
|
}
|
|
2866
|
-
}
|
|
2867
3151
|
|
|
2868
|
-
|
|
2869
|
-
|
|
3152
|
+
return Promise.resolve();
|
|
3153
|
+
}
|
|
2870
3154
|
|
|
2871
3155
|
async pruneConnections(): Promise<void> {
|
|
2872
3156
|
// TODO sort by bandwidth
|
|
@@ -2877,17 +3161,17 @@ export abstract class DirectStream<
|
|
|
2877
3161
|
const sorted = [...this.peers.values()]
|
|
2878
3162
|
.sort((x, y) => x.usedBandwidth - y.usedBandwidth)
|
|
2879
3163
|
.map((x) => x.publicKey.hashcode());
|
|
2880
|
-
|
|
2881
|
-
|
|
2882
|
-
|
|
2883
|
-
|
|
3164
|
+
const prunables = this.routes.getPrunable(sorted);
|
|
3165
|
+
if (prunables.length === 0) {
|
|
3166
|
+
return;
|
|
3167
|
+
}
|
|
2884
3168
|
|
|
2885
|
-
|
|
2886
|
-
|
|
3169
|
+
const stream = this.peers.get(prunables[0])!;
|
|
3170
|
+
this.prunedConnectionsCache?.add(stream.publicKey.hashcode());
|
|
2887
3171
|
|
|
2888
|
-
|
|
2889
|
-
|
|
2890
|
-
|
|
3172
|
+
await this.onPeerDisconnected(stream.peerId);
|
|
3173
|
+
return this.components.connectionManager.closeConnections(stream.peerId);
|
|
3174
|
+
}
|
|
2891
3175
|
|
|
2892
3176
|
getQueuedBytes(): number {
|
|
2893
3177
|
let sum = 0;
|