@peerbit/stream 4.6.0-bbf27fa → 4.6.0-c485a73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/benchmark/directstream-sim.d.ts +12 -0
- package/dist/benchmark/directstream-sim.d.ts.map +1 -0
- package/dist/benchmark/directstream-sim.js +299 -0
- package/dist/benchmark/directstream-sim.js.map +1 -0
- package/dist/benchmark/index.d.ts +10 -0
- package/dist/benchmark/index.d.ts.map +1 -0
- package/dist/benchmark/index.js +48 -0
- package/dist/benchmark/index.js.map +1 -0
- package/dist/benchmark/topology-sim.d.ts +12 -0
- package/dist/benchmark/topology-sim.d.ts.map +1 -0
- package/dist/benchmark/topology-sim.js +410 -0
- package/dist/benchmark/topology-sim.js.map +1 -0
- package/dist/benchmark/transfer.js +2 -2
- package/dist/benchmark/transfer.js.map +1 -1
- package/dist/src/core/seek-routing.d.ts +39 -0
- package/dist/src/core/seek-routing.d.ts.map +1 -0
- package/dist/src/core/seek-routing.js +33 -0
- package/dist/src/core/seek-routing.js.map +1 -0
- package/dist/src/index.d.ts +39 -5
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +522 -300
- package/dist/src/index.js.map +1 -1
- package/dist/src/routes.d.ts +15 -0
- package/dist/src/routes.d.ts.map +1 -1
- package/dist/src/routes.js +135 -16
- package/dist/src/routes.js.map +1 -1
- package/dist/src/wait-for-event.d.ts.map +1 -1
- package/dist/src/wait-for-event.js +14 -3
- package/dist/src/wait-for-event.js.map +1 -1
- package/package.json +8 -8
- package/src/core/seek-routing.ts +75 -0
- package/src/index.ts +804 -505
- package/src/routes.ts +158 -18
- package/src/wait-for-event.ts +23 -10
package/src/index.ts
CHANGED
|
@@ -29,6 +29,7 @@ import {
|
|
|
29
29
|
import type { SignatureWithKey } from "@peerbit/crypto";
|
|
30
30
|
import {
|
|
31
31
|
ACK,
|
|
32
|
+
AcknowledgeAnyWhere,
|
|
32
33
|
AcknowledgeDelivery,
|
|
33
34
|
AnyWhere,
|
|
34
35
|
DataMessage,
|
|
@@ -39,7 +40,6 @@ import {
|
|
|
39
40
|
MessageHeader,
|
|
40
41
|
MultiAddrinfo,
|
|
41
42
|
NotStartedError,
|
|
42
|
-
SeekDelivery,
|
|
43
43
|
SilentDelivery,
|
|
44
44
|
TracedDelivery,
|
|
45
45
|
coercePeerRefsToHashes,
|
|
@@ -47,6 +47,7 @@ import {
|
|
|
47
47
|
getMsgId,
|
|
48
48
|
} from "@peerbit/stream-interface";
|
|
49
49
|
import type {
|
|
50
|
+
DirectStreamAckRouteHint,
|
|
50
51
|
IdOptions,
|
|
51
52
|
PeerRefs,
|
|
52
53
|
PriorityOptions,
|
|
@@ -69,6 +70,10 @@ import { type Pushable, pushable } from "it-pushable";
|
|
|
69
70
|
import pDefer, { type DeferredPromise } from "p-defer";
|
|
70
71
|
import Queue from "p-queue";
|
|
71
72
|
import { Uint8ArrayList } from "uint8arraylist";
|
|
73
|
+
import {
|
|
74
|
+
computeSeekAckRouteUpdate,
|
|
75
|
+
shouldAcknowledgeDataMessage,
|
|
76
|
+
} from "./core/seek-routing.js";
|
|
72
77
|
import { logger } from "./logger.js";
|
|
73
78
|
import { type PushableLanes, pushableLanes } from "./pushable-lanes.js";
|
|
74
79
|
import { MAX_ROUTE_DISTANCE, Routes } from "./routes.js";
|
|
@@ -167,8 +172,6 @@ const DEFAULT_MAX_CONNECTIONS = 300;
|
|
|
167
172
|
|
|
168
173
|
const DEFAULT_PRUNED_CONNNECTIONS_TIMEOUT = 30 * 1000;
|
|
169
174
|
|
|
170
|
-
const ROUTE_UPDATE_DELAY_FACTOR = 3e4;
|
|
171
|
-
|
|
172
175
|
const DEFAULT_CREATE_OUTBOUND_STREAM_TIMEOUT = 30_000;
|
|
173
176
|
|
|
174
177
|
const PRIORITY_LANES = 4;
|
|
@@ -261,6 +264,19 @@ export class PeerStreams extends TypedEventEmitter<PeerStreamEvents> {
|
|
|
261
264
|
public _getActiveOutboundPushable(): PushableLanes<Uint8Array> | undefined {
|
|
262
265
|
return this.outboundStreams[0]?.pushable;
|
|
263
266
|
}
|
|
267
|
+
public getOutboundQueuedBytes(): number {
|
|
268
|
+
return this._getActiveOutboundPushable()?.readableLength ?? 0;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
public getOutboundQueuedBytesByLane(): number[] {
|
|
272
|
+
const p = this._getActiveOutboundPushable();
|
|
273
|
+
if (!p) return Array(PRIORITY_LANES).fill(0);
|
|
274
|
+
const out: number[] = [];
|
|
275
|
+
for (let lane = 0; lane < PRIORITY_LANES; lane++) {
|
|
276
|
+
out.push(p.getReadableLength(lane));
|
|
277
|
+
}
|
|
278
|
+
return out;
|
|
279
|
+
}
|
|
264
280
|
public _getOutboundCount() {
|
|
265
281
|
return this.outboundStreams.length;
|
|
266
282
|
}
|
|
@@ -495,7 +511,10 @@ export class PeerStreams extends TypedEventEmitter<PeerStreamEvents> {
|
|
|
495
511
|
return;
|
|
496
512
|
}
|
|
497
513
|
|
|
498
|
-
|
|
514
|
+
// Outbound stream negotiation can legitimately take several seconds in CI
|
|
515
|
+
// (identify/protocol discovery, resource contention, etc). Keep this fairly
|
|
516
|
+
// generous so control-plane messages (joins/subscriptions) don't flap.
|
|
517
|
+
const timeoutMs = 10_000;
|
|
499
518
|
|
|
500
519
|
await new Promise<void>((resolve, reject) => {
|
|
501
520
|
const onOutbound = () => {
|
|
@@ -780,21 +799,29 @@ export class PeerStreams extends TypedEventEmitter<PeerStreamEvents> {
|
|
|
780
799
|
this.outboundAbortController.abort();
|
|
781
800
|
}
|
|
782
801
|
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
802
|
+
// End inbound streams
|
|
803
|
+
if (this.inboundStreams.length) {
|
|
804
|
+
for (const inbound of this.inboundStreams) {
|
|
805
|
+
try {
|
|
806
|
+
inbound.abortController.abort();
|
|
807
|
+
} catch {
|
|
808
|
+
logger.error("Failed to abort inbound stream");
|
|
809
|
+
}
|
|
810
|
+
try {
|
|
811
|
+
// Best-effort shutdown: on some transports (notably websockets),
|
|
812
|
+
// awaiting a graceful close can hang indefinitely if the remote is
|
|
813
|
+
// concurrently stopping. Abort immediately and do not await close.
|
|
814
|
+
try {
|
|
815
|
+
inbound.raw.abort?.(new AbortError("Closed"));
|
|
816
|
+
} catch {
|
|
817
|
+
// ignore
|
|
818
|
+
}
|
|
819
|
+
void Promise.resolve(inbound.raw.close?.()).catch(() => {});
|
|
820
|
+
} catch {
|
|
821
|
+
logger.error("Failed to close inbound stream");
|
|
822
|
+
}
|
|
795
823
|
}
|
|
796
824
|
}
|
|
797
|
-
}
|
|
798
825
|
|
|
799
826
|
this.usedBandWidthTracker.stop();
|
|
800
827
|
|
|
@@ -838,9 +865,27 @@ export type DirectStreamOptions = {
|
|
|
838
865
|
maxOutboundStreams?: number;
|
|
839
866
|
inboundIdleTimeout?: number; // override PeerStreams.INBOUND_IDLE_MS
|
|
840
867
|
connectionManager?: ConnectionManagerArguments;
|
|
841
|
-
routeSeekInterval?: number;
|
|
842
868
|
seekTimeout?: number;
|
|
843
869
|
routeMaxRetentionPeriod?: number;
|
|
870
|
+
/**
|
|
871
|
+
* Best-effort bounds for the per-process route cache. These exist to prevent
|
|
872
|
+
* unbounded memory growth in large networks/simulations.
|
|
873
|
+
*/
|
|
874
|
+
routeCacheMaxFromEntries?: number;
|
|
875
|
+
routeCacheMaxTargetsPerFrom?: number;
|
|
876
|
+
routeCacheMaxRelaysPerTarget?: number;
|
|
877
|
+
/**
|
|
878
|
+
* Share node-level routing/session state across DirectStream instances created
|
|
879
|
+
* from the same libp2p private key.
|
|
880
|
+
*
|
|
881
|
+
* This reduces duplicated topology knowledge when multiple protocols run on
|
|
882
|
+
* the same node (e.g. pubsub + fanout overlays).
|
|
883
|
+
*
|
|
884
|
+
* Defaults to `true`.
|
|
885
|
+
*/
|
|
886
|
+
sharedRouting?: boolean;
|
|
887
|
+
seenCacheMax?: number;
|
|
888
|
+
seenCacheTtlMs?: number;
|
|
844
889
|
};
|
|
845
890
|
|
|
846
891
|
type ConnectionManagerLike = {
|
|
@@ -870,16 +915,25 @@ export interface DirectStreamComponents {
|
|
|
870
915
|
privateKey: PrivateKey;
|
|
871
916
|
}
|
|
872
917
|
|
|
918
|
+
type SharedRoutingState = {
|
|
919
|
+
session: number;
|
|
920
|
+
routes: Routes;
|
|
921
|
+
controller: AbortController;
|
|
922
|
+
refs: number;
|
|
923
|
+
};
|
|
924
|
+
|
|
925
|
+
const sharedRoutingByPrivateKey = new WeakMap<PrivateKey, SharedRoutingState>();
|
|
926
|
+
|
|
873
927
|
export type PublishOptions = (WithMode | WithTo) &
|
|
874
928
|
PriorityOptions &
|
|
875
929
|
WithExtraSigners;
|
|
876
930
|
|
|
877
931
|
export abstract class DirectStream<
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
{
|
|
932
|
+
Events extends { [s: string]: any } = StreamEvents,
|
|
933
|
+
>
|
|
934
|
+
extends TypedEventEmitter<Events>
|
|
935
|
+
implements WaitForPeer, PublicKeyFromHashResolver
|
|
936
|
+
{
|
|
883
937
|
public peerId: PeerId;
|
|
884
938
|
public publicKey: PublicSignKey;
|
|
885
939
|
public publicKeyHash: string;
|
|
@@ -905,26 +959,35 @@ export abstract class DirectStream<
|
|
|
905
959
|
public multicodecs: string[];
|
|
906
960
|
public seenCache: Cache<number>;
|
|
907
961
|
private _registrarTopologyIds: string[] | undefined;
|
|
962
|
+
private _peerConnectListener?: (ev: any) => void;
|
|
963
|
+
private _peerDisconnectListener?: (ev: any) => void;
|
|
908
964
|
private readonly maxInboundStreams?: number;
|
|
909
965
|
private readonly maxOutboundStreams?: number;
|
|
910
966
|
connectionManagerOptions: ConnectionManagerOptions;
|
|
911
967
|
private recentDials?: Cache<string>;
|
|
912
968
|
private healthChecks: Map<string, ReturnType<typeof setTimeout>>;
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
969
|
+
private pruneConnectionsTimeout: ReturnType<typeof setInterval>;
|
|
970
|
+
private prunedConnectionsCache?: Cache<string>;
|
|
971
|
+
private pruneToLimitsInFlight?: Promise<void>;
|
|
972
|
+
private _startInFlight?: Promise<void>;
|
|
973
|
+
private routeMaxRetentionPeriod: number;
|
|
974
|
+
private routeCacheMaxFromEntries?: number;
|
|
975
|
+
private routeCacheMaxTargetsPerFrom?: number;
|
|
976
|
+
private routeCacheMaxRelaysPerTarget?: number;
|
|
977
|
+
private readonly sharedRouting: boolean;
|
|
978
|
+
private sharedRoutingKey?: PrivateKey;
|
|
979
|
+
private sharedRoutingState?: SharedRoutingState;
|
|
916
980
|
|
|
917
981
|
// for sequential creation of outbound streams
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
982
|
+
public outboundInflightQueue: Pushable<{
|
|
983
|
+
connection: Connection;
|
|
984
|
+
peerId: PeerId;
|
|
985
|
+
}>;
|
|
922
986
|
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
_outboundPump: ReturnType<typeof pipe> | undefined;
|
|
987
|
+
seekTimeout: number;
|
|
988
|
+
closeController: AbortController;
|
|
989
|
+
session: number;
|
|
990
|
+
_outboundPump: ReturnType<typeof pipe> | undefined;
|
|
928
991
|
|
|
929
992
|
private _ackCallbacks: Map<
|
|
930
993
|
string,
|
|
@@ -948,14 +1011,19 @@ export abstract class DirectStream<
|
|
|
948
1011
|
const {
|
|
949
1012
|
canRelayMessage = true,
|
|
950
1013
|
messageProcessingConcurrency = 10,
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
1014
|
+
maxInboundStreams,
|
|
1015
|
+
maxOutboundStreams,
|
|
1016
|
+
connectionManager,
|
|
1017
|
+
seekTimeout = SEEK_DELIVERY_TIMEOUT,
|
|
1018
|
+
routeMaxRetentionPeriod = ROUTE_MAX_RETANTION_PERIOD,
|
|
1019
|
+
routeCacheMaxFromEntries,
|
|
1020
|
+
routeCacheMaxTargetsPerFrom,
|
|
1021
|
+
routeCacheMaxRelaysPerTarget,
|
|
1022
|
+
sharedRouting = true,
|
|
1023
|
+
seenCacheMax = 1e6,
|
|
1024
|
+
seenCacheTtlMs = 10 * 60 * 1e3,
|
|
1025
|
+
inboundIdleTimeout,
|
|
1026
|
+
} = options || {};
|
|
959
1027
|
|
|
960
1028
|
const signKey = getKeypairFromPrivateKey(components.privateKey);
|
|
961
1029
|
this.seekTimeout = seekTimeout;
|
|
@@ -969,19 +1037,25 @@ export abstract class DirectStream<
|
|
|
969
1037
|
this.publicKeyHash = signKey.publicKey.hashcode();
|
|
970
1038
|
this.multicodecs = multicodecs;
|
|
971
1039
|
this.started = false;
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
1040
|
+
this.peers = new Map<string, PeerStreams>();
|
|
1041
|
+
this.canRelayMessage = canRelayMessage;
|
|
1042
|
+
this.healthChecks = new Map();
|
|
1043
|
+
this.queue = new Queue({ concurrency: messageProcessingConcurrency });
|
|
1044
|
+
this.maxInboundStreams = maxInboundStreams;
|
|
1045
|
+
this.maxOutboundStreams = maxOutboundStreams;
|
|
1046
|
+
this.seenCache = new Cache({
|
|
1047
|
+
max: Math.max(1, Math.floor(seenCacheMax)),
|
|
1048
|
+
ttl: Math.max(1, Math.floor(seenCacheTtlMs)),
|
|
1049
|
+
});
|
|
1050
|
+
this.routeMaxRetentionPeriod = routeMaxRetentionPeriod;
|
|
1051
|
+
this.routeCacheMaxFromEntries = routeCacheMaxFromEntries;
|
|
1052
|
+
this.routeCacheMaxTargetsPerFrom = routeCacheMaxTargetsPerFrom;
|
|
1053
|
+
this.routeCacheMaxRelaysPerTarget = routeCacheMaxRelaysPerTarget;
|
|
1054
|
+
this.sharedRouting = sharedRouting !== false;
|
|
1055
|
+
this.peerKeyHashToPublicKey = new Map();
|
|
1056
|
+
this._onIncomingStream = this._onIncomingStream.bind(this);
|
|
1057
|
+
this.onPeerConnected = this.onPeerConnected.bind(this);
|
|
1058
|
+
this.onPeerDisconnected = this.onPeerDisconnected.bind(this);
|
|
985
1059
|
|
|
986
1060
|
this._ackCallbacks = new Map();
|
|
987
1061
|
|
|
@@ -1022,23 +1096,60 @@ export abstract class DirectStream<
|
|
|
1022
1096
|
})
|
|
1023
1097
|
: undefined;
|
|
1024
1098
|
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1099
|
+
this.prunedConnectionsCache = this.connectionManagerOptions.pruner
|
|
1100
|
+
? new Cache({
|
|
1101
|
+
max: 1e6,
|
|
1102
|
+
ttl: this.connectionManagerOptions.pruner.connectionTimeout,
|
|
1103
|
+
})
|
|
1104
|
+
: undefined;
|
|
1105
|
+
}
|
|
1032
1106
|
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1107
|
+
private pruneConnectionsToLimits(): Promise<void> {
|
|
1108
|
+
if (this.pruneToLimitsInFlight) {
|
|
1109
|
+
return this.pruneToLimitsInFlight;
|
|
1110
|
+
}
|
|
1111
|
+
this.pruneToLimitsInFlight = (async () => {
|
|
1112
|
+
// Respect minConnections as a hard floor.
|
|
1113
|
+
const maxConnections = Math.max(
|
|
1114
|
+
this.connectionManagerOptions.minConnections,
|
|
1115
|
+
this.connectionManagerOptions.maxConnections,
|
|
1116
|
+
);
|
|
1117
|
+
if (this.peers.size <= maxConnections) {
|
|
1118
|
+
return;
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1121
|
+
// Prune in batches so we can quickly recover from join storms without waiting
|
|
1122
|
+
// for repeated pruner ticks. Bound work per run to avoid starving the event loop.
|
|
1123
|
+
const maxPrunes = Math.min(256, Math.max(1, this.peers.size - maxConnections));
|
|
1124
|
+
for (let i = 0; i < maxPrunes; i++) {
|
|
1125
|
+
if (this.peers.size <= maxConnections) break;
|
|
1126
|
+
const before = this.peers.size;
|
|
1127
|
+
await this.pruneConnections();
|
|
1128
|
+
if (this.peers.size >= before) break; // nothing prunable
|
|
1129
|
+
}
|
|
1130
|
+
})().finally(() => {
|
|
1131
|
+
this.pruneToLimitsInFlight = undefined;
|
|
1132
|
+
});
|
|
1133
|
+
return this.pruneToLimitsInFlight;
|
|
1036
1134
|
}
|
|
1037
1135
|
|
|
1038
|
-
|
|
1039
|
-
|
|
1136
|
+
async start() {
|
|
1137
|
+
if (this.started) return;
|
|
1138
|
+
if (this._startInFlight) return this._startInFlight;
|
|
1139
|
+
this._startInFlight = this._startImpl().finally(() => {
|
|
1140
|
+
this._startInFlight = undefined;
|
|
1141
|
+
});
|
|
1142
|
+
return this._startInFlight;
|
|
1143
|
+
}
|
|
1144
|
+
|
|
1145
|
+
private async _startImpl() {
|
|
1146
|
+
if (this.started) {
|
|
1147
|
+
return;
|
|
1148
|
+
}
|
|
1149
|
+
|
|
1150
|
+
await ready;
|
|
1040
1151
|
|
|
1041
|
-
|
|
1152
|
+
this.closeController = new AbortController();
|
|
1042
1153
|
|
|
1043
1154
|
this.outboundInflightQueue = pushable({ objectMode: true });
|
|
1044
1155
|
|
|
@@ -1116,10 +1227,47 @@ export abstract class DirectStream<
|
|
|
1116
1227
|
this.outboundInflightQueue.return();
|
|
1117
1228
|
});
|
|
1118
1229
|
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1230
|
+
if (this.sharedRouting) {
|
|
1231
|
+
const key = this.components.privateKey;
|
|
1232
|
+
this.sharedRoutingKey = key;
|
|
1233
|
+
let state = sharedRoutingByPrivateKey.get(key);
|
|
1234
|
+
if (!state) {
|
|
1235
|
+
const controller = new AbortController();
|
|
1236
|
+
state = {
|
|
1237
|
+
session: Date.now(),
|
|
1238
|
+
controller,
|
|
1239
|
+
routes: new Routes(this.publicKeyHash, {
|
|
1240
|
+
routeMaxRetentionPeriod: this.routeMaxRetentionPeriod,
|
|
1241
|
+
signal: controller.signal,
|
|
1242
|
+
maxFromEntries: this.routeCacheMaxFromEntries,
|
|
1243
|
+
maxTargetsPerFrom: this.routeCacheMaxTargetsPerFrom,
|
|
1244
|
+
maxRelaysPerTarget: this.routeCacheMaxRelaysPerTarget,
|
|
1245
|
+
}),
|
|
1246
|
+
refs: 0,
|
|
1247
|
+
};
|
|
1248
|
+
sharedRoutingByPrivateKey.set(key, state);
|
|
1249
|
+
} else {
|
|
1250
|
+
// Best-effort: prefer the strictest cleanup policy among co-located protocols.
|
|
1251
|
+
state.routes.routeMaxRetentionPeriod = Math.min(
|
|
1252
|
+
state.routes.routeMaxRetentionPeriod,
|
|
1253
|
+
this.routeMaxRetentionPeriod,
|
|
1254
|
+
);
|
|
1255
|
+
}
|
|
1256
|
+
|
|
1257
|
+
state.refs += 1;
|
|
1258
|
+
this.sharedRoutingState = state;
|
|
1259
|
+
this.session = state.session;
|
|
1260
|
+
this.routes = state.routes;
|
|
1261
|
+
} else {
|
|
1262
|
+
this.session = Date.now();
|
|
1263
|
+
this.routes = new Routes(this.publicKeyHash, {
|
|
1264
|
+
routeMaxRetentionPeriod: this.routeMaxRetentionPeriod,
|
|
1265
|
+
signal: this.closeController.signal,
|
|
1266
|
+
maxFromEntries: this.routeCacheMaxFromEntries,
|
|
1267
|
+
maxTargetsPerFrom: this.routeCacheMaxTargetsPerFrom,
|
|
1268
|
+
maxRelaysPerTarget: this.routeCacheMaxRelaysPerTarget,
|
|
1269
|
+
});
|
|
1270
|
+
}
|
|
1123
1271
|
|
|
1124
1272
|
this.started = true;
|
|
1125
1273
|
this.stopping = false;
|
|
@@ -1149,6 +1297,46 @@ export abstract class DirectStream<
|
|
|
1149
1297
|
),
|
|
1150
1298
|
);
|
|
1151
1299
|
|
|
1300
|
+
// Best-effort fallback: topology callbacks can depend on identify/protocol
|
|
1301
|
+
// discovery. Some test environments connect peers without triggering the
|
|
1302
|
+
// per-protocol topology immediately. Listening to peer connection events
|
|
1303
|
+
// ensures we still attempt to open outbound streams opportunistically.
|
|
1304
|
+
this._peerConnectListener = (ev: any) => {
|
|
1305
|
+
if (this.stopping || !this.started) return;
|
|
1306
|
+
const peerId = (ev as any)?.detail ?? (ev as any)?.peerId;
|
|
1307
|
+
if (!peerId) return;
|
|
1308
|
+
const conns = this.components.connectionManager.getConnections(peerId);
|
|
1309
|
+
if (!conns || conns.length === 0) return;
|
|
1310
|
+
let conn = conns[0]!;
|
|
1311
|
+
for (const c of conns) {
|
|
1312
|
+
if (!isWebsocketConnection(c)) {
|
|
1313
|
+
conn = c;
|
|
1314
|
+
break;
|
|
1315
|
+
}
|
|
1316
|
+
}
|
|
1317
|
+
void this.onPeerConnected(peerId, conn);
|
|
1318
|
+
};
|
|
1319
|
+
this._peerDisconnectListener = (ev: any) => {
|
|
1320
|
+
if (this.stopping || !this.started) return;
|
|
1321
|
+
const peerId = (ev as any)?.detail ?? (ev as any)?.peerId;
|
|
1322
|
+
if (!peerId) return;
|
|
1323
|
+
const conns = this.components.connectionManager.getConnections(peerId);
|
|
1324
|
+
const conn = conns && conns.length > 0 ? conns[0] : undefined;
|
|
1325
|
+
void this.onPeerDisconnected(peerId, conn);
|
|
1326
|
+
};
|
|
1327
|
+
try {
|
|
1328
|
+
this.components.events.addEventListener(
|
|
1329
|
+
"peer:connect",
|
|
1330
|
+
this._peerConnectListener as any,
|
|
1331
|
+
);
|
|
1332
|
+
this.components.events.addEventListener(
|
|
1333
|
+
"peer:disconnect",
|
|
1334
|
+
this._peerDisconnectListener as any,
|
|
1335
|
+
);
|
|
1336
|
+
} catch {
|
|
1337
|
+
// ignore unsupported event targets
|
|
1338
|
+
}
|
|
1339
|
+
|
|
1152
1340
|
// All existing connections are like new ones for us. To deduplication on remotes so we only resuse one connection for this protocol (we could be connected with many connections)
|
|
1153
1341
|
const peerToConnections: Map<string, Connection[]> = new Map();
|
|
1154
1342
|
const connections = this.components.connectionManager.getConnections();
|
|
@@ -1198,7 +1386,28 @@ export abstract class DirectStream<
|
|
|
1198
1386
|
return;
|
|
1199
1387
|
}
|
|
1200
1388
|
|
|
1389
|
+
const sharedState = this.sharedRoutingState;
|
|
1390
|
+
const sharedKey = this.sharedRoutingKey;
|
|
1391
|
+
|
|
1201
1392
|
clearTimeout(this.pruneConnectionsTimeout);
|
|
1393
|
+
try {
|
|
1394
|
+
if (this._peerConnectListener) {
|
|
1395
|
+
this.components.events.removeEventListener(
|
|
1396
|
+
"peer:connect",
|
|
1397
|
+
this._peerConnectListener as any,
|
|
1398
|
+
);
|
|
1399
|
+
}
|
|
1400
|
+
if (this._peerDisconnectListener) {
|
|
1401
|
+
this.components.events.removeEventListener(
|
|
1402
|
+
"peer:disconnect",
|
|
1403
|
+
this._peerDisconnectListener as any,
|
|
1404
|
+
);
|
|
1405
|
+
}
|
|
1406
|
+
} catch {
|
|
1407
|
+
// ignore unsupported event targets
|
|
1408
|
+
}
|
|
1409
|
+
this._peerConnectListener = undefined;
|
|
1410
|
+
this._peerDisconnectListener = undefined;
|
|
1202
1411
|
|
|
1203
1412
|
await Promise.all(
|
|
1204
1413
|
this.multicodecs.map((x) => this.components.registrar.unhandle(x)),
|
|
@@ -1232,7 +1441,11 @@ export abstract class DirectStream<
|
|
|
1232
1441
|
this.queue.clear();
|
|
1233
1442
|
this.peers.clear();
|
|
1234
1443
|
this.seenCache.clear();
|
|
1235
|
-
|
|
1444
|
+
// When routing is shared across co-located protocols, only clear once the last
|
|
1445
|
+
// instance stops. Otherwise we'd wipe routes still in use by other services.
|
|
1446
|
+
if (!sharedState) {
|
|
1447
|
+
this.routes.clear();
|
|
1448
|
+
}
|
|
1236
1449
|
this.peerKeyHashToPublicKey.clear();
|
|
1237
1450
|
|
|
1238
1451
|
for (const [_k, v] of this._ackCallbacks) {
|
|
@@ -1240,6 +1453,24 @@ export abstract class DirectStream<
|
|
|
1240
1453
|
}
|
|
1241
1454
|
|
|
1242
1455
|
this._ackCallbacks.clear();
|
|
1456
|
+
this.sharedRoutingState = undefined;
|
|
1457
|
+
this.sharedRoutingKey = undefined;
|
|
1458
|
+
if (sharedState && sharedKey) {
|
|
1459
|
+
sharedState.refs = Math.max(0, sharedState.refs - 1);
|
|
1460
|
+
if (sharedState.refs === 0) {
|
|
1461
|
+
try {
|
|
1462
|
+
sharedState.routes.clear();
|
|
1463
|
+
} catch {
|
|
1464
|
+
// ignore
|
|
1465
|
+
}
|
|
1466
|
+
try {
|
|
1467
|
+
sharedState.controller.abort();
|
|
1468
|
+
} catch {
|
|
1469
|
+
// ignore
|
|
1470
|
+
}
|
|
1471
|
+
sharedRoutingByPrivateKey.delete(sharedKey);
|
|
1472
|
+
}
|
|
1473
|
+
}
|
|
1243
1474
|
logger.trace("stopped");
|
|
1244
1475
|
this.stopping = false;
|
|
1245
1476
|
}
|
|
@@ -1436,7 +1667,8 @@ export abstract class DirectStream<
|
|
|
1436
1667
|
).catch(dontThrowIfDeliveryError);
|
|
1437
1668
|
}
|
|
1438
1669
|
|
|
1439
|
-
|
|
1670
|
+
// Best-effort liveness probe; never let background probe failures crash callers.
|
|
1671
|
+
void this.checkIsAlive([peerKeyHash]).catch(() => false);
|
|
1440
1672
|
}
|
|
1441
1673
|
|
|
1442
1674
|
logger.trace("connection ended:" + peerKey.toString());
|
|
@@ -1461,27 +1693,31 @@ export abstract class DirectStream<
|
|
|
1461
1693
|
distance: number,
|
|
1462
1694
|
session: number,
|
|
1463
1695
|
remoteSession: number,
|
|
1464
|
-
|
|
1465
|
-
|
|
1696
|
+
) {
|
|
1697
|
+
const targetHash = typeof target === "string" ? target : target.hashcode();
|
|
1698
|
+
// Best-effort: keep a hash -> public key map for any routed targets so
|
|
1699
|
+
// peer:unreachable events can always carry a PublicSignKey when we have seen it.
|
|
1700
|
+
if (typeof target !== "string") {
|
|
1701
|
+
this.peerKeyHashToPublicKey.set(targetHash, target);
|
|
1702
|
+
}
|
|
1466
1703
|
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1704
|
+
const update = this.routes.add(
|
|
1705
|
+
from,
|
|
1706
|
+
neighbour,
|
|
1707
|
+
targetHash,
|
|
1471
1708
|
distance,
|
|
1472
1709
|
session,
|
|
1473
1710
|
remoteSession,
|
|
1474
1711
|
);
|
|
1475
1712
|
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1713
|
+
// second condition is that we don't want to emit 'reachable' events for routes where we act only as a relay
|
|
1714
|
+
// in this case, from is != this.publicKeyhash
|
|
1715
|
+
if (from === this.publicKeyHash) {
|
|
1716
|
+
if (update === "new") {
|
|
1717
|
+
this.onPeerReachable(target);
|
|
1718
|
+
}
|
|
1482
1719
|
}
|
|
1483
1720
|
}
|
|
1484
|
-
}
|
|
1485
1721
|
|
|
1486
1722
|
public onPeerReachable(publicKey: PublicSignKey) {
|
|
1487
1723
|
// override this fn
|
|
@@ -1490,16 +1726,19 @@ export abstract class DirectStream<
|
|
|
1490
1726
|
);
|
|
1491
1727
|
}
|
|
1492
1728
|
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
}
|
|
1501
|
-
|
|
1502
|
-
|
|
1729
|
+
public onPeerUnreachable(hash: string) {
|
|
1730
|
+
// override this fns
|
|
1731
|
+
const key = this.peerKeyHashToPublicKey.get(hash);
|
|
1732
|
+
if (!key) {
|
|
1733
|
+
// Best-effort: we may only have the hash (no public key) for some routes.
|
|
1734
|
+
// Avoid crashing downstream listeners that assume `detail` is a PublicSignKey.
|
|
1735
|
+
return;
|
|
1736
|
+
}
|
|
1737
|
+
this.dispatchEvent(
|
|
1738
|
+
// TODO types
|
|
1739
|
+
new CustomEvent("peer:unreachable", { detail: key }),
|
|
1740
|
+
);
|
|
1741
|
+
}
|
|
1503
1742
|
|
|
1504
1743
|
public updateSession(key: PublicSignKey, session?: number) {
|
|
1505
1744
|
if (this.routes.updateSession(key.hashcode(), session)) {
|
|
@@ -1510,6 +1749,20 @@ export abstract class DirectStream<
|
|
|
1510
1749
|
this.routes.updateSession(key, undefined);
|
|
1511
1750
|
}
|
|
1512
1751
|
|
|
1752
|
+
public getRouteHints(
|
|
1753
|
+
target: string,
|
|
1754
|
+
from: string = this.publicKeyHash,
|
|
1755
|
+
): DirectStreamAckRouteHint[] {
|
|
1756
|
+
return this.routes.getRouteHints(from, target);
|
|
1757
|
+
}
|
|
1758
|
+
|
|
1759
|
+
public getBestRouteHint(
|
|
1760
|
+
target: string,
|
|
1761
|
+
from: string = this.publicKeyHash,
|
|
1762
|
+
): DirectStreamAckRouteHint | undefined {
|
|
1763
|
+
return this.routes.getBestRouteHint(from, target);
|
|
1764
|
+
}
|
|
1765
|
+
|
|
1513
1766
|
public onPeerSession(key: PublicSignKey, session: number) {
|
|
1514
1767
|
this.dispatchEvent(
|
|
1515
1768
|
// TODO types
|
|
@@ -1522,11 +1775,11 @@ export abstract class DirectStream<
|
|
|
1522
1775
|
/**
|
|
1523
1776
|
* Notifies the router that a peer has been connected
|
|
1524
1777
|
*/
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1778
|
+
addPeer(
|
|
1779
|
+
peerId: PeerId,
|
|
1780
|
+
publicKey: PublicSignKey,
|
|
1781
|
+
protocol: string,
|
|
1782
|
+
connId: string,
|
|
1530
1783
|
): PeerStreams {
|
|
1531
1784
|
const publicKeyHash = publicKey.hashcode();
|
|
1532
1785
|
|
|
@@ -1574,17 +1827,23 @@ export abstract class DirectStream<
|
|
|
1574
1827
|
{ once: true },
|
|
1575
1828
|
);
|
|
1576
1829
|
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1830
|
+
this.addRouteConnection(
|
|
1831
|
+
this.publicKeyHash,
|
|
1832
|
+
publicKey.hashcode(),
|
|
1833
|
+
publicKey,
|
|
1834
|
+
-1,
|
|
1835
|
+
+new Date(),
|
|
1836
|
+
-1,
|
|
1837
|
+
);
|
|
1585
1838
|
|
|
1586
|
-
|
|
1587
|
-
|
|
1839
|
+
// Enforce connection manager limits eagerly when new peers are added. Without this,
|
|
1840
|
+
// join storms can create large temporary peer sets and OOM in single-process sims.
|
|
1841
|
+
if (this.peers.size > this.connectionManagerOptions.maxConnections) {
|
|
1842
|
+
void this.pruneConnectionsToLimits().catch(() => {});
|
|
1843
|
+
}
|
|
1844
|
+
|
|
1845
|
+
return peerStreams;
|
|
1846
|
+
}
|
|
1588
1847
|
|
|
1589
1848
|
/**
|
|
1590
1849
|
* Notifies the router that a peer has been disconnected
|
|
@@ -1727,24 +1986,25 @@ export abstract class DirectStream<
|
|
|
1727
1986
|
}
|
|
1728
1987
|
|
|
1729
1988
|
public shouldIgnore(message: DataMessage, seenBefore: number) {
|
|
1730
|
-
const
|
|
1731
|
-
|
|
1732
|
-
|
|
1989
|
+
const signedBySelf =
|
|
1990
|
+
message.header.signatures?.publicKeys.some((x) =>
|
|
1991
|
+
x.equals(this.publicKey),
|
|
1992
|
+
) ?? false;
|
|
1733
1993
|
|
|
1734
|
-
if (
|
|
1994
|
+
if (signedBySelf) {
|
|
1735
1995
|
return true;
|
|
1736
1996
|
}
|
|
1737
1997
|
|
|
1998
|
+
// For acknowledged modes, allow limited duplicate forwarding so that we can
|
|
1999
|
+
// discover and maintain multiple candidate routes (distance=seenCounter).
|
|
1738
2000
|
if (
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
(message.header.mode instanceof SeekDelivery &&
|
|
1742
|
-
seenBefore >= message.header.mode.redundancy)
|
|
2001
|
+
message.header.mode instanceof AcknowledgeDelivery ||
|
|
2002
|
+
message.header.mode instanceof AcknowledgeAnyWhere
|
|
1743
2003
|
) {
|
|
1744
|
-
return
|
|
2004
|
+
return seenBefore >= message.header.mode.redundancy;
|
|
1745
2005
|
}
|
|
1746
2006
|
|
|
1747
|
-
return
|
|
2007
|
+
return seenBefore > 0;
|
|
1748
2008
|
}
|
|
1749
2009
|
|
|
1750
2010
|
public async onDataMessage(
|
|
@@ -1758,14 +2018,19 @@ export abstract class DirectStream<
|
|
|
1758
2018
|
}
|
|
1759
2019
|
|
|
1760
2020
|
let isForMe = false;
|
|
1761
|
-
if (
|
|
2021
|
+
if (
|
|
2022
|
+
message.header.mode instanceof AnyWhere ||
|
|
2023
|
+
message.header.mode instanceof AcknowledgeAnyWhere
|
|
2024
|
+
) {
|
|
1762
2025
|
isForMe = true;
|
|
1763
2026
|
} else {
|
|
1764
2027
|
const isFromSelf = this.publicKey.equals(from);
|
|
1765
|
-
if (
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
message.header.mode
|
|
2028
|
+
if (
|
|
2029
|
+
!isFromSelf &&
|
|
2030
|
+
(message.header.mode instanceof SilentDelivery ||
|
|
2031
|
+
message.header.mode instanceof AcknowledgeDelivery)
|
|
2032
|
+
) {
|
|
2033
|
+
isForMe = message.header.mode.to.includes(this.publicKeyHash);
|
|
1769
2034
|
}
|
|
1770
2035
|
}
|
|
1771
2036
|
|
|
@@ -1801,25 +2066,17 @@ export abstract class DirectStream<
|
|
|
1801
2066
|
}
|
|
1802
2067
|
}
|
|
1803
2068
|
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
) && x !== peerStream,
|
|
1814
|
-
);
|
|
1815
|
-
if (to.length > 0) {
|
|
1816
|
-
this.relayMessage(from, message, to);
|
|
1817
|
-
}
|
|
1818
|
-
}
|
|
1819
|
-
} else {
|
|
2069
|
+
// Forward
|
|
2070
|
+
const shouldForward =
|
|
2071
|
+
seenBefore === 0 ||
|
|
2072
|
+
((message.header.mode instanceof AcknowledgeDelivery ||
|
|
2073
|
+
message.header.mode instanceof AcknowledgeAnyWhere) &&
|
|
2074
|
+
seenBefore < message.header.mode.redundancy);
|
|
2075
|
+
|
|
2076
|
+
if (shouldForward) {
|
|
2077
|
+
// DONT await this since it might introduce a dead-lock
|
|
1820
2078
|
this.relayMessage(from, message);
|
|
1821
2079
|
}
|
|
1822
|
-
}
|
|
1823
2080
|
}
|
|
1824
2081
|
|
|
1825
2082
|
public async verifyAndProcess(message: Message<any>) {
|
|
@@ -1843,14 +2100,20 @@ export abstract class DirectStream<
|
|
|
1843
2100
|
seenBefore: number,
|
|
1844
2101
|
) {
|
|
1845
2102
|
if (
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
seenBefore < message.header.mode.redundancy
|
|
2103
|
+
message.header.mode instanceof AcknowledgeDelivery ||
|
|
2104
|
+
message.header.mode instanceof AcknowledgeAnyWhere
|
|
1849
2105
|
) {
|
|
1850
|
-
const
|
|
1851
|
-
message.header.mode
|
|
1852
|
-
|
|
1853
|
-
|
|
2106
|
+
const isRecipient =
|
|
2107
|
+
message.header.mode instanceof AcknowledgeAnyWhere
|
|
2108
|
+
? true
|
|
2109
|
+
: message.header.mode.to.includes(this.publicKeyHash);
|
|
2110
|
+
if (
|
|
2111
|
+
!shouldAcknowledgeDataMessage({
|
|
2112
|
+
isRecipient,
|
|
2113
|
+
seenBefore,
|
|
2114
|
+
redundancy: message.header.mode.redundancy,
|
|
2115
|
+
})
|
|
2116
|
+
) {
|
|
1854
2117
|
return;
|
|
1855
2118
|
}
|
|
1856
2119
|
const signers = message.header.signatures!.publicKeys.map((x) =>
|
|
@@ -1867,14 +2130,18 @@ export abstract class DirectStream<
|
|
|
1867
2130
|
mode: new TracedDelivery(signers),
|
|
1868
2131
|
session: this.session,
|
|
1869
2132
|
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
2133
|
+
// include our origin for route-learning/dialer hints (best-effort privacy/anti-spam control):
|
|
2134
|
+
// only include once (seenBefore=0) and only if we have not recently pruned
|
|
2135
|
+
// a connection to any signer in the path
|
|
2136
|
+
origin:
|
|
2137
|
+
(message.header.mode instanceof AcknowledgeAnyWhere ||
|
|
2138
|
+
message.header.mode instanceof AcknowledgeDelivery) &&
|
|
2139
|
+
seenBefore === 0 &&
|
|
2140
|
+
!message.header.signatures!.publicKeys.find((x) =>
|
|
2141
|
+
this.prunedConnectionsCache?.has(x.hashcode()),
|
|
2142
|
+
)
|
|
2143
|
+
? new MultiAddrinfo(
|
|
2144
|
+
this.components.addressManager
|
|
1878
2145
|
.getAddresses()
|
|
1879
2146
|
.map((x) => x.toString()),
|
|
1880
2147
|
)
|
|
@@ -2017,33 +2284,42 @@ export abstract class DirectStream<
|
|
|
2017
2284
|
for (const remote of remotes) {
|
|
2018
2285
|
this.invalidateSession(remote);
|
|
2019
2286
|
}
|
|
2020
|
-
|
|
2287
|
+
// Best-effort liveness probe; never let background probe failures crash callers.
|
|
2288
|
+
void this.checkIsAlive(remotes).catch(() => false);
|
|
2021
2289
|
}
|
|
2022
2290
|
private async checkIsAlive(remotes: string[]) {
|
|
2023
2291
|
if (this.peers.size === 0) {
|
|
2024
2292
|
return false;
|
|
2025
2293
|
}
|
|
2026
2294
|
if (remotes.length > 0) {
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2295
|
+
try {
|
|
2296
|
+
// Keepalive probes are best-effort, but must still require ACKs so we can
|
|
2297
|
+
// conclusively prune stale routes when peers actually went away.
|
|
2298
|
+
await this.publish(undefined, {
|
|
2299
|
+
mode: new AcknowledgeDelivery({
|
|
2300
|
+
to: remotes,
|
|
2301
|
+
redundancy: DEFAULT_SILENT_MESSAGE_REDUDANCY,
|
|
2302
|
+
}),
|
|
2303
|
+
});
|
|
2304
|
+
return true;
|
|
2305
|
+
} catch (e: any) {
|
|
2306
|
+
const errorName =
|
|
2307
|
+
e?.name ?? e?.constructor?.name ?? "UnknownError";
|
|
2308
|
+
if (
|
|
2309
|
+
e instanceof DeliveryError ||
|
|
2310
|
+
e instanceof NotStartedError ||
|
|
2311
|
+
e instanceof TimeoutError ||
|
|
2312
|
+
e instanceof AbortError ||
|
|
2313
|
+
errorName === "DeliveryError" ||
|
|
2314
|
+
errorName === "NotStartedError" ||
|
|
2315
|
+
errorName === "TimeoutError" ||
|
|
2316
|
+
errorName === "AbortError"
|
|
2317
|
+
) {
|
|
2318
|
+
return false;
|
|
2319
|
+
}
|
|
2320
|
+
warn(`checkIsAlive unexpected error: ${errorName}`);
|
|
2321
|
+
return false;
|
|
2322
|
+
}
|
|
2047
2323
|
}
|
|
2048
2324
|
return false;
|
|
2049
2325
|
}
|
|
@@ -2056,7 +2332,11 @@ export abstract class DirectStream<
|
|
|
2056
2332
|
) {
|
|
2057
2333
|
// dispatch the event if we are interested
|
|
2058
2334
|
|
|
2059
|
-
let mode:
|
|
2335
|
+
let mode:
|
|
2336
|
+
| SilentDelivery
|
|
2337
|
+
| AcknowledgeDelivery
|
|
2338
|
+
| AcknowledgeAnyWhere
|
|
2339
|
+
| AnyWhere = (
|
|
2060
2340
|
options as WithMode
|
|
2061
2341
|
).mode
|
|
2062
2342
|
? (options as WithMode).mode!
|
|
@@ -2067,8 +2347,7 @@ export abstract class DirectStream<
|
|
|
2067
2347
|
|
|
2068
2348
|
if (
|
|
2069
2349
|
mode instanceof AcknowledgeDelivery ||
|
|
2070
|
-
mode instanceof SilentDelivery
|
|
2071
|
-
mode instanceof SeekDelivery
|
|
2350
|
+
mode instanceof SilentDelivery
|
|
2072
2351
|
) {
|
|
2073
2352
|
if (mode.to) {
|
|
2074
2353
|
let preLength = mode.to.length;
|
|
@@ -2080,7 +2359,7 @@ export abstract class DirectStream<
|
|
|
2080
2359
|
);
|
|
2081
2360
|
}
|
|
2082
2361
|
|
|
2083
|
-
if (mode.to.length === 0
|
|
2362
|
+
if (mode.to.length === 0) {
|
|
2084
2363
|
throw new InvalidMessageError(
|
|
2085
2364
|
"Unexpected to deliver message with mode: " +
|
|
2086
2365
|
mode.constructor.name +
|
|
@@ -2091,28 +2370,6 @@ export abstract class DirectStream<
|
|
|
2091
2370
|
}
|
|
2092
2371
|
}
|
|
2093
2372
|
|
|
2094
|
-
if (mode instanceof AcknowledgeDelivery || mode instanceof SilentDelivery) {
|
|
2095
|
-
const now = +new Date();
|
|
2096
|
-
for (const hash of mode.to) {
|
|
2097
|
-
const neighbourRoutes = this.routes.routes
|
|
2098
|
-
.get(this.publicKeyHash)
|
|
2099
|
-
?.get(hash);
|
|
2100
|
-
|
|
2101
|
-
if (
|
|
2102
|
-
!neighbourRoutes ||
|
|
2103
|
-
now - neighbourRoutes.session >
|
|
2104
|
-
neighbourRoutes.list.length * this.routeSeekInterval ||
|
|
2105
|
-
!this.routes.isUpToDate(hash, neighbourRoutes)
|
|
2106
|
-
) {
|
|
2107
|
-
mode = new SeekDelivery({
|
|
2108
|
-
to: mode.to,
|
|
2109
|
-
redundancy: DEFAULT_SEEK_MESSAGE_REDUDANCY,
|
|
2110
|
-
});
|
|
2111
|
-
break;
|
|
2112
|
-
}
|
|
2113
|
-
}
|
|
2114
|
-
}
|
|
2115
|
-
|
|
2116
2373
|
const message = new DataMessage({
|
|
2117
2374
|
data: data instanceof Uint8ArrayList ? data.subarray() : data,
|
|
2118
2375
|
header: new MessageHeader({
|
|
@@ -2139,7 +2396,9 @@ export abstract class DirectStream<
|
|
|
2139
2396
|
async publish(
|
|
2140
2397
|
data: Uint8Array | Uint8ArrayList | undefined,
|
|
2141
2398
|
options: PublishOptions = {
|
|
2142
|
-
mode: new
|
|
2399
|
+
mode: new AcknowledgeAnyWhere({
|
|
2400
|
+
redundancy: DEFAULT_SEEK_MESSAGE_REDUDANCY,
|
|
2401
|
+
}),
|
|
2143
2402
|
},
|
|
2144
2403
|
): Promise<Uint8Array | undefined> {
|
|
2145
2404
|
if (!this.started) {
|
|
@@ -2184,7 +2443,7 @@ export abstract class DirectStream<
|
|
|
2184
2443
|
if (message instanceof DataMessage) {
|
|
2185
2444
|
if (
|
|
2186
2445
|
message.header.mode instanceof AcknowledgeDelivery ||
|
|
2187
|
-
message.header.mode instanceof
|
|
2446
|
+
message.header.mode instanceof AcknowledgeAnyWhere
|
|
2188
2447
|
) {
|
|
2189
2448
|
await message.sign(this.sign);
|
|
2190
2449
|
}
|
|
@@ -2212,6 +2471,37 @@ export abstract class DirectStream<
|
|
|
2212
2471
|
this.healthChecks.delete(to);
|
|
2213
2472
|
}
|
|
2214
2473
|
|
|
2474
|
+
private formatDeliveryDebugState(
|
|
2475
|
+
targets: Set<string>,
|
|
2476
|
+
fastestNodesReached: Map<string, number[]>,
|
|
2477
|
+
): string {
|
|
2478
|
+
const sampleSize = 16;
|
|
2479
|
+
const sampledTargets = [...targets].slice(0, sampleSize);
|
|
2480
|
+
const sampled = sampledTargets.map((target) => {
|
|
2481
|
+
const route = this.routes.findNeighbor(this.publicKeyHash, target);
|
|
2482
|
+
const routeSample =
|
|
2483
|
+
route?.list
|
|
2484
|
+
.slice(0, 3)
|
|
2485
|
+
.map((r) => `${r.hash}:${r.distance}${r.expireAt ? ":old" : ""}`)
|
|
2486
|
+
.join(",") ?? "-";
|
|
2487
|
+
const reachable = this.routes.isReachable(this.publicKeyHash, target);
|
|
2488
|
+
const stream = this.peers.get(target);
|
|
2489
|
+
let openConnections = 0;
|
|
2490
|
+
if (stream) {
|
|
2491
|
+
try {
|
|
2492
|
+
openConnections = this.components.connectionManager.getConnections(
|
|
2493
|
+
stream.peerId,
|
|
2494
|
+
).length;
|
|
2495
|
+
} catch {
|
|
2496
|
+
openConnections = stream.isReadable || stream.isWritable ? 1 : 0;
|
|
2497
|
+
}
|
|
2498
|
+
}
|
|
2499
|
+
return `${target}{reachable=${reachable ? 1 : 0},ack=${fastestNodesReached.has(target) ? 1 : 0},stream=${stream ? 1 : 0},conns=${openConnections},routes=${routeSample}}`;
|
|
2500
|
+
});
|
|
2501
|
+
const more = targets.size - sampledTargets.length;
|
|
2502
|
+
return `deliveryState peers=${this.peers.size} routesLocal=${this.routes.count()} routesAll=${this.routes.countAll()} targets=[${sampled.join(";")}]${more > 0 ? ` (+${more} more)` : ""}`;
|
|
2503
|
+
}
|
|
2504
|
+
|
|
2215
2505
|
private async createDeliveryPromise(
|
|
2216
2506
|
from: PublicSignKey,
|
|
2217
2507
|
message: DataMessage | Goodbye,
|
|
@@ -2235,11 +2525,9 @@ export abstract class DirectStream<
|
|
|
2235
2525
|
|
|
2236
2526
|
const fastestNodesReached = new Map<string, number[]>();
|
|
2237
2527
|
const messageToSet: Set<string> = new Set();
|
|
2238
|
-
if (message.header.mode
|
|
2528
|
+
if (deliveryModeHasReceiver(message.header.mode)) {
|
|
2239
2529
|
for (const to of message.header.mode.to) {
|
|
2240
|
-
if (to === from.hashcode())
|
|
2241
|
-
continue;
|
|
2242
|
-
}
|
|
2530
|
+
if (to === from.hashcode()) continue;
|
|
2243
2531
|
messageToSet.add(to);
|
|
2244
2532
|
|
|
2245
2533
|
if (!relayed && !this.healthChecks.has(to)) {
|
|
@@ -2273,29 +2561,25 @@ export abstract class DirectStream<
|
|
|
2273
2561
|
const willGetAllAcknowledgements = !relayed; // Only the origin will get all acks
|
|
2274
2562
|
|
|
2275
2563
|
// Expected to receive at least 'filterMessageForSeenCounter' acknowledgements from each peer
|
|
2276
|
-
const filterMessageForSeenCounter = relayed
|
|
2277
|
-
? undefined
|
|
2278
|
-
: message.header.mode instanceof SeekDelivery
|
|
2279
|
-
? Math.min(this.peers.size, message.header.mode.redundancy)
|
|
2280
|
-
: 1; /* message.deliveryMode instanceof SeekDelivery ? Math.min(this.peers.size - (relayed ? 1 : 0), message.deliveryMode.redundancy) : 1 */
|
|
2564
|
+
const filterMessageForSeenCounter = relayed ? undefined : 1;
|
|
2281
2565
|
|
|
2282
2566
|
const uniqueAcks = new Set();
|
|
2283
2567
|
const session = +new Date();
|
|
2284
2568
|
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
2569
|
+
const onUnreachable =
|
|
2570
|
+
!relayed &&
|
|
2571
|
+
((ev: any) => {
|
|
2572
|
+
const target = ev.detail.hashcode();
|
|
2573
|
+
if (messageToSet.has(target)) {
|
|
2574
|
+
// A peer can transiently appear unreachable while routes are being updated.
|
|
2575
|
+
// Keep waiting for ACK/timeout instead of failing delivery immediately.
|
|
2576
|
+
logger.trace(
|
|
2577
|
+
"peer unreachable during delivery (msg=%s, target=%s); waiting for ACK/timeout",
|
|
2578
|
+
idString,
|
|
2579
|
+
target,
|
|
2580
|
+
);
|
|
2581
|
+
}
|
|
2582
|
+
});
|
|
2299
2583
|
|
|
2300
2584
|
onUnreachable && this.addEventListener("peer:unreachable", onUnreachable);
|
|
2301
2585
|
|
|
@@ -2308,8 +2592,8 @@ export abstract class DirectStream<
|
|
|
2308
2592
|
onAbort && signal?.removeEventListener("abort", onAbort);
|
|
2309
2593
|
};
|
|
2310
2594
|
|
|
2311
|
-
|
|
2312
|
-
|
|
2595
|
+
const timeout = setTimeout(async () => {
|
|
2596
|
+
clear();
|
|
2313
2597
|
|
|
2314
2598
|
let hasAll = true;
|
|
2315
2599
|
|
|
@@ -2327,22 +2611,31 @@ export abstract class DirectStream<
|
|
|
2327
2611
|
}
|
|
2328
2612
|
}
|
|
2329
2613
|
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
|
|
2614
|
+
if (!hasAll && willGetAllAcknowledgements) {
|
|
2615
|
+
const debugState = this.formatDeliveryDebugState(
|
|
2616
|
+
messageToSet,
|
|
2617
|
+
fastestNodesReached,
|
|
2618
|
+
);
|
|
2619
|
+
const msgMeta = `msgType=${message.constructor.name} dataBytes=${
|
|
2620
|
+
message instanceof DataMessage
|
|
2621
|
+
? (message.data?.byteLength ?? 0)
|
|
2622
|
+
: 0
|
|
2623
|
+
} relayed=${relayed ? 1 : 0}`;
|
|
2624
|
+
deliveryDeferredPromise.reject(
|
|
2625
|
+
new DeliveryError(
|
|
2626
|
+
`Failed to get message ${idString} ${filterMessageForSeenCounter} ${[
|
|
2627
|
+
...messageToSet,
|
|
2628
|
+
]} delivery acknowledges from all nodes (${
|
|
2629
|
+
fastestNodesReached.size
|
|
2630
|
+
}/${messageToSet.size}). Mode: ${
|
|
2631
|
+
message.header.mode.constructor.name
|
|
2632
|
+
}. Redundancy: ${(message.header.mode as any)["redundancy"]}. ${msgMeta}. ${debugState}`,
|
|
2633
|
+
),
|
|
2634
|
+
);
|
|
2342
2635
|
} else {
|
|
2343
2636
|
deliveryDeferredPromise.resolve();
|
|
2344
2637
|
}
|
|
2345
|
-
|
|
2638
|
+
}, this.seekTimeout);
|
|
2346
2639
|
|
|
2347
2640
|
if (signal) {
|
|
2348
2641
|
onAbort = () => {
|
|
@@ -2356,22 +2649,33 @@ export abstract class DirectStream<
|
|
|
2356
2649
|
}
|
|
2357
2650
|
}
|
|
2358
2651
|
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
2652
|
+
const checkDone = () => {
|
|
2653
|
+
// This if clause should never enter for relayed connections, since we don't
|
|
2654
|
+
// know how many ACKs we will get
|
|
2655
|
+
if (
|
|
2656
|
+
filterMessageForSeenCounter != null &&
|
|
2657
|
+
uniqueAcks.size >= messageToSet.size * filterMessageForSeenCounter
|
|
2658
|
+
) {
|
|
2659
|
+
const shouldKeepCallbackForRouteLearning =
|
|
2660
|
+
!relayed &&
|
|
2661
|
+
message.header.mode instanceof AcknowledgeDelivery &&
|
|
2662
|
+
message.header.mode.redundancy > 1;
|
|
2663
|
+
if (haveReceivers && !shouldKeepCallbackForRouteLearning) {
|
|
2664
|
+
// If we have an explicit recipient list we can clear the ACK callback once we
|
|
2665
|
+
// got the expected acknowledgements.
|
|
2666
|
+
clear();
|
|
2667
|
+
} else if (haveReceivers && shouldKeepCallbackForRouteLearning) {
|
|
2668
|
+
// Resolve delivery early, but keep ACK callbacks alive until timeout so we can
|
|
2669
|
+
// learn additional redundant routes (seenCounter=1..redundancy-1).
|
|
2670
|
+
onUnreachable &&
|
|
2671
|
+
this.removeEventListener("peer:unreachable", onUnreachable);
|
|
2672
|
+
onAbort && signal?.removeEventListener("abort", onAbort);
|
|
2673
|
+
onAbort = undefined;
|
|
2674
|
+
}
|
|
2371
2675
|
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2676
|
+
deliveryDeferredPromise.resolve();
|
|
2677
|
+
return true;
|
|
2678
|
+
}
|
|
2375
2679
|
return false;
|
|
2376
2680
|
};
|
|
2377
2681
|
|
|
@@ -2385,26 +2689,39 @@ export abstract class DirectStream<
|
|
|
2385
2689
|
// remove the automatic removal of route timeout since we have observed lifesigns of a peer
|
|
2386
2690
|
this.clearHealthcheckTimer(messageTargetHash);
|
|
2387
2691
|
|
|
2388
|
-
//
|
|
2389
|
-
//
|
|
2390
|
-
|
|
2692
|
+
// If the target is not inside the original message `to`, we can still add the target to our routes.
|
|
2693
|
+
// This matters because relays may modify `to`, and because "ack-anywhere" style probes intentionally
|
|
2694
|
+
// do not provide an explicit recipient list.
|
|
2695
|
+
if (
|
|
2696
|
+
message.header.mode instanceof AcknowledgeDelivery ||
|
|
2697
|
+
message.header.mode instanceof AcknowledgeAnyWhere
|
|
2698
|
+
) {
|
|
2699
|
+
const upstreamHash = messageFrom?.publicKey.hashcode();
|
|
2700
|
+
|
|
2701
|
+
const routeUpdate = computeSeekAckRouteUpdate({
|
|
2702
|
+
current: this.publicKeyHash,
|
|
2703
|
+
upstream: upstreamHash,
|
|
2704
|
+
downstream: messageThrough.publicKey.hashcode(),
|
|
2705
|
+
target: messageTargetHash,
|
|
2706
|
+
// Route "distance" is based on recipient-seen order (0 = fastest). This is relied upon by
|
|
2707
|
+
// `Routes.getFanout(...)` which uses `distance < redundancy` to select redundant next-hops.
|
|
2708
|
+
distance: seenCounter,
|
|
2709
|
+
});
|
|
2710
|
+
|
|
2391
2711
|
this.addRouteConnection(
|
|
2392
|
-
|
|
2393
|
-
|
|
2712
|
+
routeUpdate.from,
|
|
2713
|
+
routeUpdate.neighbour,
|
|
2394
2714
|
messageTarget,
|
|
2395
|
-
|
|
2715
|
+
routeUpdate.distance,
|
|
2396
2716
|
session,
|
|
2397
2717
|
Number(ack.header.session),
|
|
2398
|
-
);
|
|
2718
|
+
);
|
|
2399
2719
|
}
|
|
2400
2720
|
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
seenCounter < filterMessageForSeenCounter
|
|
2406
|
-
) {
|
|
2407
|
-
// TODO set limit correctly
|
|
2721
|
+
if (messageToSet.has(messageTargetHash)) {
|
|
2722
|
+
// Any ACK from the target should satisfy delivery semantics.
|
|
2723
|
+
// Relying on only seenCounter=0 can fail under churn if the first ACK
|
|
2724
|
+
// is lost but a later ACK (seenCounter>0) arrives.
|
|
2408
2725
|
if (seenCounter < MAX_ROUTE_DISTANCE) {
|
|
2409
2726
|
let arr = fastestNodesReached.get(messageTargetHash);
|
|
2410
2727
|
if (!arr) {
|
|
@@ -2413,10 +2730,9 @@ export abstract class DirectStream<
|
|
|
2413
2730
|
}
|
|
2414
2731
|
arr.push(seenCounter);
|
|
2415
2732
|
|
|
2416
|
-
uniqueAcks.add(messageTargetHash
|
|
2733
|
+
uniqueAcks.add(messageTargetHash);
|
|
2417
2734
|
}
|
|
2418
2735
|
}
|
|
2419
|
-
}
|
|
2420
2736
|
|
|
2421
2737
|
checkDone();
|
|
2422
2738
|
},
|
|
@@ -2440,7 +2756,9 @@ export abstract class DirectStream<
|
|
|
2440
2756
|
throw new NotStartedError();
|
|
2441
2757
|
}
|
|
2442
2758
|
|
|
2759
|
+
const isRelayed = relayed ?? from.hashcode() !== this.publicKeyHash;
|
|
2443
2760
|
let delivereyPromise: Promise<void> | undefined = undefined as any;
|
|
2761
|
+
let ackCallbackId: string | undefined;
|
|
2444
2762
|
|
|
2445
2763
|
if (
|
|
2446
2764
|
(!message.header.signatures ||
|
|
@@ -2455,123 +2773,182 @@ export abstract class DirectStream<
|
|
|
2455
2773
|
* Logic for handling acknowledge messages when we receive them (later)
|
|
2456
2774
|
*/
|
|
2457
2775
|
|
|
2458
|
-
if (
|
|
2459
|
-
message instanceof DataMessage &&
|
|
2460
|
-
message.header.mode instanceof SeekDelivery &&
|
|
2461
|
-
!relayed
|
|
2462
|
-
) {
|
|
2463
|
-
to = this.peers; // seek delivery will not work unless we try all possible paths
|
|
2464
|
-
}
|
|
2465
|
-
|
|
2466
|
-
if (message.header.mode instanceof AcknowledgeDelivery) {
|
|
2467
|
-
to = undefined;
|
|
2468
|
-
}
|
|
2469
|
-
|
|
2470
2776
|
if (
|
|
2471
2777
|
(message instanceof DataMessage || message instanceof Goodbye) &&
|
|
2472
|
-
(message.header.mode instanceof
|
|
2473
|
-
message.header.mode instanceof
|
|
2778
|
+
(message.header.mode instanceof AcknowledgeDelivery ||
|
|
2779
|
+
message.header.mode instanceof AcknowledgeAnyWhere)
|
|
2474
2780
|
) {
|
|
2475
2781
|
const deliveryDeferredPromise = await this.createDeliveryPromise(
|
|
2476
2782
|
from,
|
|
2477
2783
|
message,
|
|
2478
|
-
|
|
2784
|
+
isRelayed,
|
|
2479
2785
|
signal,
|
|
2480
2786
|
);
|
|
2481
2787
|
delivereyPromise = deliveryDeferredPromise.promise;
|
|
2788
|
+
ackCallbackId = toBase64(message.id);
|
|
2482
2789
|
}
|
|
2483
2790
|
|
|
2484
|
-
|
|
2791
|
+
try {
|
|
2792
|
+
const bytes = message.bytes();
|
|
2485
2793
|
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
|
|
2489
|
-
|
|
2794
|
+
if (!isRelayed) {
|
|
2795
|
+
const bytesArray = bytes instanceof Uint8Array ? bytes : bytes.subarray();
|
|
2796
|
+
await this.modifySeenCache(bytesArray);
|
|
2797
|
+
}
|
|
2490
2798
|
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
|
|
2799
|
+
/**
|
|
2800
|
+
* For non SEEKing message delivery modes, use routing
|
|
2801
|
+
*/
|
|
2494
2802
|
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2803
|
+
if (message instanceof DataMessage) {
|
|
2804
|
+
if (
|
|
2805
|
+
(message.header.mode instanceof AcknowledgeDelivery ||
|
|
2806
|
+
message.header.mode instanceof SilentDelivery) &&
|
|
2807
|
+
!to
|
|
2808
|
+
) {
|
|
2809
|
+
if (message.header.mode.to.length === 0) {
|
|
2810
|
+
// we definitely know that we should not forward the message anywhere
|
|
2811
|
+
return delivereyPromise;
|
|
2812
|
+
}
|
|
2504
2813
|
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
|
|
2814
|
+
const fanout = this.routes.getFanout(
|
|
2815
|
+
from.hashcode(),
|
|
2816
|
+
message.header.mode.to,
|
|
2817
|
+
message.header.mode.redundancy,
|
|
2818
|
+
);
|
|
2510
2819
|
|
|
2511
|
-
|
|
2512
|
-
|
|
2820
|
+
// If we have explicit routing information, send only along the chosen next-hops.
|
|
2821
|
+
// If `fanout` is empty (no route info yet), fall through to the flooding logic below
|
|
2822
|
+
// so acknowledged deliveries can discover/repair routes instead of timing out.
|
|
2823
|
+
if (fanout && fanout.size > 0) {
|
|
2513
2824
|
const promises: Promise<any>[] = [];
|
|
2825
|
+
const usedNeighbours = new Set<string>();
|
|
2826
|
+
const originalTo = message.header.mode.to;
|
|
2514
2827
|
for (const [neighbour, _distantPeers] of fanout) {
|
|
2515
2828
|
const stream = this.peers.get(neighbour);
|
|
2516
|
-
stream
|
|
2829
|
+
if (!stream) continue;
|
|
2830
|
+
if (message.header.mode instanceof SilentDelivery) {
|
|
2831
|
+
message.header.mode.to = [..._distantPeers.keys()];
|
|
2517
2832
|
promises.push(
|
|
2518
|
-
stream.waitForWrite(bytes, message.header.priority),
|
|
2833
|
+
stream.waitForWrite(message.bytes(), message.header.priority),
|
|
2519
2834
|
);
|
|
2835
|
+
} else {
|
|
2836
|
+
promises.push(stream.waitForWrite(bytes, message.header.priority));
|
|
2837
|
+
}
|
|
2838
|
+
usedNeighbours.add(neighbour);
|
|
2839
|
+
}
|
|
2840
|
+
if (message.header.mode instanceof SilentDelivery) {
|
|
2841
|
+
message.header.mode.to = originalTo;
|
|
2842
|
+
}
|
|
2843
|
+
|
|
2844
|
+
// If the sender requested redundancy but we don't yet have enough distinct
|
|
2845
|
+
// next-hops for the target(s), opportunistically probe additional neighbours.
|
|
2846
|
+
// This replaces the previous "greedy fanout" probing behavior without needing
|
|
2847
|
+
// a separate delivery mode.
|
|
2848
|
+
if (
|
|
2849
|
+
!isRelayed &&
|
|
2850
|
+
message.header.mode instanceof AcknowledgeDelivery &&
|
|
2851
|
+
usedNeighbours.size < message.header.mode.redundancy
|
|
2852
|
+
) {
|
|
2853
|
+
for (const [neighbour, stream] of this.peers) {
|
|
2854
|
+
if (usedNeighbours.size >= message.header.mode.redundancy) {
|
|
2855
|
+
break;
|
|
2856
|
+
}
|
|
2857
|
+
if (usedNeighbours.has(neighbour)) continue;
|
|
2858
|
+
usedNeighbours.add(neighbour);
|
|
2859
|
+
promises.push(stream.waitForWrite(bytes, message.header.priority));
|
|
2860
|
+
}
|
|
2520
2861
|
}
|
|
2862
|
+
|
|
2521
2863
|
await Promise.all(promises);
|
|
2522
|
-
return delivereyPromise;
|
|
2864
|
+
return delivereyPromise;
|
|
2523
2865
|
}
|
|
2524
2866
|
|
|
2525
|
-
|
|
2867
|
+
// If we don't have routing information:
|
|
2868
|
+
// - For acknowledged deliveries, fall through to flooding (route discovery / repair).
|
|
2869
|
+
// - For silent deliveries, relays should not flood (prevents unnecessary fanout); origin may still flood.
|
|
2870
|
+
// We still allow direct neighbour delivery to explicit recipients (if connected).
|
|
2871
|
+
if (isRelayed && message.header.mode instanceof SilentDelivery) {
|
|
2872
|
+
const promises: Promise<any>[] = [];
|
|
2873
|
+
const originalTo = message.header.mode.to;
|
|
2874
|
+
for (const recipient of originalTo) {
|
|
2875
|
+
if (recipient === this.publicKeyHash) continue;
|
|
2876
|
+
if (recipient === from.hashcode()) continue; // never send back to previous hop
|
|
2877
|
+
const stream = this.peers.get(recipient);
|
|
2878
|
+
if (!stream) continue;
|
|
2879
|
+
if (
|
|
2880
|
+
message.header.signatures?.publicKeys.find(
|
|
2881
|
+
(x) => x.hashcode() === recipient,
|
|
2882
|
+
)
|
|
2883
|
+
) {
|
|
2884
|
+
continue; // recipient already signed/seen this message
|
|
2885
|
+
}
|
|
2886
|
+
message.header.mode.to = [recipient];
|
|
2887
|
+
promises.push(
|
|
2888
|
+
stream.waitForWrite(message.bytes(), message.header.priority),
|
|
2889
|
+
);
|
|
2890
|
+
}
|
|
2891
|
+
message.header.mode.to = originalTo;
|
|
2892
|
+
if (promises.length > 0) {
|
|
2893
|
+
await Promise.all(promises);
|
|
2894
|
+
}
|
|
2895
|
+
return delivereyPromise;
|
|
2896
|
+
}
|
|
2526
2897
|
}
|
|
2898
|
+
}
|
|
2527
2899
|
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
|
|
2900
|
+
// We fail to send the message directly, instead fallback to floodsub
|
|
2901
|
+
const peers: PeerStreams[] | Map<string, PeerStreams> = to || this.peers;
|
|
2902
|
+
if (
|
|
2903
|
+
peers == null ||
|
|
2904
|
+
(Array.isArray(peers) && peers.length === 0) ||
|
|
2905
|
+
(peers instanceof Map && peers.size === 0)
|
|
2906
|
+
) {
|
|
2907
|
+
logger.trace("No peers to send to");
|
|
2908
|
+
return delivereyPromise;
|
|
2909
|
+
}
|
|
2534
2910
|
|
|
2535
|
-
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
(Array.isArray(peers) && peers.length === 0) ||
|
|
2540
|
-
(peers instanceof Map && peers.size === 0)
|
|
2541
|
-
) {
|
|
2542
|
-
logger.trace("No peers to send to");
|
|
2543
|
-
return delivereyPromise;
|
|
2544
|
-
}
|
|
2911
|
+
let sentOnce = false;
|
|
2912
|
+
const promises: Promise<any>[] = [];
|
|
2913
|
+
for (const stream of peers.values()) {
|
|
2914
|
+
const id = stream as PeerStreams;
|
|
2545
2915
|
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
|
|
2916
|
+
// Dont sent back to the sender
|
|
2917
|
+
if (id.publicKey.equals(from)) {
|
|
2918
|
+
continue;
|
|
2919
|
+
}
|
|
2920
|
+
// Dont send message back to any of the signers (they have already seen the message)
|
|
2921
|
+
if (
|
|
2922
|
+
message.header.signatures?.publicKeys.find((x) =>
|
|
2923
|
+
x.equals(id.publicKey),
|
|
2924
|
+
)
|
|
2925
|
+
) {
|
|
2926
|
+
continue;
|
|
2927
|
+
}
|
|
2550
2928
|
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
continue;
|
|
2554
|
-
}
|
|
2555
|
-
// Dont send message back to any of the signers (they have already seen the message)
|
|
2556
|
-
if (
|
|
2557
|
-
message.header.signatures?.publicKeys.find((x) =>
|
|
2558
|
-
x.equals(id.publicKey),
|
|
2559
|
-
)
|
|
2560
|
-
) {
|
|
2561
|
-
continue;
|
|
2929
|
+
sentOnce = true;
|
|
2930
|
+
promises.push(id.waitForWrite(bytes, message.header.priority));
|
|
2562
2931
|
}
|
|
2932
|
+
await Promise.all(promises);
|
|
2563
2933
|
|
|
2564
|
-
sentOnce
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
|
|
2934
|
+
if (!sentOnce) {
|
|
2935
|
+
// If the caller provided an explicit peer list, treat "no valid receivers" as an error
|
|
2936
|
+
// even when forwarding. This catches programming mistakes early and matches test expectations.
|
|
2937
|
+
if (!isRelayed || to != null) {
|
|
2938
|
+
throw new DeliveryError("Message did not have any valid receivers");
|
|
2939
|
+
}
|
|
2940
|
+
}
|
|
2568
2941
|
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2942
|
+
return delivereyPromise;
|
|
2943
|
+
} catch (error) {
|
|
2944
|
+
// If message fanout/write fails before publishMessage returns its delivery
|
|
2945
|
+
// promise, clear any ACK callback to avoid late timer rejections leaking as
|
|
2946
|
+
// unhandled rejections in fire-and-forget call paths.
|
|
2947
|
+
if (ackCallbackId) {
|
|
2948
|
+
this._ackCallbacks.get(ackCallbackId)?.clear();
|
|
2572
2949
|
}
|
|
2950
|
+
throw error;
|
|
2573
2951
|
}
|
|
2574
|
-
return delivereyPromise;
|
|
2575
2952
|
}
|
|
2576
2953
|
|
|
2577
2954
|
async maybeConnectDirectly(toHash: string, origin: MultiAddrinfo) {
|
|
@@ -2694,8 +3071,11 @@ export abstract class DirectStream<
|
|
|
2694
3071
|
const wins = new Set<string>();
|
|
2695
3072
|
for (const h of admitted) if (reached(h, target)) wins.add(h);
|
|
2696
3073
|
|
|
2697
|
-
|
|
2698
|
-
|
|
3074
|
+
// Preserve input order in the returned list (important for deterministic callers/tests).
|
|
3075
|
+
const orderedWins = () => admitted.filter((h) => wins.has(h));
|
|
3076
|
+
|
|
3077
|
+
if (settle === "any" && wins.size > 0) return orderedWins();
|
|
3078
|
+
if (settle === "all" && wins.size === admitted.length) return orderedWins();
|
|
2699
3079
|
|
|
2700
3080
|
// Abort/timeout
|
|
2701
3081
|
const abortSignals = [this.closeController.signal];
|
|
@@ -2711,143 +3091,62 @@ export abstract class DirectStream<
|
|
|
2711
3091
|
return defer.resolve();
|
|
2712
3092
|
};
|
|
2713
3093
|
|
|
2714
|
-
try {
|
|
2715
|
-
await waitForEvent(this, eventsFor(target), check, {
|
|
2716
|
-
signals: abortSignals,
|
|
2717
|
-
timeout,
|
|
2718
|
-
});
|
|
2719
|
-
return [...wins];
|
|
2720
|
-
} catch (e) {
|
|
2721
|
-
const abortSignal = abortSignals.find((s) => s.aborted);
|
|
2722
|
-
if (abortSignal) {
|
|
2723
|
-
if (abortSignal.reason instanceof Error) {
|
|
2724
|
-
throw abortSignal.reason;
|
|
2725
|
-
}
|
|
2726
|
-
throw new AbortError(
|
|
2727
|
-
"Aborted waiting for peers: " + abortSignal.reason,
|
|
2728
|
-
);
|
|
2729
|
-
}
|
|
2730
|
-
if (e instanceof Error) {
|
|
2731
|
-
throw e;
|
|
2732
|
-
}
|
|
2733
|
-
if (settle === "all") throw new TimeoutError("Timeout waiting for peers");
|
|
2734
|
-
return [...wins]; // settle:any: return whatever successes we got
|
|
2735
|
-
}
|
|
2736
|
-
}
|
|
2737
|
-
|
|
2738
|
-
/* async waitFor(
|
|
2739
|
-
peer: PeerId | PublicSignKey | string,
|
|
2740
|
-
options?: {
|
|
2741
|
-
timeout?: number;
|
|
2742
|
-
signal?: AbortSignal;
|
|
2743
|
-
neighbour?: boolean;
|
|
2744
|
-
inflight?: boolean;
|
|
2745
|
-
},
|
|
2746
|
-
) {
|
|
2747
|
-
const hash =
|
|
2748
|
-
typeof peer === "string"
|
|
2749
|
-
? peer
|
|
2750
|
-
: (peer instanceof PublicSignKey
|
|
2751
|
-
? peer
|
|
2752
|
-
: getPublicKeyFromPeerId(peer)
|
|
2753
|
-
).hashcode();
|
|
2754
|
-
if (hash === this.publicKeyHash) {
|
|
2755
|
-
return; // TODO throw error instead?
|
|
2756
|
-
}
|
|
2757
|
-
|
|
2758
|
-
if (options?.inflight) {
|
|
2759
|
-
// if peer is not in active connections or dialQueue, return silenty
|
|
2760
|
-
if (
|
|
2761
|
-
!this.peers.has(hash) &&
|
|
2762
|
-
!this.components.connectionManager
|
|
2763
|
-
.getDialQueue()
|
|
2764
|
-
.some((x) => getPublicKeyFromPeerId(x.peerId).hashcode() === hash) &&
|
|
2765
|
-
!this.components.connectionManager
|
|
2766
|
-
.getConnections()
|
|
2767
|
-
.some((x) => getPublicKeyFromPeerId(x.remotePeer).hashcode() === hash)
|
|
2768
|
-
) {
|
|
2769
|
-
return;
|
|
2770
|
-
}
|
|
2771
|
-
}
|
|
2772
|
-
|
|
2773
|
-
const checkIsReachable = (deferred: DeferredPromise<void>) => {
|
|
2774
|
-
if (options?.neighbour && !this.peers.has(hash)) {
|
|
2775
|
-
return;
|
|
2776
|
-
}
|
|
2777
|
-
|
|
2778
|
-
if (!this.routes.isReachable(this.publicKeyHash, hash, 0)) {
|
|
2779
|
-
return;
|
|
2780
|
-
}
|
|
2781
|
-
|
|
2782
|
-
deferred.resolve();
|
|
2783
|
-
};
|
|
2784
|
-
const abortSignals = [this.closeController.signal];
|
|
2785
|
-
if (options?.signal) {
|
|
2786
|
-
abortSignals.push(options.signal);
|
|
2787
|
-
}
|
|
2788
|
-
|
|
2789
|
-
try {
|
|
2790
|
-
await waitForEvent(this, ["peer:reachable"], checkIsReachable, {
|
|
2791
|
-
signals: abortSignals,
|
|
2792
|
-
timeout: options?.timeout,
|
|
2793
|
-
});
|
|
2794
|
-
} catch (error) {
|
|
2795
|
-
throw new Error(
|
|
2796
|
-
"Stream to " +
|
|
2797
|
-
hash +
|
|
2798
|
-
" from " +
|
|
2799
|
-
this.publicKeyHash +
|
|
2800
|
-
" does not exist. Connection exist: " +
|
|
2801
|
-
this.peers.has(hash) +
|
|
2802
|
-
". Route exist: " +
|
|
2803
|
-
this.routes.isReachable(this.publicKeyHash, hash, 0),
|
|
2804
|
-
);
|
|
2805
|
-
}
|
|
2806
|
-
|
|
2807
|
-
if (options?.neighbour) {
|
|
2808
|
-
const stream = this.peers.get(hash)!;
|
|
2809
3094
|
try {
|
|
2810
|
-
|
|
2811
|
-
|
|
2812
|
-
|
|
3095
|
+
await waitForEvent(this, eventsFor(target), check, {
|
|
3096
|
+
signals: abortSignals,
|
|
3097
|
+
timeout,
|
|
3098
|
+
});
|
|
3099
|
+
return orderedWins();
|
|
3100
|
+
} catch (e) {
|
|
3101
|
+
const abortSignal = abortSignals.find((s) => s.aborted);
|
|
3102
|
+
if (abortSignal) {
|
|
3103
|
+
if (abortSignal.reason instanceof Error) {
|
|
3104
|
+
throw abortSignal.reason;
|
|
2813
3105
|
}
|
|
2814
|
-
|
|
2815
|
-
|
|
2816
|
-
|
|
2817
|
-
|
|
2818
|
-
|
|
2819
|
-
{
|
|
2820
|
-
|
|
2821
|
-
|
|
2822
|
-
|
|
2823
|
-
|
|
2824
|
-
|
|
2825
|
-
|
|
2826
|
-
|
|
2827
|
-
|
|
2828
|
-
|
|
2829
|
-
|
|
2830
|
-
|
|
2831
|
-
|
|
2832
|
-
|
|
3106
|
+
throw new AbortError(
|
|
3107
|
+
"Aborted waiting for peers: " + abortSignal.reason,
|
|
3108
|
+
);
|
|
3109
|
+
}
|
|
3110
|
+
if (e instanceof TimeoutError) {
|
|
3111
|
+
if (settle === "any") {
|
|
3112
|
+
if (wins.size > 0) return orderedWins();
|
|
3113
|
+
throw new TimeoutError(
|
|
3114
|
+
`Timeout waiting for peers (target=${target}, seek=${seek}, missing=${admitted.length}/${admitted.length})`,
|
|
3115
|
+
);
|
|
3116
|
+
}
|
|
3117
|
+
const missing = admitted.filter((h) => !wins.has(h));
|
|
3118
|
+
const preview = missing.slice(0, 5).join(", ");
|
|
3119
|
+
throw new TimeoutError(
|
|
3120
|
+
`Timeout waiting for peers (target=${target}, seek=${seek}, missing=${missing.length}/${admitted.length}${
|
|
3121
|
+
preview ? `, e.g. ${preview}` : ""
|
|
3122
|
+
})`,
|
|
3123
|
+
);
|
|
3124
|
+
}
|
|
3125
|
+
if (e instanceof Error) throw e;
|
|
3126
|
+
if (settle === "all") throw new TimeoutError("Timeout waiting for peers");
|
|
3127
|
+
return orderedWins(); // settle:any: return whatever successes we got
|
|
2833
3128
|
}
|
|
2834
3129
|
}
|
|
2835
|
-
} */
|
|
2836
3130
|
|
|
2837
|
-
|
|
2838
|
-
|
|
2839
|
-
|
|
3131
|
+
getPublicKey(hash: string): PublicSignKey | undefined {
|
|
3132
|
+
return this.peerKeyHashToPublicKey.get(hash);
|
|
3133
|
+
}
|
|
2840
3134
|
|
|
2841
3135
|
get pending(): boolean {
|
|
2842
3136
|
return this._ackCallbacks.size > 0;
|
|
2843
3137
|
}
|
|
2844
3138
|
|
|
2845
3139
|
// make this into a job? run every few ms
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
if (this.
|
|
2849
|
-
|
|
2850
|
-
|
|
3140
|
+
maybePruneConnections(): Promise<void> {
|
|
3141
|
+
// Hard cap on peer streams: treat as a primary pruning signal.
|
|
3142
|
+
if (this.peers.size > this.connectionManagerOptions.maxConnections) {
|
|
3143
|
+
return this.pruneConnectionsToLimits();
|
|
3144
|
+
}
|
|
3145
|
+
|
|
3146
|
+
if (this.connectionManagerOptions.pruner) {
|
|
3147
|
+
if (this.connectionManagerOptions.pruner.bandwidth != null) {
|
|
3148
|
+
let usedBandwidth = 0;
|
|
3149
|
+
for (const [_k, v] of this.peers) {
|
|
2851
3150
|
usedBandwidth += v.usedBandwidth;
|
|
2852
3151
|
}
|
|
2853
3152
|
usedBandwidth /= this.peers.size;
|
|
@@ -2856,17 +3155,17 @@ export abstract class DirectStream<
|
|
|
2856
3155
|
// prune
|
|
2857
3156
|
return this.pruneConnections();
|
|
2858
3157
|
}
|
|
2859
|
-
|
|
2860
|
-
|
|
2861
|
-
|
|
2862
|
-
|
|
2863
|
-
|
|
3158
|
+
} else if (this.connectionManagerOptions.pruner.maxBuffer != null) {
|
|
3159
|
+
const queuedBytes = this.getQueuedBytes();
|
|
3160
|
+
if (queuedBytes > this.connectionManagerOptions.pruner.maxBuffer) {
|
|
3161
|
+
// prune
|
|
3162
|
+
return this.pruneConnections();
|
|
3163
|
+
}
|
|
2864
3164
|
}
|
|
2865
3165
|
}
|
|
2866
|
-
}
|
|
2867
3166
|
|
|
2868
|
-
|
|
2869
|
-
|
|
3167
|
+
return Promise.resolve();
|
|
3168
|
+
}
|
|
2870
3169
|
|
|
2871
3170
|
async pruneConnections(): Promise<void> {
|
|
2872
3171
|
// TODO sort by bandwidth
|
|
@@ -2877,17 +3176,17 @@ export abstract class DirectStream<
|
|
|
2877
3176
|
const sorted = [...this.peers.values()]
|
|
2878
3177
|
.sort((x, y) => x.usedBandwidth - y.usedBandwidth)
|
|
2879
3178
|
.map((x) => x.publicKey.hashcode());
|
|
2880
|
-
|
|
2881
|
-
|
|
2882
|
-
|
|
2883
|
-
|
|
3179
|
+
const prunables = this.routes.getPrunable(sorted);
|
|
3180
|
+
if (prunables.length === 0) {
|
|
3181
|
+
return;
|
|
3182
|
+
}
|
|
2884
3183
|
|
|
2885
|
-
|
|
2886
|
-
|
|
3184
|
+
const stream = this.peers.get(prunables[0])!;
|
|
3185
|
+
this.prunedConnectionsCache?.add(stream.publicKey.hashcode());
|
|
2887
3186
|
|
|
2888
|
-
|
|
2889
|
-
|
|
2890
|
-
|
|
3187
|
+
await this.onPeerDisconnected(stream.peerId);
|
|
3188
|
+
return this.components.connectionManager.closeConnections(stream.peerId);
|
|
3189
|
+
}
|
|
2891
3190
|
|
|
2892
3191
|
getQueuedBytes(): number {
|
|
2893
3192
|
let sum = 0;
|