@peerbit/stream 4.5.3-e38a67b → 4.6.0-000e3f1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/benchmark/directstream-sim.d.ts +12 -0
- package/dist/benchmark/directstream-sim.d.ts.map +1 -0
- package/dist/benchmark/directstream-sim.js +299 -0
- package/dist/benchmark/directstream-sim.js.map +1 -0
- package/dist/benchmark/index.d.ts +10 -0
- package/dist/benchmark/index.d.ts.map +1 -0
- package/dist/benchmark/index.js +48 -0
- package/dist/benchmark/index.js.map +1 -0
- package/dist/benchmark/topology-sim.d.ts +12 -0
- package/dist/benchmark/topology-sim.d.ts.map +1 -0
- package/dist/benchmark/topology-sim.js +410 -0
- package/dist/benchmark/topology-sim.js.map +1 -0
- package/dist/benchmark/transfer.js +2 -2
- package/dist/benchmark/transfer.js.map +1 -1
- package/dist/src/core/seek-routing.d.ts +39 -0
- package/dist/src/core/seek-routing.d.ts.map +1 -0
- package/dist/src/core/seek-routing.js +33 -0
- package/dist/src/core/seek-routing.js.map +1 -0
- package/dist/src/index.d.ts +36 -4
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +516 -300
- package/dist/src/index.js.map +1 -1
- package/dist/src/routes.d.ts +11 -0
- package/dist/src/routes.d.ts.map +1 -1
- package/dist/src/routes.js +105 -16
- package/dist/src/routes.js.map +1 -1
- package/dist/src/wait-for-event.d.ts.map +1 -1
- package/dist/src/wait-for-event.js +34 -10
- package/dist/src/wait-for-event.js.map +1 -1
- package/package.json +8 -8
- package/src/core/seek-routing.ts +75 -0
- package/src/index.ts +790 -509
- package/src/routes.ts +121 -19
- package/src/wait-for-event.ts +46 -15
package/src/index.ts
CHANGED
|
@@ -29,6 +29,7 @@ import {
|
|
|
29
29
|
import type { SignatureWithKey } from "@peerbit/crypto";
|
|
30
30
|
import {
|
|
31
31
|
ACK,
|
|
32
|
+
AcknowledgeAnyWhere,
|
|
32
33
|
AcknowledgeDelivery,
|
|
33
34
|
AnyWhere,
|
|
34
35
|
DataMessage,
|
|
@@ -39,7 +40,6 @@ import {
|
|
|
39
40
|
MessageHeader,
|
|
40
41
|
MultiAddrinfo,
|
|
41
42
|
NotStartedError,
|
|
42
|
-
SeekDelivery,
|
|
43
43
|
SilentDelivery,
|
|
44
44
|
TracedDelivery,
|
|
45
45
|
coercePeerRefsToHashes,
|
|
@@ -69,6 +69,10 @@ import { type Pushable, pushable } from "it-pushable";
|
|
|
69
69
|
import pDefer, { type DeferredPromise } from "p-defer";
|
|
70
70
|
import Queue from "p-queue";
|
|
71
71
|
import { Uint8ArrayList } from "uint8arraylist";
|
|
72
|
+
import {
|
|
73
|
+
computeSeekAckRouteUpdate,
|
|
74
|
+
shouldAcknowledgeDataMessage,
|
|
75
|
+
} from "./core/seek-routing.js";
|
|
72
76
|
import { logger } from "./logger.js";
|
|
73
77
|
import { type PushableLanes, pushableLanes } from "./pushable-lanes.js";
|
|
74
78
|
import { MAX_ROUTE_DISTANCE, Routes } from "./routes.js";
|
|
@@ -167,8 +171,6 @@ const DEFAULT_MAX_CONNECTIONS = 300;
|
|
|
167
171
|
|
|
168
172
|
const DEFAULT_PRUNED_CONNNECTIONS_TIMEOUT = 30 * 1000;
|
|
169
173
|
|
|
170
|
-
const ROUTE_UPDATE_DELAY_FACTOR = 3e4;
|
|
171
|
-
|
|
172
174
|
const DEFAULT_CREATE_OUTBOUND_STREAM_TIMEOUT = 30_000;
|
|
173
175
|
|
|
174
176
|
const PRIORITY_LANES = 4;
|
|
@@ -261,6 +263,19 @@ export class PeerStreams extends TypedEventEmitter<PeerStreamEvents> {
|
|
|
261
263
|
public _getActiveOutboundPushable(): PushableLanes<Uint8Array> | undefined {
|
|
262
264
|
return this.outboundStreams[0]?.pushable;
|
|
263
265
|
}
|
|
266
|
+
public getOutboundQueuedBytes(): number {
|
|
267
|
+
return this._getActiveOutboundPushable()?.readableLength ?? 0;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
public getOutboundQueuedBytesByLane(): number[] {
|
|
271
|
+
const p = this._getActiveOutboundPushable();
|
|
272
|
+
if (!p) return Array(PRIORITY_LANES).fill(0);
|
|
273
|
+
const out: number[] = [];
|
|
274
|
+
for (let lane = 0; lane < PRIORITY_LANES; lane++) {
|
|
275
|
+
out.push(p.getReadableLength(lane));
|
|
276
|
+
}
|
|
277
|
+
return out;
|
|
278
|
+
}
|
|
264
279
|
public _getOutboundCount() {
|
|
265
280
|
return this.outboundStreams.length;
|
|
266
281
|
}
|
|
@@ -433,10 +448,7 @@ export class PeerStreams extends TypedEventEmitter<PeerStreamEvents> {
|
|
|
433
448
|
}
|
|
434
449
|
|
|
435
450
|
try {
|
|
436
|
-
c.pushable.push(
|
|
437
|
-
payload,
|
|
438
|
-
getLaneFromPriority(priority),
|
|
439
|
-
);
|
|
451
|
+
c.pushable.push(payload, getLaneFromPriority(priority));
|
|
440
452
|
successes++;
|
|
441
453
|
} catch (e) {
|
|
442
454
|
failures.push(e);
|
|
@@ -498,7 +510,10 @@ export class PeerStreams extends TypedEventEmitter<PeerStreamEvents> {
|
|
|
498
510
|
return;
|
|
499
511
|
}
|
|
500
512
|
|
|
501
|
-
|
|
513
|
+
// Outbound stream negotiation can legitimately take several seconds in CI
|
|
514
|
+
// (identify/protocol discovery, resource contention, etc). Keep this fairly
|
|
515
|
+
// generous so control-plane messages (joins/subscriptions) don't flap.
|
|
516
|
+
const timeoutMs = 10_000;
|
|
502
517
|
|
|
503
518
|
await new Promise<void>((resolve, reject) => {
|
|
504
519
|
const onOutbound = () => {
|
|
@@ -783,21 +798,29 @@ export class PeerStreams extends TypedEventEmitter<PeerStreamEvents> {
|
|
|
783
798
|
this.outboundAbortController.abort();
|
|
784
799
|
}
|
|
785
800
|
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
801
|
+
// End inbound streams
|
|
802
|
+
if (this.inboundStreams.length) {
|
|
803
|
+
for (const inbound of this.inboundStreams) {
|
|
804
|
+
try {
|
|
805
|
+
inbound.abortController.abort();
|
|
806
|
+
} catch {
|
|
807
|
+
logger.error("Failed to abort inbound stream");
|
|
808
|
+
}
|
|
809
|
+
try {
|
|
810
|
+
// Best-effort shutdown: on some transports (notably websockets),
|
|
811
|
+
// awaiting a graceful close can hang indefinitely if the remote is
|
|
812
|
+
// concurrently stopping. Abort immediately and do not await close.
|
|
813
|
+
try {
|
|
814
|
+
inbound.raw.abort?.(new AbortError("Closed"));
|
|
815
|
+
} catch {
|
|
816
|
+
// ignore
|
|
817
|
+
}
|
|
818
|
+
void Promise.resolve(inbound.raw.close?.()).catch(() => {});
|
|
819
|
+
} catch {
|
|
820
|
+
logger.error("Failed to close inbound stream");
|
|
821
|
+
}
|
|
798
822
|
}
|
|
799
823
|
}
|
|
800
|
-
}
|
|
801
824
|
|
|
802
825
|
this.usedBandWidthTracker.stop();
|
|
803
826
|
|
|
@@ -841,9 +864,27 @@ export type DirectStreamOptions = {
|
|
|
841
864
|
maxOutboundStreams?: number;
|
|
842
865
|
inboundIdleTimeout?: number; // override PeerStreams.INBOUND_IDLE_MS
|
|
843
866
|
connectionManager?: ConnectionManagerArguments;
|
|
844
|
-
routeSeekInterval?: number;
|
|
845
867
|
seekTimeout?: number;
|
|
846
868
|
routeMaxRetentionPeriod?: number;
|
|
869
|
+
/**
|
|
870
|
+
* Best-effort bounds for the per-process route cache. These exist to prevent
|
|
871
|
+
* unbounded memory growth in large networks/simulations.
|
|
872
|
+
*/
|
|
873
|
+
routeCacheMaxFromEntries?: number;
|
|
874
|
+
routeCacheMaxTargetsPerFrom?: number;
|
|
875
|
+
routeCacheMaxRelaysPerTarget?: number;
|
|
876
|
+
/**
|
|
877
|
+
* Share node-level routing/session state across DirectStream instances created
|
|
878
|
+
* from the same libp2p private key.
|
|
879
|
+
*
|
|
880
|
+
* This reduces duplicated topology knowledge when multiple protocols run on
|
|
881
|
+
* the same node (e.g. pubsub + fanout overlays).
|
|
882
|
+
*
|
|
883
|
+
* Defaults to `true`.
|
|
884
|
+
*/
|
|
885
|
+
sharedRouting?: boolean;
|
|
886
|
+
seenCacheMax?: number;
|
|
887
|
+
seenCacheTtlMs?: number;
|
|
847
888
|
};
|
|
848
889
|
|
|
849
890
|
type ConnectionManagerLike = {
|
|
@@ -873,16 +914,25 @@ export interface DirectStreamComponents {
|
|
|
873
914
|
privateKey: PrivateKey;
|
|
874
915
|
}
|
|
875
916
|
|
|
917
|
+
type SharedRoutingState = {
|
|
918
|
+
session: number;
|
|
919
|
+
routes: Routes;
|
|
920
|
+
controller: AbortController;
|
|
921
|
+
refs: number;
|
|
922
|
+
};
|
|
923
|
+
|
|
924
|
+
const sharedRoutingByPrivateKey = new WeakMap<PrivateKey, SharedRoutingState>();
|
|
925
|
+
|
|
876
926
|
export type PublishOptions = (WithMode | WithTo) &
|
|
877
927
|
PriorityOptions &
|
|
878
928
|
WithExtraSigners;
|
|
879
929
|
|
|
880
930
|
export abstract class DirectStream<
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
{
|
|
931
|
+
Events extends { [s: string]: any } = StreamEvents,
|
|
932
|
+
>
|
|
933
|
+
extends TypedEventEmitter<Events>
|
|
934
|
+
implements WaitForPeer, PublicKeyFromHashResolver
|
|
935
|
+
{
|
|
886
936
|
public peerId: PeerId;
|
|
887
937
|
public publicKey: PublicSignKey;
|
|
888
938
|
public publicKeyHash: string;
|
|
@@ -908,26 +958,35 @@ export abstract class DirectStream<
|
|
|
908
958
|
public multicodecs: string[];
|
|
909
959
|
public seenCache: Cache<number>;
|
|
910
960
|
private _registrarTopologyIds: string[] | undefined;
|
|
961
|
+
private _peerConnectListener?: (ev: any) => void;
|
|
962
|
+
private _peerDisconnectListener?: (ev: any) => void;
|
|
911
963
|
private readonly maxInboundStreams?: number;
|
|
912
964
|
private readonly maxOutboundStreams?: number;
|
|
913
965
|
connectionManagerOptions: ConnectionManagerOptions;
|
|
914
966
|
private recentDials?: Cache<string>;
|
|
915
967
|
private healthChecks: Map<string, ReturnType<typeof setTimeout>>;
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
968
|
+
private pruneConnectionsTimeout: ReturnType<typeof setInterval>;
|
|
969
|
+
private prunedConnectionsCache?: Cache<string>;
|
|
970
|
+
private pruneToLimitsInFlight?: Promise<void>;
|
|
971
|
+
private _startInFlight?: Promise<void>;
|
|
972
|
+
private routeMaxRetentionPeriod: number;
|
|
973
|
+
private routeCacheMaxFromEntries?: number;
|
|
974
|
+
private routeCacheMaxTargetsPerFrom?: number;
|
|
975
|
+
private routeCacheMaxRelaysPerTarget?: number;
|
|
976
|
+
private readonly sharedRouting: boolean;
|
|
977
|
+
private sharedRoutingKey?: PrivateKey;
|
|
978
|
+
private sharedRoutingState?: SharedRoutingState;
|
|
919
979
|
|
|
920
980
|
// for sequential creation of outbound streams
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
981
|
+
public outboundInflightQueue: Pushable<{
|
|
982
|
+
connection: Connection;
|
|
983
|
+
peerId: PeerId;
|
|
984
|
+
}>;
|
|
925
985
|
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
_outboundPump: ReturnType<typeof pipe> | undefined;
|
|
986
|
+
seekTimeout: number;
|
|
987
|
+
closeController: AbortController;
|
|
988
|
+
session: number;
|
|
989
|
+
_outboundPump: ReturnType<typeof pipe> | undefined;
|
|
931
990
|
|
|
932
991
|
private _ackCallbacks: Map<
|
|
933
992
|
string,
|
|
@@ -951,14 +1010,19 @@ export abstract class DirectStream<
|
|
|
951
1010
|
const {
|
|
952
1011
|
canRelayMessage = true,
|
|
953
1012
|
messageProcessingConcurrency = 10,
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
1013
|
+
maxInboundStreams,
|
|
1014
|
+
maxOutboundStreams,
|
|
1015
|
+
connectionManager,
|
|
1016
|
+
seekTimeout = SEEK_DELIVERY_TIMEOUT,
|
|
1017
|
+
routeMaxRetentionPeriod = ROUTE_MAX_RETANTION_PERIOD,
|
|
1018
|
+
routeCacheMaxFromEntries,
|
|
1019
|
+
routeCacheMaxTargetsPerFrom,
|
|
1020
|
+
routeCacheMaxRelaysPerTarget,
|
|
1021
|
+
sharedRouting = true,
|
|
1022
|
+
seenCacheMax = 1e6,
|
|
1023
|
+
seenCacheTtlMs = 10 * 60 * 1e3,
|
|
1024
|
+
inboundIdleTimeout,
|
|
1025
|
+
} = options || {};
|
|
962
1026
|
|
|
963
1027
|
const signKey = getKeypairFromPrivateKey(components.privateKey);
|
|
964
1028
|
this.seekTimeout = seekTimeout;
|
|
@@ -972,19 +1036,25 @@ export abstract class DirectStream<
|
|
|
972
1036
|
this.publicKeyHash = signKey.publicKey.hashcode();
|
|
973
1037
|
this.multicodecs = multicodecs;
|
|
974
1038
|
this.started = false;
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
1039
|
+
this.peers = new Map<string, PeerStreams>();
|
|
1040
|
+
this.canRelayMessage = canRelayMessage;
|
|
1041
|
+
this.healthChecks = new Map();
|
|
1042
|
+
this.queue = new Queue({ concurrency: messageProcessingConcurrency });
|
|
1043
|
+
this.maxInboundStreams = maxInboundStreams;
|
|
1044
|
+
this.maxOutboundStreams = maxOutboundStreams;
|
|
1045
|
+
this.seenCache = new Cache({
|
|
1046
|
+
max: Math.max(1, Math.floor(seenCacheMax)),
|
|
1047
|
+
ttl: Math.max(1, Math.floor(seenCacheTtlMs)),
|
|
1048
|
+
});
|
|
1049
|
+
this.routeMaxRetentionPeriod = routeMaxRetentionPeriod;
|
|
1050
|
+
this.routeCacheMaxFromEntries = routeCacheMaxFromEntries;
|
|
1051
|
+
this.routeCacheMaxTargetsPerFrom = routeCacheMaxTargetsPerFrom;
|
|
1052
|
+
this.routeCacheMaxRelaysPerTarget = routeCacheMaxRelaysPerTarget;
|
|
1053
|
+
this.sharedRouting = sharedRouting !== false;
|
|
1054
|
+
this.peerKeyHashToPublicKey = new Map();
|
|
1055
|
+
this._onIncomingStream = this._onIncomingStream.bind(this);
|
|
1056
|
+
this.onPeerConnected = this.onPeerConnected.bind(this);
|
|
1057
|
+
this.onPeerDisconnected = this.onPeerDisconnected.bind(this);
|
|
988
1058
|
|
|
989
1059
|
this._ackCallbacks = new Map();
|
|
990
1060
|
|
|
@@ -1025,23 +1095,60 @@ export abstract class DirectStream<
|
|
|
1025
1095
|
})
|
|
1026
1096
|
: undefined;
|
|
1027
1097
|
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1098
|
+
this.prunedConnectionsCache = this.connectionManagerOptions.pruner
|
|
1099
|
+
? new Cache({
|
|
1100
|
+
max: 1e6,
|
|
1101
|
+
ttl: this.connectionManagerOptions.pruner.connectionTimeout,
|
|
1102
|
+
})
|
|
1103
|
+
: undefined;
|
|
1104
|
+
}
|
|
1035
1105
|
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1106
|
+
private pruneConnectionsToLimits(): Promise<void> {
|
|
1107
|
+
if (this.pruneToLimitsInFlight) {
|
|
1108
|
+
return this.pruneToLimitsInFlight;
|
|
1109
|
+
}
|
|
1110
|
+
this.pruneToLimitsInFlight = (async () => {
|
|
1111
|
+
// Respect minConnections as a hard floor.
|
|
1112
|
+
const maxConnections = Math.max(
|
|
1113
|
+
this.connectionManagerOptions.minConnections,
|
|
1114
|
+
this.connectionManagerOptions.maxConnections,
|
|
1115
|
+
);
|
|
1116
|
+
if (this.peers.size <= maxConnections) {
|
|
1117
|
+
return;
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
// Prune in batches so we can quickly recover from join storms without waiting
|
|
1121
|
+
// for repeated pruner ticks. Bound work per run to avoid starving the event loop.
|
|
1122
|
+
const maxPrunes = Math.min(256, Math.max(1, this.peers.size - maxConnections));
|
|
1123
|
+
for (let i = 0; i < maxPrunes; i++) {
|
|
1124
|
+
if (this.peers.size <= maxConnections) break;
|
|
1125
|
+
const before = this.peers.size;
|
|
1126
|
+
await this.pruneConnections();
|
|
1127
|
+
if (this.peers.size >= before) break; // nothing prunable
|
|
1128
|
+
}
|
|
1129
|
+
})().finally(() => {
|
|
1130
|
+
this.pruneToLimitsInFlight = undefined;
|
|
1131
|
+
});
|
|
1132
|
+
return this.pruneToLimitsInFlight;
|
|
1133
|
+
}
|
|
1134
|
+
|
|
1135
|
+
async start() {
|
|
1136
|
+
if (this.started) return;
|
|
1137
|
+
if (this._startInFlight) return this._startInFlight;
|
|
1138
|
+
this._startInFlight = this._startImpl().finally(() => {
|
|
1139
|
+
this._startInFlight = undefined;
|
|
1140
|
+
});
|
|
1141
|
+
return this._startInFlight;
|
|
1039
1142
|
}
|
|
1040
1143
|
|
|
1041
|
-
|
|
1042
|
-
|
|
1144
|
+
private async _startImpl() {
|
|
1145
|
+
if (this.started) {
|
|
1146
|
+
return;
|
|
1147
|
+
}
|
|
1043
1148
|
|
|
1044
|
-
|
|
1149
|
+
await ready;
|
|
1150
|
+
|
|
1151
|
+
this.closeController = new AbortController();
|
|
1045
1152
|
|
|
1046
1153
|
this.outboundInflightQueue = pushable({ objectMode: true });
|
|
1047
1154
|
|
|
@@ -1119,10 +1226,47 @@ export abstract class DirectStream<
|
|
|
1119
1226
|
this.outboundInflightQueue.return();
|
|
1120
1227
|
});
|
|
1121
1228
|
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1229
|
+
if (this.sharedRouting) {
|
|
1230
|
+
const key = this.components.privateKey;
|
|
1231
|
+
this.sharedRoutingKey = key;
|
|
1232
|
+
let state = sharedRoutingByPrivateKey.get(key);
|
|
1233
|
+
if (!state) {
|
|
1234
|
+
const controller = new AbortController();
|
|
1235
|
+
state = {
|
|
1236
|
+
session: Date.now(),
|
|
1237
|
+
controller,
|
|
1238
|
+
routes: new Routes(this.publicKeyHash, {
|
|
1239
|
+
routeMaxRetentionPeriod: this.routeMaxRetentionPeriod,
|
|
1240
|
+
signal: controller.signal,
|
|
1241
|
+
maxFromEntries: this.routeCacheMaxFromEntries,
|
|
1242
|
+
maxTargetsPerFrom: this.routeCacheMaxTargetsPerFrom,
|
|
1243
|
+
maxRelaysPerTarget: this.routeCacheMaxRelaysPerTarget,
|
|
1244
|
+
}),
|
|
1245
|
+
refs: 0,
|
|
1246
|
+
};
|
|
1247
|
+
sharedRoutingByPrivateKey.set(key, state);
|
|
1248
|
+
} else {
|
|
1249
|
+
// Best-effort: prefer the strictest cleanup policy among co-located protocols.
|
|
1250
|
+
state.routes.routeMaxRetentionPeriod = Math.min(
|
|
1251
|
+
state.routes.routeMaxRetentionPeriod,
|
|
1252
|
+
this.routeMaxRetentionPeriod,
|
|
1253
|
+
);
|
|
1254
|
+
}
|
|
1255
|
+
|
|
1256
|
+
state.refs += 1;
|
|
1257
|
+
this.sharedRoutingState = state;
|
|
1258
|
+
this.session = state.session;
|
|
1259
|
+
this.routes = state.routes;
|
|
1260
|
+
} else {
|
|
1261
|
+
this.session = Date.now();
|
|
1262
|
+
this.routes = new Routes(this.publicKeyHash, {
|
|
1263
|
+
routeMaxRetentionPeriod: this.routeMaxRetentionPeriod,
|
|
1264
|
+
signal: this.closeController.signal,
|
|
1265
|
+
maxFromEntries: this.routeCacheMaxFromEntries,
|
|
1266
|
+
maxTargetsPerFrom: this.routeCacheMaxTargetsPerFrom,
|
|
1267
|
+
maxRelaysPerTarget: this.routeCacheMaxRelaysPerTarget,
|
|
1268
|
+
});
|
|
1269
|
+
}
|
|
1126
1270
|
|
|
1127
1271
|
this.started = true;
|
|
1128
1272
|
this.stopping = false;
|
|
@@ -1152,6 +1296,46 @@ export abstract class DirectStream<
|
|
|
1152
1296
|
),
|
|
1153
1297
|
);
|
|
1154
1298
|
|
|
1299
|
+
// Best-effort fallback: topology callbacks can depend on identify/protocol
|
|
1300
|
+
// discovery. Some test environments connect peers without triggering the
|
|
1301
|
+
// per-protocol topology immediately. Listening to peer connection events
|
|
1302
|
+
// ensures we still attempt to open outbound streams opportunistically.
|
|
1303
|
+
this._peerConnectListener = (ev: any) => {
|
|
1304
|
+
if (this.stopping || !this.started) return;
|
|
1305
|
+
const peerId = (ev as any)?.detail ?? (ev as any)?.peerId;
|
|
1306
|
+
if (!peerId) return;
|
|
1307
|
+
const conns = this.components.connectionManager.getConnections(peerId);
|
|
1308
|
+
if (!conns || conns.length === 0) return;
|
|
1309
|
+
let conn = conns[0]!;
|
|
1310
|
+
for (const c of conns) {
|
|
1311
|
+
if (!isWebsocketConnection(c)) {
|
|
1312
|
+
conn = c;
|
|
1313
|
+
break;
|
|
1314
|
+
}
|
|
1315
|
+
}
|
|
1316
|
+
void this.onPeerConnected(peerId, conn);
|
|
1317
|
+
};
|
|
1318
|
+
this._peerDisconnectListener = (ev: any) => {
|
|
1319
|
+
if (this.stopping || !this.started) return;
|
|
1320
|
+
const peerId = (ev as any)?.detail ?? (ev as any)?.peerId;
|
|
1321
|
+
if (!peerId) return;
|
|
1322
|
+
const conns = this.components.connectionManager.getConnections(peerId);
|
|
1323
|
+
const conn = conns && conns.length > 0 ? conns[0] : undefined;
|
|
1324
|
+
void this.onPeerDisconnected(peerId, conn);
|
|
1325
|
+
};
|
|
1326
|
+
try {
|
|
1327
|
+
this.components.events.addEventListener(
|
|
1328
|
+
"peer:connect",
|
|
1329
|
+
this._peerConnectListener as any,
|
|
1330
|
+
);
|
|
1331
|
+
this.components.events.addEventListener(
|
|
1332
|
+
"peer:disconnect",
|
|
1333
|
+
this._peerDisconnectListener as any,
|
|
1334
|
+
);
|
|
1335
|
+
} catch {
|
|
1336
|
+
// ignore unsupported event targets
|
|
1337
|
+
}
|
|
1338
|
+
|
|
1155
1339
|
// All existing connections are like new ones for us. To deduplication on remotes so we only resuse one connection for this protocol (we could be connected with many connections)
|
|
1156
1340
|
const peerToConnections: Map<string, Connection[]> = new Map();
|
|
1157
1341
|
const connections = this.components.connectionManager.getConnections();
|
|
@@ -1201,7 +1385,28 @@ export abstract class DirectStream<
|
|
|
1201
1385
|
return;
|
|
1202
1386
|
}
|
|
1203
1387
|
|
|
1388
|
+
const sharedState = this.sharedRoutingState;
|
|
1389
|
+
const sharedKey = this.sharedRoutingKey;
|
|
1390
|
+
|
|
1204
1391
|
clearTimeout(this.pruneConnectionsTimeout);
|
|
1392
|
+
try {
|
|
1393
|
+
if (this._peerConnectListener) {
|
|
1394
|
+
this.components.events.removeEventListener(
|
|
1395
|
+
"peer:connect",
|
|
1396
|
+
this._peerConnectListener as any,
|
|
1397
|
+
);
|
|
1398
|
+
}
|
|
1399
|
+
if (this._peerDisconnectListener) {
|
|
1400
|
+
this.components.events.removeEventListener(
|
|
1401
|
+
"peer:disconnect",
|
|
1402
|
+
this._peerDisconnectListener as any,
|
|
1403
|
+
);
|
|
1404
|
+
}
|
|
1405
|
+
} catch {
|
|
1406
|
+
// ignore unsupported event targets
|
|
1407
|
+
}
|
|
1408
|
+
this._peerConnectListener = undefined;
|
|
1409
|
+
this._peerDisconnectListener = undefined;
|
|
1205
1410
|
|
|
1206
1411
|
await Promise.all(
|
|
1207
1412
|
this.multicodecs.map((x) => this.components.registrar.unhandle(x)),
|
|
@@ -1235,7 +1440,11 @@ export abstract class DirectStream<
|
|
|
1235
1440
|
this.queue.clear();
|
|
1236
1441
|
this.peers.clear();
|
|
1237
1442
|
this.seenCache.clear();
|
|
1238
|
-
|
|
1443
|
+
// When routing is shared across co-located protocols, only clear once the last
|
|
1444
|
+
// instance stops. Otherwise we'd wipe routes still in use by other services.
|
|
1445
|
+
if (!sharedState) {
|
|
1446
|
+
this.routes.clear();
|
|
1447
|
+
}
|
|
1239
1448
|
this.peerKeyHashToPublicKey.clear();
|
|
1240
1449
|
|
|
1241
1450
|
for (const [_k, v] of this._ackCallbacks) {
|
|
@@ -1243,6 +1452,24 @@ export abstract class DirectStream<
|
|
|
1243
1452
|
}
|
|
1244
1453
|
|
|
1245
1454
|
this._ackCallbacks.clear();
|
|
1455
|
+
this.sharedRoutingState = undefined;
|
|
1456
|
+
this.sharedRoutingKey = undefined;
|
|
1457
|
+
if (sharedState && sharedKey) {
|
|
1458
|
+
sharedState.refs = Math.max(0, sharedState.refs - 1);
|
|
1459
|
+
if (sharedState.refs === 0) {
|
|
1460
|
+
try {
|
|
1461
|
+
sharedState.routes.clear();
|
|
1462
|
+
} catch {
|
|
1463
|
+
// ignore
|
|
1464
|
+
}
|
|
1465
|
+
try {
|
|
1466
|
+
sharedState.controller.abort();
|
|
1467
|
+
} catch {
|
|
1468
|
+
// ignore
|
|
1469
|
+
}
|
|
1470
|
+
sharedRoutingByPrivateKey.delete(sharedKey);
|
|
1471
|
+
}
|
|
1472
|
+
}
|
|
1246
1473
|
logger.trace("stopped");
|
|
1247
1474
|
this.stopping = false;
|
|
1248
1475
|
}
|
|
@@ -1439,7 +1666,8 @@ export abstract class DirectStream<
|
|
|
1439
1666
|
).catch(dontThrowIfDeliveryError);
|
|
1440
1667
|
}
|
|
1441
1668
|
|
|
1442
|
-
|
|
1669
|
+
// Best-effort liveness probe; never let background probe failures crash callers.
|
|
1670
|
+
void this.checkIsAlive([peerKeyHash]).catch(() => false);
|
|
1443
1671
|
}
|
|
1444
1672
|
|
|
1445
1673
|
logger.trace("connection ended:" + peerKey.toString());
|
|
@@ -1464,27 +1692,31 @@ export abstract class DirectStream<
|
|
|
1464
1692
|
distance: number,
|
|
1465
1693
|
session: number,
|
|
1466
1694
|
remoteSession: number,
|
|
1467
|
-
|
|
1468
|
-
|
|
1695
|
+
) {
|
|
1696
|
+
const targetHash = typeof target === "string" ? target : target.hashcode();
|
|
1697
|
+
// Best-effort: keep a hash -> public key map for any routed targets so
|
|
1698
|
+
// peer:unreachable events can always carry a PublicSignKey when we have seen it.
|
|
1699
|
+
if (typeof target !== "string") {
|
|
1700
|
+
this.peerKeyHashToPublicKey.set(targetHash, target);
|
|
1701
|
+
}
|
|
1469
1702
|
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1703
|
+
const update = this.routes.add(
|
|
1704
|
+
from,
|
|
1705
|
+
neighbour,
|
|
1706
|
+
targetHash,
|
|
1474
1707
|
distance,
|
|
1475
1708
|
session,
|
|
1476
1709
|
remoteSession,
|
|
1477
1710
|
);
|
|
1478
1711
|
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1712
|
+
// second condition is that we don't want to emit 'reachable' events for routes where we act only as a relay
|
|
1713
|
+
// in this case, from is != this.publicKeyhash
|
|
1714
|
+
if (from === this.publicKeyHash) {
|
|
1715
|
+
if (update === "new") {
|
|
1716
|
+
this.onPeerReachable(target);
|
|
1717
|
+
}
|
|
1485
1718
|
}
|
|
1486
1719
|
}
|
|
1487
|
-
}
|
|
1488
1720
|
|
|
1489
1721
|
public onPeerReachable(publicKey: PublicSignKey) {
|
|
1490
1722
|
// override this fn
|
|
@@ -1493,16 +1725,19 @@ export abstract class DirectStream<
|
|
|
1493
1725
|
);
|
|
1494
1726
|
}
|
|
1495
1727
|
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
}
|
|
1504
|
-
|
|
1505
|
-
|
|
1728
|
+
public onPeerUnreachable(hash: string) {
|
|
1729
|
+
// override this fns
|
|
1730
|
+
const key = this.peerKeyHashToPublicKey.get(hash);
|
|
1731
|
+
if (!key) {
|
|
1732
|
+
// Best-effort: we may only have the hash (no public key) for some routes.
|
|
1733
|
+
// Avoid crashing downstream listeners that assume `detail` is a PublicSignKey.
|
|
1734
|
+
return;
|
|
1735
|
+
}
|
|
1736
|
+
this.dispatchEvent(
|
|
1737
|
+
// TODO types
|
|
1738
|
+
new CustomEvent("peer:unreachable", { detail: key }),
|
|
1739
|
+
);
|
|
1740
|
+
}
|
|
1506
1741
|
|
|
1507
1742
|
public updateSession(key: PublicSignKey, session?: number) {
|
|
1508
1743
|
if (this.routes.updateSession(key.hashcode(), session)) {
|
|
@@ -1525,11 +1760,11 @@ export abstract class DirectStream<
|
|
|
1525
1760
|
/**
|
|
1526
1761
|
* Notifies the router that a peer has been connected
|
|
1527
1762
|
*/
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1763
|
+
addPeer(
|
|
1764
|
+
peerId: PeerId,
|
|
1765
|
+
publicKey: PublicSignKey,
|
|
1766
|
+
protocol: string,
|
|
1767
|
+
connId: string,
|
|
1533
1768
|
): PeerStreams {
|
|
1534
1769
|
const publicKeyHash = publicKey.hashcode();
|
|
1535
1770
|
|
|
@@ -1577,17 +1812,23 @@ export abstract class DirectStream<
|
|
|
1577
1812
|
{ once: true },
|
|
1578
1813
|
);
|
|
1579
1814
|
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1815
|
+
this.addRouteConnection(
|
|
1816
|
+
this.publicKeyHash,
|
|
1817
|
+
publicKey.hashcode(),
|
|
1818
|
+
publicKey,
|
|
1819
|
+
-1,
|
|
1820
|
+
+new Date(),
|
|
1821
|
+
-1,
|
|
1822
|
+
);
|
|
1588
1823
|
|
|
1589
|
-
|
|
1590
|
-
|
|
1824
|
+
// Enforce connection manager limits eagerly when new peers are added. Without this,
|
|
1825
|
+
// join storms can create large temporary peer sets and OOM in single-process sims.
|
|
1826
|
+
if (this.peers.size > this.connectionManagerOptions.maxConnections) {
|
|
1827
|
+
void this.pruneConnectionsToLimits().catch(() => {});
|
|
1828
|
+
}
|
|
1829
|
+
|
|
1830
|
+
return peerStreams;
|
|
1831
|
+
}
|
|
1591
1832
|
|
|
1592
1833
|
/**
|
|
1593
1834
|
* Notifies the router that a peer has been disconnected
|
|
@@ -1730,24 +1971,25 @@ export abstract class DirectStream<
|
|
|
1730
1971
|
}
|
|
1731
1972
|
|
|
1732
1973
|
public shouldIgnore(message: DataMessage, seenBefore: number) {
|
|
1733
|
-
const
|
|
1734
|
-
|
|
1735
|
-
|
|
1974
|
+
const signedBySelf =
|
|
1975
|
+
message.header.signatures?.publicKeys.some((x) =>
|
|
1976
|
+
x.equals(this.publicKey),
|
|
1977
|
+
) ?? false;
|
|
1736
1978
|
|
|
1737
|
-
if (
|
|
1979
|
+
if (signedBySelf) {
|
|
1738
1980
|
return true;
|
|
1739
1981
|
}
|
|
1740
1982
|
|
|
1983
|
+
// For acknowledged modes, allow limited duplicate forwarding so that we can
|
|
1984
|
+
// discover and maintain multiple candidate routes (distance=seenCounter).
|
|
1741
1985
|
if (
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
(message.header.mode instanceof SeekDelivery &&
|
|
1745
|
-
seenBefore >= message.header.mode.redundancy)
|
|
1986
|
+
message.header.mode instanceof AcknowledgeDelivery ||
|
|
1987
|
+
message.header.mode instanceof AcknowledgeAnyWhere
|
|
1746
1988
|
) {
|
|
1747
|
-
return
|
|
1989
|
+
return seenBefore >= message.header.mode.redundancy;
|
|
1748
1990
|
}
|
|
1749
1991
|
|
|
1750
|
-
return
|
|
1992
|
+
return seenBefore > 0;
|
|
1751
1993
|
}
|
|
1752
1994
|
|
|
1753
1995
|
public async onDataMessage(
|
|
@@ -1761,14 +2003,19 @@ export abstract class DirectStream<
|
|
|
1761
2003
|
}
|
|
1762
2004
|
|
|
1763
2005
|
let isForMe = false;
|
|
1764
|
-
if (
|
|
2006
|
+
if (
|
|
2007
|
+
message.header.mode instanceof AnyWhere ||
|
|
2008
|
+
message.header.mode instanceof AcknowledgeAnyWhere
|
|
2009
|
+
) {
|
|
1765
2010
|
isForMe = true;
|
|
1766
2011
|
} else {
|
|
1767
2012
|
const isFromSelf = this.publicKey.equals(from);
|
|
1768
|
-
if (
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
message.header.mode
|
|
2013
|
+
if (
|
|
2014
|
+
!isFromSelf &&
|
|
2015
|
+
(message.header.mode instanceof SilentDelivery ||
|
|
2016
|
+
message.header.mode instanceof AcknowledgeDelivery)
|
|
2017
|
+
) {
|
|
2018
|
+
isForMe = message.header.mode.to.includes(this.publicKeyHash);
|
|
1772
2019
|
}
|
|
1773
2020
|
}
|
|
1774
2021
|
|
|
@@ -1804,25 +2051,17 @@ export abstract class DirectStream<
|
|
|
1804
2051
|
}
|
|
1805
2052
|
}
|
|
1806
2053
|
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
) && x !== peerStream,
|
|
1817
|
-
);
|
|
1818
|
-
if (to.length > 0) {
|
|
1819
|
-
this.relayMessage(from, message, to);
|
|
1820
|
-
}
|
|
1821
|
-
}
|
|
1822
|
-
} else {
|
|
2054
|
+
// Forward
|
|
2055
|
+
const shouldForward =
|
|
2056
|
+
seenBefore === 0 ||
|
|
2057
|
+
((message.header.mode instanceof AcknowledgeDelivery ||
|
|
2058
|
+
message.header.mode instanceof AcknowledgeAnyWhere) &&
|
|
2059
|
+
seenBefore < message.header.mode.redundancy);
|
|
2060
|
+
|
|
2061
|
+
if (shouldForward) {
|
|
2062
|
+
// DONT await this since it might introduce a dead-lock
|
|
1823
2063
|
this.relayMessage(from, message);
|
|
1824
2064
|
}
|
|
1825
|
-
}
|
|
1826
2065
|
}
|
|
1827
2066
|
|
|
1828
2067
|
public async verifyAndProcess(message: Message<any>) {
|
|
@@ -1846,14 +2085,20 @@ export abstract class DirectStream<
|
|
|
1846
2085
|
seenBefore: number,
|
|
1847
2086
|
) {
|
|
1848
2087
|
if (
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
seenBefore < message.header.mode.redundancy
|
|
2088
|
+
message.header.mode instanceof AcknowledgeDelivery ||
|
|
2089
|
+
message.header.mode instanceof AcknowledgeAnyWhere
|
|
1852
2090
|
) {
|
|
1853
|
-
const
|
|
1854
|
-
message.header.mode
|
|
1855
|
-
|
|
1856
|
-
|
|
2091
|
+
const isRecipient =
|
|
2092
|
+
message.header.mode instanceof AcknowledgeAnyWhere
|
|
2093
|
+
? true
|
|
2094
|
+
: message.header.mode.to.includes(this.publicKeyHash);
|
|
2095
|
+
if (
|
|
2096
|
+
!shouldAcknowledgeDataMessage({
|
|
2097
|
+
isRecipient,
|
|
2098
|
+
seenBefore,
|
|
2099
|
+
redundancy: message.header.mode.redundancy,
|
|
2100
|
+
})
|
|
2101
|
+
) {
|
|
1857
2102
|
return;
|
|
1858
2103
|
}
|
|
1859
2104
|
const signers = message.header.signatures!.publicKeys.map((x) =>
|
|
@@ -1870,14 +2115,18 @@ export abstract class DirectStream<
|
|
|
1870
2115
|
mode: new TracedDelivery(signers),
|
|
1871
2116
|
session: this.session,
|
|
1872
2117
|
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
2118
|
+
// include our origin for route-learning/dialer hints (best-effort privacy/anti-spam control):
|
|
2119
|
+
// only include once (seenBefore=0) and only if we have not recently pruned
|
|
2120
|
+
// a connection to any signer in the path
|
|
2121
|
+
origin:
|
|
2122
|
+
(message.header.mode instanceof AcknowledgeAnyWhere ||
|
|
2123
|
+
message.header.mode instanceof AcknowledgeDelivery) &&
|
|
2124
|
+
seenBefore === 0 &&
|
|
2125
|
+
!message.header.signatures!.publicKeys.find((x) =>
|
|
2126
|
+
this.prunedConnectionsCache?.has(x.hashcode()),
|
|
2127
|
+
)
|
|
2128
|
+
? new MultiAddrinfo(
|
|
2129
|
+
this.components.addressManager
|
|
1881
2130
|
.getAddresses()
|
|
1882
2131
|
.map((x) => x.toString()),
|
|
1883
2132
|
)
|
|
@@ -2020,33 +2269,42 @@ export abstract class DirectStream<
|
|
|
2020
2269
|
for (const remote of remotes) {
|
|
2021
2270
|
this.invalidateSession(remote);
|
|
2022
2271
|
}
|
|
2023
|
-
|
|
2272
|
+
// Best-effort liveness probe; never let background probe failures crash callers.
|
|
2273
|
+
void this.checkIsAlive(remotes).catch(() => false);
|
|
2024
2274
|
}
|
|
2025
2275
|
private async checkIsAlive(remotes: string[]) {
|
|
2026
2276
|
if (this.peers.size === 0) {
|
|
2027
2277
|
return false;
|
|
2028
2278
|
}
|
|
2029
2279
|
if (remotes.length > 0) {
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2280
|
+
try {
|
|
2281
|
+
// Keepalive probes are best-effort, but must still require ACKs so we can
|
|
2282
|
+
// conclusively prune stale routes when peers actually went away.
|
|
2283
|
+
await this.publish(undefined, {
|
|
2284
|
+
mode: new AcknowledgeDelivery({
|
|
2285
|
+
to: remotes,
|
|
2286
|
+
redundancy: DEFAULT_SILENT_MESSAGE_REDUDANCY,
|
|
2287
|
+
}),
|
|
2288
|
+
});
|
|
2289
|
+
return true;
|
|
2290
|
+
} catch (e: any) {
|
|
2291
|
+
const errorName =
|
|
2292
|
+
e?.name ?? e?.constructor?.name ?? "UnknownError";
|
|
2293
|
+
if (
|
|
2294
|
+
e instanceof DeliveryError ||
|
|
2295
|
+
e instanceof NotStartedError ||
|
|
2296
|
+
e instanceof TimeoutError ||
|
|
2297
|
+
e instanceof AbortError ||
|
|
2298
|
+
errorName === "DeliveryError" ||
|
|
2299
|
+
errorName === "NotStartedError" ||
|
|
2300
|
+
errorName === "TimeoutError" ||
|
|
2301
|
+
errorName === "AbortError"
|
|
2302
|
+
) {
|
|
2303
|
+
return false;
|
|
2304
|
+
}
|
|
2305
|
+
warn(`checkIsAlive unexpected error: ${errorName}`);
|
|
2306
|
+
return false;
|
|
2307
|
+
}
|
|
2050
2308
|
}
|
|
2051
2309
|
return false;
|
|
2052
2310
|
}
|
|
@@ -2059,7 +2317,11 @@ export abstract class DirectStream<
|
|
|
2059
2317
|
) {
|
|
2060
2318
|
// dispatch the event if we are interested
|
|
2061
2319
|
|
|
2062
|
-
let mode:
|
|
2320
|
+
let mode:
|
|
2321
|
+
| SilentDelivery
|
|
2322
|
+
| AcknowledgeDelivery
|
|
2323
|
+
| AcknowledgeAnyWhere
|
|
2324
|
+
| AnyWhere = (
|
|
2063
2325
|
options as WithMode
|
|
2064
2326
|
).mode
|
|
2065
2327
|
? (options as WithMode).mode!
|
|
@@ -2070,8 +2332,7 @@ export abstract class DirectStream<
|
|
|
2070
2332
|
|
|
2071
2333
|
if (
|
|
2072
2334
|
mode instanceof AcknowledgeDelivery ||
|
|
2073
|
-
mode instanceof SilentDelivery
|
|
2074
|
-
mode instanceof SeekDelivery
|
|
2335
|
+
mode instanceof SilentDelivery
|
|
2075
2336
|
) {
|
|
2076
2337
|
if (mode.to) {
|
|
2077
2338
|
let preLength = mode.to.length;
|
|
@@ -2083,7 +2344,7 @@ export abstract class DirectStream<
|
|
|
2083
2344
|
);
|
|
2084
2345
|
}
|
|
2085
2346
|
|
|
2086
|
-
if (mode.to.length === 0
|
|
2347
|
+
if (mode.to.length === 0) {
|
|
2087
2348
|
throw new InvalidMessageError(
|
|
2088
2349
|
"Unexpected to deliver message with mode: " +
|
|
2089
2350
|
mode.constructor.name +
|
|
@@ -2094,28 +2355,6 @@ export abstract class DirectStream<
|
|
|
2094
2355
|
}
|
|
2095
2356
|
}
|
|
2096
2357
|
|
|
2097
|
-
if (mode instanceof AcknowledgeDelivery || mode instanceof SilentDelivery) {
|
|
2098
|
-
const now = +new Date();
|
|
2099
|
-
for (const hash of mode.to) {
|
|
2100
|
-
const neighbourRoutes = this.routes.routes
|
|
2101
|
-
.get(this.publicKeyHash)
|
|
2102
|
-
?.get(hash);
|
|
2103
|
-
|
|
2104
|
-
if (
|
|
2105
|
-
!neighbourRoutes ||
|
|
2106
|
-
now - neighbourRoutes.session >
|
|
2107
|
-
neighbourRoutes.list.length * this.routeSeekInterval ||
|
|
2108
|
-
!this.routes.isUpToDate(hash, neighbourRoutes)
|
|
2109
|
-
) {
|
|
2110
|
-
mode = new SeekDelivery({
|
|
2111
|
-
to: mode.to,
|
|
2112
|
-
redundancy: DEFAULT_SEEK_MESSAGE_REDUDANCY,
|
|
2113
|
-
});
|
|
2114
|
-
break;
|
|
2115
|
-
}
|
|
2116
|
-
}
|
|
2117
|
-
}
|
|
2118
|
-
|
|
2119
2358
|
const message = new DataMessage({
|
|
2120
2359
|
data: data instanceof Uint8ArrayList ? data.subarray() : data,
|
|
2121
2360
|
header: new MessageHeader({
|
|
@@ -2142,7 +2381,9 @@ export abstract class DirectStream<
|
|
|
2142
2381
|
async publish(
|
|
2143
2382
|
data: Uint8Array | Uint8ArrayList | undefined,
|
|
2144
2383
|
options: PublishOptions = {
|
|
2145
|
-
mode: new
|
|
2384
|
+
mode: new AcknowledgeAnyWhere({
|
|
2385
|
+
redundancy: DEFAULT_SEEK_MESSAGE_REDUDANCY,
|
|
2386
|
+
}),
|
|
2146
2387
|
},
|
|
2147
2388
|
): Promise<Uint8Array | undefined> {
|
|
2148
2389
|
if (!this.started) {
|
|
@@ -2187,7 +2428,7 @@ export abstract class DirectStream<
|
|
|
2187
2428
|
if (message instanceof DataMessage) {
|
|
2188
2429
|
if (
|
|
2189
2430
|
message.header.mode instanceof AcknowledgeDelivery ||
|
|
2190
|
-
message.header.mode instanceof
|
|
2431
|
+
message.header.mode instanceof AcknowledgeAnyWhere
|
|
2191
2432
|
) {
|
|
2192
2433
|
await message.sign(this.sign);
|
|
2193
2434
|
}
|
|
@@ -2215,6 +2456,37 @@ export abstract class DirectStream<
|
|
|
2215
2456
|
this.healthChecks.delete(to);
|
|
2216
2457
|
}
|
|
2217
2458
|
|
|
2459
|
+
private formatDeliveryDebugState(
|
|
2460
|
+
targets: Set<string>,
|
|
2461
|
+
fastestNodesReached: Map<string, number[]>,
|
|
2462
|
+
): string {
|
|
2463
|
+
const sampleSize = 16;
|
|
2464
|
+
const sampledTargets = [...targets].slice(0, sampleSize);
|
|
2465
|
+
const sampled = sampledTargets.map((target) => {
|
|
2466
|
+
const route = this.routes.findNeighbor(this.publicKeyHash, target);
|
|
2467
|
+
const routeSample =
|
|
2468
|
+
route?.list
|
|
2469
|
+
.slice(0, 3)
|
|
2470
|
+
.map((r) => `${r.hash}:${r.distance}${r.expireAt ? ":old" : ""}`)
|
|
2471
|
+
.join(",") ?? "-";
|
|
2472
|
+
const reachable = this.routes.isReachable(this.publicKeyHash, target);
|
|
2473
|
+
const stream = this.peers.get(target);
|
|
2474
|
+
let openConnections = 0;
|
|
2475
|
+
if (stream) {
|
|
2476
|
+
try {
|
|
2477
|
+
openConnections = this.components.connectionManager.getConnections(
|
|
2478
|
+
stream.peerId,
|
|
2479
|
+
).length;
|
|
2480
|
+
} catch {
|
|
2481
|
+
openConnections = stream.isReadable || stream.isWritable ? 1 : 0;
|
|
2482
|
+
}
|
|
2483
|
+
}
|
|
2484
|
+
return `${target}{reachable=${reachable ? 1 : 0},ack=${fastestNodesReached.has(target) ? 1 : 0},stream=${stream ? 1 : 0},conns=${openConnections},routes=${routeSample}}`;
|
|
2485
|
+
});
|
|
2486
|
+
const more = targets.size - sampledTargets.length;
|
|
2487
|
+
return `deliveryState peers=${this.peers.size} routesLocal=${this.routes.count()} routesAll=${this.routes.countAll()} targets=[${sampled.join(";")}]${more > 0 ? ` (+${more} more)` : ""}`;
|
|
2488
|
+
}
|
|
2489
|
+
|
|
2218
2490
|
private async createDeliveryPromise(
|
|
2219
2491
|
from: PublicSignKey,
|
|
2220
2492
|
message: DataMessage | Goodbye,
|
|
@@ -2238,11 +2510,9 @@ export abstract class DirectStream<
|
|
|
2238
2510
|
|
|
2239
2511
|
const fastestNodesReached = new Map<string, number[]>();
|
|
2240
2512
|
const messageToSet: Set<string> = new Set();
|
|
2241
|
-
if (message.header.mode
|
|
2513
|
+
if (deliveryModeHasReceiver(message.header.mode)) {
|
|
2242
2514
|
for (const to of message.header.mode.to) {
|
|
2243
|
-
if (to === from.hashcode())
|
|
2244
|
-
continue;
|
|
2245
|
-
}
|
|
2515
|
+
if (to === from.hashcode()) continue;
|
|
2246
2516
|
messageToSet.add(to);
|
|
2247
2517
|
|
|
2248
2518
|
if (!relayed && !this.healthChecks.has(to)) {
|
|
@@ -2276,29 +2546,25 @@ export abstract class DirectStream<
|
|
|
2276
2546
|
const willGetAllAcknowledgements = !relayed; // Only the origin will get all acks
|
|
2277
2547
|
|
|
2278
2548
|
// Expected to receive at least 'filterMessageForSeenCounter' acknowledgements from each peer
|
|
2279
|
-
const filterMessageForSeenCounter = relayed
|
|
2280
|
-
? undefined
|
|
2281
|
-
: message.header.mode instanceof SeekDelivery
|
|
2282
|
-
? Math.min(this.peers.size, message.header.mode.redundancy)
|
|
2283
|
-
: 1; /* message.deliveryMode instanceof SeekDelivery ? Math.min(this.peers.size - (relayed ? 1 : 0), message.deliveryMode.redundancy) : 1 */
|
|
2549
|
+
const filterMessageForSeenCounter = relayed ? undefined : 1;
|
|
2284
2550
|
|
|
2285
2551
|
const uniqueAcks = new Set();
|
|
2286
2552
|
const session = +new Date();
|
|
2287
2553
|
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2554
|
+
const onUnreachable =
|
|
2555
|
+
!relayed &&
|
|
2556
|
+
((ev: any) => {
|
|
2557
|
+
const target = ev.detail.hashcode();
|
|
2558
|
+
if (messageToSet.has(target)) {
|
|
2559
|
+
// A peer can transiently appear unreachable while routes are being updated.
|
|
2560
|
+
// Keep waiting for ACK/timeout instead of failing delivery immediately.
|
|
2561
|
+
logger.trace(
|
|
2562
|
+
"peer unreachable during delivery (msg=%s, target=%s); waiting for ACK/timeout",
|
|
2563
|
+
idString,
|
|
2564
|
+
target,
|
|
2565
|
+
);
|
|
2566
|
+
}
|
|
2567
|
+
});
|
|
2302
2568
|
|
|
2303
2569
|
onUnreachable && this.addEventListener("peer:unreachable", onUnreachable);
|
|
2304
2570
|
|
|
@@ -2311,8 +2577,8 @@ export abstract class DirectStream<
|
|
|
2311
2577
|
onAbort && signal?.removeEventListener("abort", onAbort);
|
|
2312
2578
|
};
|
|
2313
2579
|
|
|
2314
|
-
|
|
2315
|
-
|
|
2580
|
+
const timeout = setTimeout(async () => {
|
|
2581
|
+
clear();
|
|
2316
2582
|
|
|
2317
2583
|
let hasAll = true;
|
|
2318
2584
|
|
|
@@ -2330,22 +2596,31 @@ export abstract class DirectStream<
|
|
|
2330
2596
|
}
|
|
2331
2597
|
}
|
|
2332
2598
|
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
|
|
2599
|
+
if (!hasAll && willGetAllAcknowledgements) {
|
|
2600
|
+
const debugState = this.formatDeliveryDebugState(
|
|
2601
|
+
messageToSet,
|
|
2602
|
+
fastestNodesReached,
|
|
2603
|
+
);
|
|
2604
|
+
const msgMeta = `msgType=${message.constructor.name} dataBytes=${
|
|
2605
|
+
message instanceof DataMessage
|
|
2606
|
+
? (message.data?.byteLength ?? 0)
|
|
2607
|
+
: 0
|
|
2608
|
+
} relayed=${relayed ? 1 : 0}`;
|
|
2609
|
+
deliveryDeferredPromise.reject(
|
|
2610
|
+
new DeliveryError(
|
|
2611
|
+
`Failed to get message ${idString} ${filterMessageForSeenCounter} ${[
|
|
2612
|
+
...messageToSet,
|
|
2613
|
+
]} delivery acknowledges from all nodes (${
|
|
2614
|
+
fastestNodesReached.size
|
|
2615
|
+
}/${messageToSet.size}). Mode: ${
|
|
2616
|
+
message.header.mode.constructor.name
|
|
2617
|
+
}. Redundancy: ${(message.header.mode as any)["redundancy"]}. ${msgMeta}. ${debugState}`,
|
|
2618
|
+
),
|
|
2619
|
+
);
|
|
2345
2620
|
} else {
|
|
2346
2621
|
deliveryDeferredPromise.resolve();
|
|
2347
2622
|
}
|
|
2348
|
-
|
|
2623
|
+
}, this.seekTimeout);
|
|
2349
2624
|
|
|
2350
2625
|
if (signal) {
|
|
2351
2626
|
onAbort = () => {
|
|
@@ -2359,22 +2634,33 @@ export abstract class DirectStream<
|
|
|
2359
2634
|
}
|
|
2360
2635
|
}
|
|
2361
2636
|
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2637
|
+
const checkDone = () => {
|
|
2638
|
+
// This if clause should never enter for relayed connections, since we don't
|
|
2639
|
+
// know how many ACKs we will get
|
|
2640
|
+
if (
|
|
2641
|
+
filterMessageForSeenCounter != null &&
|
|
2642
|
+
uniqueAcks.size >= messageToSet.size * filterMessageForSeenCounter
|
|
2643
|
+
) {
|
|
2644
|
+
const shouldKeepCallbackForRouteLearning =
|
|
2645
|
+
!relayed &&
|
|
2646
|
+
message.header.mode instanceof AcknowledgeDelivery &&
|
|
2647
|
+
message.header.mode.redundancy > 1;
|
|
2648
|
+
if (haveReceivers && !shouldKeepCallbackForRouteLearning) {
|
|
2649
|
+
// If we have an explicit recipient list we can clear the ACK callback once we
|
|
2650
|
+
// got the expected acknowledgements.
|
|
2651
|
+
clear();
|
|
2652
|
+
} else if (haveReceivers && shouldKeepCallbackForRouteLearning) {
|
|
2653
|
+
// Resolve delivery early, but keep ACK callbacks alive until timeout so we can
|
|
2654
|
+
// learn additional redundant routes (seenCounter=1..redundancy-1).
|
|
2655
|
+
onUnreachable &&
|
|
2656
|
+
this.removeEventListener("peer:unreachable", onUnreachable);
|
|
2657
|
+
onAbort && signal?.removeEventListener("abort", onAbort);
|
|
2658
|
+
onAbort = undefined;
|
|
2659
|
+
}
|
|
2374
2660
|
|
|
2375
|
-
|
|
2376
|
-
|
|
2377
|
-
|
|
2661
|
+
deliveryDeferredPromise.resolve();
|
|
2662
|
+
return true;
|
|
2663
|
+
}
|
|
2378
2664
|
return false;
|
|
2379
2665
|
};
|
|
2380
2666
|
|
|
@@ -2388,26 +2674,39 @@ export abstract class DirectStream<
|
|
|
2388
2674
|
// remove the automatic removal of route timeout since we have observed lifesigns of a peer
|
|
2389
2675
|
this.clearHealthcheckTimer(messageTargetHash);
|
|
2390
2676
|
|
|
2391
|
-
//
|
|
2392
|
-
//
|
|
2393
|
-
|
|
2677
|
+
// If the target is not inside the original message `to`, we can still add the target to our routes.
|
|
2678
|
+
// This matters because relays may modify `to`, and because "ack-anywhere" style probes intentionally
|
|
2679
|
+
// do not provide an explicit recipient list.
|
|
2680
|
+
if (
|
|
2681
|
+
message.header.mode instanceof AcknowledgeDelivery ||
|
|
2682
|
+
message.header.mode instanceof AcknowledgeAnyWhere
|
|
2683
|
+
) {
|
|
2684
|
+
const upstreamHash = messageFrom?.publicKey.hashcode();
|
|
2685
|
+
|
|
2686
|
+
const routeUpdate = computeSeekAckRouteUpdate({
|
|
2687
|
+
current: this.publicKeyHash,
|
|
2688
|
+
upstream: upstreamHash,
|
|
2689
|
+
downstream: messageThrough.publicKey.hashcode(),
|
|
2690
|
+
target: messageTargetHash,
|
|
2691
|
+
// Route "distance" is based on recipient-seen order (0 = fastest). This is relied upon by
|
|
2692
|
+
// `Routes.getFanout(...)` which uses `distance < redundancy` to select redundant next-hops.
|
|
2693
|
+
distance: seenCounter,
|
|
2694
|
+
});
|
|
2695
|
+
|
|
2394
2696
|
this.addRouteConnection(
|
|
2395
|
-
|
|
2396
|
-
|
|
2697
|
+
routeUpdate.from,
|
|
2698
|
+
routeUpdate.neighbour,
|
|
2397
2699
|
messageTarget,
|
|
2398
|
-
|
|
2700
|
+
routeUpdate.distance,
|
|
2399
2701
|
session,
|
|
2400
2702
|
Number(ack.header.session),
|
|
2401
|
-
);
|
|
2703
|
+
);
|
|
2402
2704
|
}
|
|
2403
2705
|
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
seenCounter < filterMessageForSeenCounter
|
|
2409
|
-
) {
|
|
2410
|
-
// TODO set limit correctly
|
|
2706
|
+
if (messageToSet.has(messageTargetHash)) {
|
|
2707
|
+
// Any ACK from the target should satisfy delivery semantics.
|
|
2708
|
+
// Relying on only seenCounter=0 can fail under churn if the first ACK
|
|
2709
|
+
// is lost but a later ACK (seenCounter>0) arrives.
|
|
2411
2710
|
if (seenCounter < MAX_ROUTE_DISTANCE) {
|
|
2412
2711
|
let arr = fastestNodesReached.get(messageTargetHash);
|
|
2413
2712
|
if (!arr) {
|
|
@@ -2416,10 +2715,9 @@ export abstract class DirectStream<
|
|
|
2416
2715
|
}
|
|
2417
2716
|
arr.push(seenCounter);
|
|
2418
2717
|
|
|
2419
|
-
uniqueAcks.add(messageTargetHash
|
|
2718
|
+
uniqueAcks.add(messageTargetHash);
|
|
2420
2719
|
}
|
|
2421
2720
|
}
|
|
2422
|
-
}
|
|
2423
2721
|
|
|
2424
2722
|
checkDone();
|
|
2425
2723
|
},
|
|
@@ -2443,7 +2741,9 @@ export abstract class DirectStream<
|
|
|
2443
2741
|
throw new NotStartedError();
|
|
2444
2742
|
}
|
|
2445
2743
|
|
|
2744
|
+
const isRelayed = relayed ?? from.hashcode() !== this.publicKeyHash;
|
|
2446
2745
|
let delivereyPromise: Promise<void> | undefined = undefined as any;
|
|
2746
|
+
let ackCallbackId: string | undefined;
|
|
2447
2747
|
|
|
2448
2748
|
if (
|
|
2449
2749
|
(!message.header.signatures ||
|
|
@@ -2458,123 +2758,182 @@ export abstract class DirectStream<
|
|
|
2458
2758
|
* Logic for handling acknowledge messages when we receive them (later)
|
|
2459
2759
|
*/
|
|
2460
2760
|
|
|
2461
|
-
if (
|
|
2462
|
-
message instanceof DataMessage &&
|
|
2463
|
-
message.header.mode instanceof SeekDelivery &&
|
|
2464
|
-
!relayed
|
|
2465
|
-
) {
|
|
2466
|
-
to = this.peers; // seek delivery will not work unless we try all possible paths
|
|
2467
|
-
}
|
|
2468
|
-
|
|
2469
|
-
if (message.header.mode instanceof AcknowledgeDelivery) {
|
|
2470
|
-
to = undefined;
|
|
2471
|
-
}
|
|
2472
|
-
|
|
2473
2761
|
if (
|
|
2474
2762
|
(message instanceof DataMessage || message instanceof Goodbye) &&
|
|
2475
|
-
(message.header.mode instanceof
|
|
2476
|
-
message.header.mode instanceof
|
|
2763
|
+
(message.header.mode instanceof AcknowledgeDelivery ||
|
|
2764
|
+
message.header.mode instanceof AcknowledgeAnyWhere)
|
|
2477
2765
|
) {
|
|
2478
2766
|
const deliveryDeferredPromise = await this.createDeliveryPromise(
|
|
2479
2767
|
from,
|
|
2480
2768
|
message,
|
|
2481
|
-
|
|
2769
|
+
isRelayed,
|
|
2482
2770
|
signal,
|
|
2483
2771
|
);
|
|
2484
2772
|
delivereyPromise = deliveryDeferredPromise.promise;
|
|
2773
|
+
ackCallbackId = toBase64(message.id);
|
|
2485
2774
|
}
|
|
2486
2775
|
|
|
2487
|
-
|
|
2776
|
+
try {
|
|
2777
|
+
const bytes = message.bytes();
|
|
2488
2778
|
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
|
|
2492
|
-
|
|
2779
|
+
if (!isRelayed) {
|
|
2780
|
+
const bytesArray = bytes instanceof Uint8Array ? bytes : bytes.subarray();
|
|
2781
|
+
await this.modifySeenCache(bytesArray);
|
|
2782
|
+
}
|
|
2493
2783
|
|
|
2494
|
-
|
|
2495
|
-
|
|
2496
|
-
|
|
2784
|
+
/**
|
|
2785
|
+
* For non SEEKing message delivery modes, use routing
|
|
2786
|
+
*/
|
|
2497
2787
|
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
|
|
2788
|
+
if (message instanceof DataMessage) {
|
|
2789
|
+
if (
|
|
2790
|
+
(message.header.mode instanceof AcknowledgeDelivery ||
|
|
2791
|
+
message.header.mode instanceof SilentDelivery) &&
|
|
2792
|
+
!to
|
|
2793
|
+
) {
|
|
2794
|
+
if (message.header.mode.to.length === 0) {
|
|
2795
|
+
// we definitely know that we should not forward the message anywhere
|
|
2796
|
+
return delivereyPromise;
|
|
2797
|
+
}
|
|
2507
2798
|
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
|
|
2511
|
-
|
|
2512
|
-
|
|
2799
|
+
const fanout = this.routes.getFanout(
|
|
2800
|
+
from.hashcode(),
|
|
2801
|
+
message.header.mode.to,
|
|
2802
|
+
message.header.mode.redundancy,
|
|
2803
|
+
);
|
|
2513
2804
|
|
|
2514
|
-
|
|
2515
|
-
|
|
2805
|
+
// If we have explicit routing information, send only along the chosen next-hops.
|
|
2806
|
+
// If `fanout` is empty (no route info yet), fall through to the flooding logic below
|
|
2807
|
+
// so acknowledged deliveries can discover/repair routes instead of timing out.
|
|
2808
|
+
if (fanout && fanout.size > 0) {
|
|
2516
2809
|
const promises: Promise<any>[] = [];
|
|
2810
|
+
const usedNeighbours = new Set<string>();
|
|
2811
|
+
const originalTo = message.header.mode.to;
|
|
2517
2812
|
for (const [neighbour, _distantPeers] of fanout) {
|
|
2518
2813
|
const stream = this.peers.get(neighbour);
|
|
2519
|
-
stream
|
|
2814
|
+
if (!stream) continue;
|
|
2815
|
+
if (message.header.mode instanceof SilentDelivery) {
|
|
2816
|
+
message.header.mode.to = [..._distantPeers.keys()];
|
|
2520
2817
|
promises.push(
|
|
2521
|
-
stream.waitForWrite(bytes, message.header.priority),
|
|
2818
|
+
stream.waitForWrite(message.bytes(), message.header.priority),
|
|
2522
2819
|
);
|
|
2820
|
+
} else {
|
|
2821
|
+
promises.push(stream.waitForWrite(bytes, message.header.priority));
|
|
2822
|
+
}
|
|
2823
|
+
usedNeighbours.add(neighbour);
|
|
2523
2824
|
}
|
|
2825
|
+
if (message.header.mode instanceof SilentDelivery) {
|
|
2826
|
+
message.header.mode.to = originalTo;
|
|
2827
|
+
}
|
|
2828
|
+
|
|
2829
|
+
// If the sender requested redundancy but we don't yet have enough distinct
|
|
2830
|
+
// next-hops for the target(s), opportunistically probe additional neighbours.
|
|
2831
|
+
// This replaces the previous "greedy fanout" probing behavior without needing
|
|
2832
|
+
// a separate delivery mode.
|
|
2833
|
+
if (
|
|
2834
|
+
!isRelayed &&
|
|
2835
|
+
message.header.mode instanceof AcknowledgeDelivery &&
|
|
2836
|
+
usedNeighbours.size < message.header.mode.redundancy
|
|
2837
|
+
) {
|
|
2838
|
+
for (const [neighbour, stream] of this.peers) {
|
|
2839
|
+
if (usedNeighbours.size >= message.header.mode.redundancy) {
|
|
2840
|
+
break;
|
|
2841
|
+
}
|
|
2842
|
+
if (usedNeighbours.has(neighbour)) continue;
|
|
2843
|
+
usedNeighbours.add(neighbour);
|
|
2844
|
+
promises.push(stream.waitForWrite(bytes, message.header.priority));
|
|
2845
|
+
}
|
|
2846
|
+
}
|
|
2847
|
+
|
|
2524
2848
|
await Promise.all(promises);
|
|
2525
|
-
return delivereyPromise;
|
|
2849
|
+
return delivereyPromise;
|
|
2526
2850
|
}
|
|
2527
2851
|
|
|
2528
|
-
|
|
2852
|
+
// If we don't have routing information:
|
|
2853
|
+
// - For acknowledged deliveries, fall through to flooding (route discovery / repair).
|
|
2854
|
+
// - For silent deliveries, relays should not flood (prevents unnecessary fanout); origin may still flood.
|
|
2855
|
+
// We still allow direct neighbour delivery to explicit recipients (if connected).
|
|
2856
|
+
if (isRelayed && message.header.mode instanceof SilentDelivery) {
|
|
2857
|
+
const promises: Promise<any>[] = [];
|
|
2858
|
+
const originalTo = message.header.mode.to;
|
|
2859
|
+
for (const recipient of originalTo) {
|
|
2860
|
+
if (recipient === this.publicKeyHash) continue;
|
|
2861
|
+
if (recipient === from.hashcode()) continue; // never send back to previous hop
|
|
2862
|
+
const stream = this.peers.get(recipient);
|
|
2863
|
+
if (!stream) continue;
|
|
2864
|
+
if (
|
|
2865
|
+
message.header.signatures?.publicKeys.find(
|
|
2866
|
+
(x) => x.hashcode() === recipient,
|
|
2867
|
+
)
|
|
2868
|
+
) {
|
|
2869
|
+
continue; // recipient already signed/seen this message
|
|
2870
|
+
}
|
|
2871
|
+
message.header.mode.to = [recipient];
|
|
2872
|
+
promises.push(
|
|
2873
|
+
stream.waitForWrite(message.bytes(), message.header.priority),
|
|
2874
|
+
);
|
|
2875
|
+
}
|
|
2876
|
+
message.header.mode.to = originalTo;
|
|
2877
|
+
if (promises.length > 0) {
|
|
2878
|
+
await Promise.all(promises);
|
|
2879
|
+
}
|
|
2880
|
+
return delivereyPromise;
|
|
2881
|
+
}
|
|
2529
2882
|
}
|
|
2883
|
+
}
|
|
2530
2884
|
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
|
|
2534
|
-
|
|
2535
|
-
|
|
2536
|
-
|
|
2885
|
+
// We fail to send the message directly, instead fallback to floodsub
|
|
2886
|
+
const peers: PeerStreams[] | Map<string, PeerStreams> = to || this.peers;
|
|
2887
|
+
if (
|
|
2888
|
+
peers == null ||
|
|
2889
|
+
(Array.isArray(peers) && peers.length === 0) ||
|
|
2890
|
+
(peers instanceof Map && peers.size === 0)
|
|
2891
|
+
) {
|
|
2892
|
+
logger.trace("No peers to send to");
|
|
2893
|
+
return delivereyPromise;
|
|
2894
|
+
}
|
|
2537
2895
|
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
2541
|
-
|
|
2542
|
-
(Array.isArray(peers) && peers.length === 0) ||
|
|
2543
|
-
(peers instanceof Map && peers.size === 0)
|
|
2544
|
-
) {
|
|
2545
|
-
logger.trace("No peers to send to");
|
|
2546
|
-
return delivereyPromise;
|
|
2547
|
-
}
|
|
2896
|
+
let sentOnce = false;
|
|
2897
|
+
const promises: Promise<any>[] = [];
|
|
2898
|
+
for (const stream of peers.values()) {
|
|
2899
|
+
const id = stream as PeerStreams;
|
|
2548
2900
|
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2901
|
+
// Dont sent back to the sender
|
|
2902
|
+
if (id.publicKey.equals(from)) {
|
|
2903
|
+
continue;
|
|
2904
|
+
}
|
|
2905
|
+
// Dont send message back to any of the signers (they have already seen the message)
|
|
2906
|
+
if (
|
|
2907
|
+
message.header.signatures?.publicKeys.find((x) =>
|
|
2908
|
+
x.equals(id.publicKey),
|
|
2909
|
+
)
|
|
2910
|
+
) {
|
|
2911
|
+
continue;
|
|
2912
|
+
}
|
|
2553
2913
|
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
continue;
|
|
2557
|
-
}
|
|
2558
|
-
// Dont send message back to any of the signers (they have already seen the message)
|
|
2559
|
-
if (
|
|
2560
|
-
message.header.signatures?.publicKeys.find((x) =>
|
|
2561
|
-
x.equals(id.publicKey),
|
|
2562
|
-
)
|
|
2563
|
-
) {
|
|
2564
|
-
continue;
|
|
2914
|
+
sentOnce = true;
|
|
2915
|
+
promises.push(id.waitForWrite(bytes, message.header.priority));
|
|
2565
2916
|
}
|
|
2917
|
+
await Promise.all(promises);
|
|
2566
2918
|
|
|
2567
|
-
sentOnce
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
|
|
2919
|
+
if (!sentOnce) {
|
|
2920
|
+
// If the caller provided an explicit peer list, treat "no valid receivers" as an error
|
|
2921
|
+
// even when forwarding. This catches programming mistakes early and matches test expectations.
|
|
2922
|
+
if (!isRelayed || to != null) {
|
|
2923
|
+
throw new DeliveryError("Message did not have any valid receivers");
|
|
2924
|
+
}
|
|
2925
|
+
}
|
|
2571
2926
|
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
2927
|
+
return delivereyPromise;
|
|
2928
|
+
} catch (error) {
|
|
2929
|
+
// If message fanout/write fails before publishMessage returns its delivery
|
|
2930
|
+
// promise, clear any ACK callback to avoid late timer rejections leaking as
|
|
2931
|
+
// unhandled rejections in fire-and-forget call paths.
|
|
2932
|
+
if (ackCallbackId) {
|
|
2933
|
+
this._ackCallbacks.get(ackCallbackId)?.clear();
|
|
2575
2934
|
}
|
|
2935
|
+
throw error;
|
|
2576
2936
|
}
|
|
2577
|
-
return delivereyPromise;
|
|
2578
2937
|
}
|
|
2579
2938
|
|
|
2580
2939
|
async maybeConnectDirectly(toHash: string, origin: MultiAddrinfo) {
|
|
@@ -2697,8 +3056,11 @@ export abstract class DirectStream<
|
|
|
2697
3056
|
const wins = new Set<string>();
|
|
2698
3057
|
for (const h of admitted) if (reached(h, target)) wins.add(h);
|
|
2699
3058
|
|
|
2700
|
-
|
|
2701
|
-
|
|
3059
|
+
// Preserve input order in the returned list (important for deterministic callers/tests).
|
|
3060
|
+
const orderedWins = () => admitted.filter((h) => wins.has(h));
|
|
3061
|
+
|
|
3062
|
+
if (settle === "any" && wins.size > 0) return orderedWins();
|
|
3063
|
+
if (settle === "all" && wins.size === admitted.length) return orderedWins();
|
|
2702
3064
|
|
|
2703
3065
|
// Abort/timeout
|
|
2704
3066
|
const abortSignals = [this.closeController.signal];
|
|
@@ -2714,143 +3076,62 @@ export abstract class DirectStream<
|
|
|
2714
3076
|
return defer.resolve();
|
|
2715
3077
|
};
|
|
2716
3078
|
|
|
2717
|
-
try {
|
|
2718
|
-
await waitForEvent(this, eventsFor(target), check, {
|
|
2719
|
-
signals: abortSignals,
|
|
2720
|
-
timeout,
|
|
2721
|
-
});
|
|
2722
|
-
return [...wins];
|
|
2723
|
-
} catch (e) {
|
|
2724
|
-
const abortSignal = abortSignals.find((s) => s.aborted);
|
|
2725
|
-
if (abortSignal) {
|
|
2726
|
-
if (abortSignal.reason instanceof Error) {
|
|
2727
|
-
throw abortSignal.reason;
|
|
2728
|
-
}
|
|
2729
|
-
throw new AbortError(
|
|
2730
|
-
"Aborted waiting for peers: " + abortSignal.reason,
|
|
2731
|
-
);
|
|
2732
|
-
}
|
|
2733
|
-
if (e instanceof Error) {
|
|
2734
|
-
throw e;
|
|
2735
|
-
}
|
|
2736
|
-
if (settle === "all") throw new TimeoutError("Timeout waiting for peers");
|
|
2737
|
-
return [...wins]; // settle:any: return whatever successes we got
|
|
2738
|
-
}
|
|
2739
|
-
}
|
|
2740
|
-
|
|
2741
|
-
/* async waitFor(
|
|
2742
|
-
peer: PeerId | PublicSignKey | string,
|
|
2743
|
-
options?: {
|
|
2744
|
-
timeout?: number;
|
|
2745
|
-
signal?: AbortSignal;
|
|
2746
|
-
neighbour?: boolean;
|
|
2747
|
-
inflight?: boolean;
|
|
2748
|
-
},
|
|
2749
|
-
) {
|
|
2750
|
-
const hash =
|
|
2751
|
-
typeof peer === "string"
|
|
2752
|
-
? peer
|
|
2753
|
-
: (peer instanceof PublicSignKey
|
|
2754
|
-
? peer
|
|
2755
|
-
: getPublicKeyFromPeerId(peer)
|
|
2756
|
-
).hashcode();
|
|
2757
|
-
if (hash === this.publicKeyHash) {
|
|
2758
|
-
return; // TODO throw error instead?
|
|
2759
|
-
}
|
|
2760
|
-
|
|
2761
|
-
if (options?.inflight) {
|
|
2762
|
-
// if peer is not in active connections or dialQueue, return silenty
|
|
2763
|
-
if (
|
|
2764
|
-
!this.peers.has(hash) &&
|
|
2765
|
-
!this.components.connectionManager
|
|
2766
|
-
.getDialQueue()
|
|
2767
|
-
.some((x) => getPublicKeyFromPeerId(x.peerId).hashcode() === hash) &&
|
|
2768
|
-
!this.components.connectionManager
|
|
2769
|
-
.getConnections()
|
|
2770
|
-
.some((x) => getPublicKeyFromPeerId(x.remotePeer).hashcode() === hash)
|
|
2771
|
-
) {
|
|
2772
|
-
return;
|
|
2773
|
-
}
|
|
2774
|
-
}
|
|
2775
|
-
|
|
2776
|
-
const checkIsReachable = (deferred: DeferredPromise<void>) => {
|
|
2777
|
-
if (options?.neighbour && !this.peers.has(hash)) {
|
|
2778
|
-
return;
|
|
2779
|
-
}
|
|
2780
|
-
|
|
2781
|
-
if (!this.routes.isReachable(this.publicKeyHash, hash, 0)) {
|
|
2782
|
-
return;
|
|
2783
|
-
}
|
|
2784
|
-
|
|
2785
|
-
deferred.resolve();
|
|
2786
|
-
};
|
|
2787
|
-
const abortSignals = [this.closeController.signal];
|
|
2788
|
-
if (options?.signal) {
|
|
2789
|
-
abortSignals.push(options.signal);
|
|
2790
|
-
}
|
|
2791
|
-
|
|
2792
|
-
try {
|
|
2793
|
-
await waitForEvent(this, ["peer:reachable"], checkIsReachable, {
|
|
2794
|
-
signals: abortSignals,
|
|
2795
|
-
timeout: options?.timeout,
|
|
2796
|
-
});
|
|
2797
|
-
} catch (error) {
|
|
2798
|
-
throw new Error(
|
|
2799
|
-
"Stream to " +
|
|
2800
|
-
hash +
|
|
2801
|
-
" from " +
|
|
2802
|
-
this.publicKeyHash +
|
|
2803
|
-
" does not exist. Connection exist: " +
|
|
2804
|
-
this.peers.has(hash) +
|
|
2805
|
-
". Route exist: " +
|
|
2806
|
-
this.routes.isReachable(this.publicKeyHash, hash, 0),
|
|
2807
|
-
);
|
|
2808
|
-
}
|
|
2809
|
-
|
|
2810
|
-
if (options?.neighbour) {
|
|
2811
|
-
const stream = this.peers.get(hash)!;
|
|
2812
3079
|
try {
|
|
2813
|
-
|
|
2814
|
-
|
|
2815
|
-
|
|
3080
|
+
await waitForEvent(this, eventsFor(target), check, {
|
|
3081
|
+
signals: abortSignals,
|
|
3082
|
+
timeout,
|
|
3083
|
+
});
|
|
3084
|
+
return orderedWins();
|
|
3085
|
+
} catch (e) {
|
|
3086
|
+
const abortSignal = abortSignals.find((s) => s.aborted);
|
|
3087
|
+
if (abortSignal) {
|
|
3088
|
+
if (abortSignal.reason instanceof Error) {
|
|
3089
|
+
throw abortSignal.reason;
|
|
2816
3090
|
}
|
|
2817
|
-
|
|
2818
|
-
|
|
2819
|
-
|
|
2820
|
-
|
|
2821
|
-
|
|
2822
|
-
{
|
|
2823
|
-
|
|
2824
|
-
|
|
2825
|
-
|
|
2826
|
-
|
|
2827
|
-
|
|
2828
|
-
|
|
2829
|
-
|
|
2830
|
-
|
|
2831
|
-
|
|
2832
|
-
|
|
2833
|
-
|
|
2834
|
-
|
|
2835
|
-
|
|
3091
|
+
throw new AbortError(
|
|
3092
|
+
"Aborted waiting for peers: " + abortSignal.reason,
|
|
3093
|
+
);
|
|
3094
|
+
}
|
|
3095
|
+
if (e instanceof TimeoutError) {
|
|
3096
|
+
if (settle === "any") {
|
|
3097
|
+
if (wins.size > 0) return orderedWins();
|
|
3098
|
+
throw new TimeoutError(
|
|
3099
|
+
`Timeout waiting for peers (target=${target}, seek=${seek}, missing=${admitted.length}/${admitted.length})`,
|
|
3100
|
+
);
|
|
3101
|
+
}
|
|
3102
|
+
const missing = admitted.filter((h) => !wins.has(h));
|
|
3103
|
+
const preview = missing.slice(0, 5).join(", ");
|
|
3104
|
+
throw new TimeoutError(
|
|
3105
|
+
`Timeout waiting for peers (target=${target}, seek=${seek}, missing=${missing.length}/${admitted.length}${
|
|
3106
|
+
preview ? `, e.g. ${preview}` : ""
|
|
3107
|
+
})`,
|
|
3108
|
+
);
|
|
3109
|
+
}
|
|
3110
|
+
if (e instanceof Error) throw e;
|
|
3111
|
+
if (settle === "all") throw new TimeoutError("Timeout waiting for peers");
|
|
3112
|
+
return orderedWins(); // settle:any: return whatever successes we got
|
|
2836
3113
|
}
|
|
2837
3114
|
}
|
|
2838
|
-
} */
|
|
2839
3115
|
|
|
2840
|
-
|
|
2841
|
-
|
|
2842
|
-
|
|
3116
|
+
getPublicKey(hash: string): PublicSignKey | undefined {
|
|
3117
|
+
return this.peerKeyHashToPublicKey.get(hash);
|
|
3118
|
+
}
|
|
2843
3119
|
|
|
2844
3120
|
get pending(): boolean {
|
|
2845
3121
|
return this._ackCallbacks.size > 0;
|
|
2846
3122
|
}
|
|
2847
3123
|
|
|
2848
3124
|
// make this into a job? run every few ms
|
|
2849
|
-
|
|
2850
|
-
|
|
2851
|
-
if (this.
|
|
2852
|
-
|
|
2853
|
-
|
|
3125
|
+
maybePruneConnections(): Promise<void> {
|
|
3126
|
+
// Hard cap on peer streams: treat as a primary pruning signal.
|
|
3127
|
+
if (this.peers.size > this.connectionManagerOptions.maxConnections) {
|
|
3128
|
+
return this.pruneConnectionsToLimits();
|
|
3129
|
+
}
|
|
3130
|
+
|
|
3131
|
+
if (this.connectionManagerOptions.pruner) {
|
|
3132
|
+
if (this.connectionManagerOptions.pruner.bandwidth != null) {
|
|
3133
|
+
let usedBandwidth = 0;
|
|
3134
|
+
for (const [_k, v] of this.peers) {
|
|
2854
3135
|
usedBandwidth += v.usedBandwidth;
|
|
2855
3136
|
}
|
|
2856
3137
|
usedBandwidth /= this.peers.size;
|
|
@@ -2859,17 +3140,17 @@ export abstract class DirectStream<
|
|
|
2859
3140
|
// prune
|
|
2860
3141
|
return this.pruneConnections();
|
|
2861
3142
|
}
|
|
2862
|
-
|
|
2863
|
-
|
|
2864
|
-
|
|
2865
|
-
|
|
2866
|
-
|
|
3143
|
+
} else if (this.connectionManagerOptions.pruner.maxBuffer != null) {
|
|
3144
|
+
const queuedBytes = this.getQueuedBytes();
|
|
3145
|
+
if (queuedBytes > this.connectionManagerOptions.pruner.maxBuffer) {
|
|
3146
|
+
// prune
|
|
3147
|
+
return this.pruneConnections();
|
|
3148
|
+
}
|
|
2867
3149
|
}
|
|
2868
3150
|
}
|
|
2869
|
-
}
|
|
2870
3151
|
|
|
2871
|
-
|
|
2872
|
-
|
|
3152
|
+
return Promise.resolve();
|
|
3153
|
+
}
|
|
2873
3154
|
|
|
2874
3155
|
async pruneConnections(): Promise<void> {
|
|
2875
3156
|
// TODO sort by bandwidth
|
|
@@ -2880,17 +3161,17 @@ export abstract class DirectStream<
|
|
|
2880
3161
|
const sorted = [...this.peers.values()]
|
|
2881
3162
|
.sort((x, y) => x.usedBandwidth - y.usedBandwidth)
|
|
2882
3163
|
.map((x) => x.publicKey.hashcode());
|
|
2883
|
-
|
|
2884
|
-
|
|
2885
|
-
|
|
2886
|
-
|
|
3164
|
+
const prunables = this.routes.getPrunable(sorted);
|
|
3165
|
+
if (prunables.length === 0) {
|
|
3166
|
+
return;
|
|
3167
|
+
}
|
|
2887
3168
|
|
|
2888
|
-
|
|
2889
|
-
|
|
3169
|
+
const stream = this.peers.get(prunables[0])!;
|
|
3170
|
+
this.prunedConnectionsCache?.add(stream.publicKey.hashcode());
|
|
2890
3171
|
|
|
2891
|
-
|
|
2892
|
-
|
|
2893
|
-
|
|
3172
|
+
await this.onPeerDisconnected(stream.peerId);
|
|
3173
|
+
return this.components.connectionManager.closeConnections(stream.peerId);
|
|
3174
|
+
}
|
|
2894
3175
|
|
|
2895
3176
|
getQueuedBytes(): number {
|
|
2896
3177
|
let sum = 0;
|