@peerbit/pubsub 5.4.4 → 5.4.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/fanout-tree.d.ts +7 -2
- package/dist/src/fanout-tree.d.ts.map +1 -1
- package/dist/src/fanout-tree.js +339 -81
- package/dist/src/fanout-tree.js.map +1 -1
- package/dist/src/index.d.ts +3 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +303 -78
- package/dist/src/index.js.map +1 -1
- package/package.json +4 -4
- package/src/fanout-tree.ts +462 -111
- package/src/index.ts +381 -98
package/dist/src/fanout-tree.js
CHANGED
|
@@ -104,6 +104,14 @@ const createEmptyMetrics = () => ({
|
|
|
104
104
|
joinRejectSent: 0,
|
|
105
105
|
joinRejectReceived: 0,
|
|
106
106
|
joinPeerResets: 0,
|
|
107
|
+
joinBootstrapDialAttempts: 0,
|
|
108
|
+
joinBootstrapDialFailures: 0,
|
|
109
|
+
joinCandidateDialAttempts: 0,
|
|
110
|
+
joinCandidateDialFailures: 0,
|
|
111
|
+
joinConnectedCandidateAttempts: 0,
|
|
112
|
+
joinUnconnectedCandidateAttempts: 0,
|
|
113
|
+
joinReqTimeouts: 0,
|
|
114
|
+
joinDeadlineExpirations: 0,
|
|
107
115
|
kickSent: 0,
|
|
108
116
|
kickReceived: 0,
|
|
109
117
|
reparentDisconnect: 0,
|
|
@@ -2465,7 +2473,10 @@ export class FanoutTree extends DirectStream {
|
|
|
2465
2473
|
// Best-effort reset. The join loop keeps retrying other candidates.
|
|
2466
2474
|
}
|
|
2467
2475
|
}
|
|
2468
|
-
async _sendControl(to, bytes) {
|
|
2476
|
+
async _sendControl(to, bytes, signal) {
|
|
2477
|
+
if (signal?.aborted) {
|
|
2478
|
+
throw signal.reason ?? new AbortError("fanout control send aborted");
|
|
2479
|
+
}
|
|
2469
2480
|
const stream = this.peers.get(to);
|
|
2470
2481
|
if (!stream)
|
|
2471
2482
|
return;
|
|
@@ -2474,7 +2485,10 @@ export class FanoutTree extends DirectStream {
|
|
|
2474
2485
|
mode: new AnyWhere(),
|
|
2475
2486
|
priority: CONTROL_PRIORITY,
|
|
2476
2487
|
});
|
|
2477
|
-
|
|
2488
|
+
if (signal?.aborted) {
|
|
2489
|
+
throw signal.reason ?? new AbortError("fanout control send aborted");
|
|
2490
|
+
}
|
|
2491
|
+
await this.publishMessageMaybe(this.publicKey, message, [stream], undefined, signal);
|
|
2478
2492
|
}
|
|
2479
2493
|
async _sendControlMany(to, bytes) {
|
|
2480
2494
|
if (to.length === 0)
|
|
@@ -3082,34 +3096,138 @@ export class FanoutTree extends DirectStream {
|
|
|
3082
3096
|
}
|
|
3083
3097
|
return out;
|
|
3084
3098
|
}
|
|
3085
|
-
|
|
3099
|
+
isPeerReadyForJoin(hash) {
|
|
3100
|
+
const stream = this.peers.get(hash);
|
|
3101
|
+
if (!stream || !stream.isReadable || !stream.isWritable)
|
|
3102
|
+
return false;
|
|
3103
|
+
try {
|
|
3104
|
+
return (this.components.connectionManager.getConnections(stream.peerId).length > 0);
|
|
3105
|
+
}
|
|
3106
|
+
catch {
|
|
3107
|
+
// Test/mocked connection managers may not expose peer-scoped snapshots.
|
|
3108
|
+
return true;
|
|
3109
|
+
}
|
|
3110
|
+
}
|
|
3111
|
+
connectedPeerHashForBootstrap(address) {
|
|
3112
|
+
const peerId = address
|
|
3113
|
+
.getComponents()
|
|
3114
|
+
.filter((component) => component.name === "p2p")
|
|
3115
|
+
.at(-1)?.value;
|
|
3116
|
+
if (!peerId)
|
|
3117
|
+
return;
|
|
3118
|
+
for (const [hash, stream] of this.peers) {
|
|
3119
|
+
if (stream.peerId.toString() !== peerId)
|
|
3120
|
+
continue;
|
|
3121
|
+
if (this.isPeerReadyForJoin(hash))
|
|
3122
|
+
return hash;
|
|
3123
|
+
}
|
|
3124
|
+
return;
|
|
3125
|
+
}
|
|
3126
|
+
createBoundedDialAttempt(signal, timeoutMs, deadlineAt) {
|
|
3127
|
+
if (signal.aborted)
|
|
3128
|
+
return;
|
|
3129
|
+
const remainingMs = deadlineAt == null
|
|
3130
|
+
? Number.POSITIVE_INFINITY
|
|
3131
|
+
: Math.max(0, deadlineAt - Date.now());
|
|
3132
|
+
if (remainingMs <= 0)
|
|
3133
|
+
return;
|
|
3134
|
+
const boundedTimeoutMs = Math.max(1, Math.min(Math.max(1, Math.floor(timeoutMs)), remainingMs));
|
|
3135
|
+
const timeoutSignal = AbortSignal.timeout(boundedTimeoutMs);
|
|
3136
|
+
const combined = anySignal([signal, timeoutSignal]);
|
|
3137
|
+
return {
|
|
3138
|
+
signal: combined,
|
|
3139
|
+
timeoutMs: boundedTimeoutMs,
|
|
3140
|
+
clear: () => combined.clear?.(),
|
|
3141
|
+
};
|
|
3142
|
+
}
|
|
3143
|
+
async ensureBootstrapPeers(addrs, timeoutMs, signal, maxPeers = 0, diagnostics) {
|
|
3086
3144
|
if (addrs.length === 0)
|
|
3087
3145
|
return [];
|
|
3088
3146
|
const max = Math.max(0, Math.floor(maxPeers));
|
|
3089
|
-
const
|
|
3147
|
+
const connected = [];
|
|
3148
|
+
const disconnected = [];
|
|
3149
|
+
const connectedSeen = new Set();
|
|
3150
|
+
for (const address of addrs) {
|
|
3151
|
+
const readyHash = diagnostics
|
|
3152
|
+
? this.connectedPeerHashForBootstrap(address)
|
|
3153
|
+
: undefined;
|
|
3154
|
+
if (readyHash &&
|
|
3155
|
+
diagnostics?.excludeReadyPeerHashes?.has(readyHash)) {
|
|
3156
|
+
continue;
|
|
3157
|
+
}
|
|
3158
|
+
const hash = diagnostics?.preferConnected ? readyHash : undefined;
|
|
3159
|
+
if (!hash) {
|
|
3160
|
+
disconnected.push(address);
|
|
3161
|
+
continue;
|
|
3162
|
+
}
|
|
3163
|
+
if (!connectedSeen.has(hash)) {
|
|
3164
|
+
connectedSeen.add(hash);
|
|
3165
|
+
connected.push(hash);
|
|
3166
|
+
}
|
|
3167
|
+
}
|
|
3168
|
+
const connectedLimit = max > 0 ? Math.min(max, connected.length) : connected.length;
|
|
3169
|
+
const out = connected.slice(0, connectedLimit);
|
|
3170
|
+
if (diagnostics?.preferConnected && out.length > 0)
|
|
3171
|
+
return out;
|
|
3172
|
+
const shuffled = disconnected.slice();
|
|
3090
3173
|
for (let i = shuffled.length - 1; i > 0; i--) {
|
|
3091
3174
|
const j = Math.floor(this.random() * (i + 1));
|
|
3092
3175
|
const tmp = shuffled[i];
|
|
3093
3176
|
shuffled[i] = shuffled[j];
|
|
3094
3177
|
shuffled[j] = tmp;
|
|
3095
3178
|
}
|
|
3096
|
-
const target = max > 0 ? Math.min(max,
|
|
3097
|
-
const out = [];
|
|
3179
|
+
const target = max > 0 ? Math.min(max, addrs.length) : addrs.length;
|
|
3098
3180
|
for (const a of shuffled) {
|
|
3099
3181
|
if (signal.aborted)
|
|
3100
3182
|
break;
|
|
3101
3183
|
if (target > 0 && out.length >= target)
|
|
3102
3184
|
break;
|
|
3185
|
+
const attempt = diagnostics
|
|
3186
|
+
? this.createBoundedDialAttempt(signal, timeoutMs, diagnostics.deadlineAt)
|
|
3187
|
+
: undefined;
|
|
3188
|
+
if (diagnostics && !attempt)
|
|
3189
|
+
break;
|
|
3190
|
+
if (diagnostics)
|
|
3191
|
+
diagnostics.metrics.joinBootstrapDialAttempts += 1;
|
|
3192
|
+
let ready = false;
|
|
3193
|
+
let skippedExcluded = false;
|
|
3103
3194
|
try {
|
|
3104
|
-
const conn =
|
|
3195
|
+
const conn = attempt
|
|
3196
|
+
? await this.components.connectionManager.openConnection(a, {
|
|
3197
|
+
signal: attempt.signal,
|
|
3198
|
+
})
|
|
3199
|
+
: await this.components.connectionManager.openConnection(a);
|
|
3105
3200
|
const h = getPublicKeyFromPeerId(conn.remotePeer).hashcode();
|
|
3106
|
-
await this.waitFor(h, {
|
|
3107
|
-
|
|
3201
|
+
await this.waitFor(h, {
|
|
3202
|
+
seek: "present",
|
|
3203
|
+
timeout: attempt?.timeoutMs ?? timeoutMs,
|
|
3204
|
+
signal: attempt?.signal ?? signal,
|
|
3205
|
+
});
|
|
3206
|
+
skippedExcluded =
|
|
3207
|
+
diagnostics?.excludeReadyPeerHashes?.has(h) === true;
|
|
3208
|
+
ready =
|
|
3209
|
+
!skippedExcluded &&
|
|
3210
|
+
(diagnostics ? this.isPeerReadyForJoin(h) : true);
|
|
3211
|
+
if (ready)
|
|
3212
|
+
out.push(h);
|
|
3108
3213
|
}
|
|
3109
3214
|
catch {
|
|
3110
3215
|
// ignore dial failures
|
|
3111
3216
|
}
|
|
3217
|
+
finally {
|
|
3218
|
+
attempt?.clear();
|
|
3219
|
+
if (!ready && !skippedExcluded && diagnostics) {
|
|
3220
|
+
diagnostics.metrics.joinBootstrapDialFailures += 1;
|
|
3221
|
+
}
|
|
3222
|
+
}
|
|
3223
|
+
if (ready && diagnostics?.preferConnected)
|
|
3224
|
+
break;
|
|
3112
3225
|
}
|
|
3226
|
+
// Exhaustion completes one bounded pass over the configured bootstrap set.
|
|
3227
|
+
// Allow a later round to revisit ready trackers because their candidate view
|
|
3228
|
+
// may have changed while this cold join was progressing.
|
|
3229
|
+
if (out.length === 0)
|
|
3230
|
+
diagnostics?.excludeReadyPeerHashes?.clear();
|
|
3113
3231
|
return [...new Set(out)];
|
|
3114
3232
|
}
|
|
3115
3233
|
async announceToTrackers(ch, signal) {
|
|
@@ -3476,24 +3594,47 @@ export class FanoutTree extends DirectStream {
|
|
|
3476
3594
|
return true;
|
|
3477
3595
|
});
|
|
3478
3596
|
}
|
|
3479
|
-
async ensurePeerConnection(hash, addrs, timeoutMs, signal) {
|
|
3480
|
-
if (this.
|
|
3597
|
+
async ensurePeerConnection(hash, addrs, timeoutMs, signal, diagnostics) {
|
|
3598
|
+
if (this.isPeerReadyForJoin(hash))
|
|
3481
3599
|
return true;
|
|
3482
3600
|
for (const a of addrs) {
|
|
3483
3601
|
if (signal.aborted)
|
|
3484
3602
|
return false;
|
|
3603
|
+
const attempt = diagnostics
|
|
3604
|
+
? this.createBoundedDialAttempt(signal, timeoutMs, diagnostics.deadlineAt)
|
|
3605
|
+
: undefined;
|
|
3606
|
+
if (diagnostics && !attempt)
|
|
3607
|
+
return false;
|
|
3608
|
+
if (diagnostics)
|
|
3609
|
+
diagnostics.metrics.joinCandidateDialAttempts += 1;
|
|
3610
|
+
let ready = false;
|
|
3485
3611
|
try {
|
|
3486
|
-
|
|
3612
|
+
if (attempt) {
|
|
3613
|
+
await this.components.connectionManager.openConnection(a, {
|
|
3614
|
+
signal: attempt.signal,
|
|
3615
|
+
});
|
|
3616
|
+
}
|
|
3617
|
+
else {
|
|
3618
|
+
await this.components.connectionManager.openConnection(a);
|
|
3619
|
+
}
|
|
3487
3620
|
await this.waitFor(hash, {
|
|
3488
3621
|
seek: "present",
|
|
3489
|
-
timeout: timeoutMs,
|
|
3490
|
-
signal,
|
|
3622
|
+
timeout: attempt?.timeoutMs ?? timeoutMs,
|
|
3623
|
+
signal: attempt?.signal ?? signal,
|
|
3491
3624
|
});
|
|
3492
|
-
|
|
3625
|
+
ready = diagnostics ? this.isPeerReadyForJoin(hash) : true;
|
|
3493
3626
|
}
|
|
3494
3627
|
catch {
|
|
3495
3628
|
// ignore and try next
|
|
3496
3629
|
}
|
|
3630
|
+
finally {
|
|
3631
|
+
attempt?.clear();
|
|
3632
|
+
if (!ready && diagnostics) {
|
|
3633
|
+
diagnostics.metrics.joinCandidateDialFailures += 1;
|
|
3634
|
+
}
|
|
3635
|
+
}
|
|
3636
|
+
if (ready)
|
|
3637
|
+
return true;
|
|
3497
3638
|
}
|
|
3498
3639
|
return false;
|
|
3499
3640
|
}
|
|
@@ -3613,6 +3754,9 @@ export class FanoutTree extends DirectStream {
|
|
|
3613
3754
|
return;
|
|
3614
3755
|
await this._sendControlMany(trackerPeers, this.codec.encodeTrackerFeedback(ch.id.key, candidateHash, event, reason));
|
|
3615
3756
|
}
|
|
3757
|
+
sendTrackerFeedbackBestEffort(ch, trackerPeers, candidateHash, event, reason = 0) {
|
|
3758
|
+
void this.sendTrackerFeedback(ch, trackerPeers, candidateHash, event, reason).catch(() => { });
|
|
3759
|
+
}
|
|
3616
3760
|
pruneParentUpgradeReservations(ch, now = Date.now()) {
|
|
3617
3761
|
for (const [hash, reservation] of ch.parentUpgradeReservationsByHash) {
|
|
3618
3762
|
if (reservation.expiresAt <= now) {
|
|
@@ -3798,14 +3942,40 @@ export class FanoutTree extends DirectStream {
|
|
|
3798
3942
|
source: Number(joinOpts.candidateScoringWeights?.source ?? 0.25),
|
|
3799
3943
|
};
|
|
3800
3944
|
const start = Date.now();
|
|
3945
|
+
const initialJoinDeadlineAt = timeoutMs > 0 ? start + timeoutMs : undefined;
|
|
3801
3946
|
const cooldownUntilByHash = new Map();
|
|
3802
3947
|
const combinedSignal = joinOpts.signal
|
|
3803
3948
|
? anySignal([ch.closeController.signal, joinOpts.signal])
|
|
3804
3949
|
: ch.closeController.signal;
|
|
3805
3950
|
const signal = combinedSignal;
|
|
3951
|
+
const initialJoinRemainingMs = () => !ch.joinedAtLeastOnce && initialJoinDeadlineAt != null
|
|
3952
|
+
? Math.max(0, initialJoinDeadlineAt - Date.now())
|
|
3953
|
+
: undefined;
|
|
3954
|
+
const clampInitialJoinWait = (requestedMs) => {
|
|
3955
|
+
const remainingMs = initialJoinRemainingMs();
|
|
3956
|
+
return remainingMs == null
|
|
3957
|
+
? requestedMs
|
|
3958
|
+
: Math.min(requestedMs, remainingMs);
|
|
3959
|
+
};
|
|
3960
|
+
const throwInitialJoinTimeout = () => {
|
|
3961
|
+
ch.metrics.joinDeadlineExpirations += 1;
|
|
3962
|
+
const bootstrapsCount = this.getBootstrapsForChannel(ch).length;
|
|
3963
|
+
const rootPeer = this.peers.get(ch.id.root);
|
|
3964
|
+
const rootNeighbor = Boolean(rootPeer && rootPeer.isReadable && rootPeer.isWritable);
|
|
3965
|
+
const bootstrapHint = bootstrapsCount === 0 && !rootNeighbor
|
|
3966
|
+
? " No fanout bootstraps are configured for this channel, and the root is not a direct neighbor. If this peer reached the network via a bootstrap or relay node, initialize it with Peerbit.bootstrap(...) instead of Peerbit.dial(...), or configure FanoutTree.setBootstraps(...) before joining sharded topics."
|
|
3967
|
+
: "";
|
|
3968
|
+
throw new Error(`fanout join timed out after ${timeoutMs}ms (topic=${ch.id.topic} root=${ch.id.root} self=${this.publicKeyHash} rootNeighbor=${rootNeighbor} peers=${this.peers.size} bootstraps=${bootstrapsCount} joinReqSent=${ch.metrics.joinReqSent} joinAcceptReceived=${ch.metrics.joinAcceptReceived} joinRejectReceived=${ch.metrics.joinRejectReceived} peerResets=${ch.metrics.joinPeerResets}).${bootstrapHint}`);
|
|
3969
|
+
};
|
|
3970
|
+
const throwIfInitialJoinTimedOut = () => {
|
|
3971
|
+
if (initialJoinRemainingMs() === 0)
|
|
3972
|
+
throwInitialJoinTimeout();
|
|
3973
|
+
};
|
|
3806
3974
|
let nextParentUpgradeCheckAt = 0;
|
|
3807
3975
|
let parentUpgradeCheckSeq = 0;
|
|
3808
3976
|
let parentUpgradeActiveGuardBackoffMs = 0;
|
|
3977
|
+
const unsuccessfulColdBootstrapPeers = new Set();
|
|
3978
|
+
let bootstrapFallbackRetryAt = 0;
|
|
3809
3979
|
const scheduleNextParentUpgradeCheck = (now, first = false, minDelayMs = parentUpgrade.intervalMs) => {
|
|
3810
3980
|
if (parentUpgrade.intervalMs <= 0) {
|
|
3811
3981
|
nextParentUpgradeCheckAt = 0;
|
|
@@ -4004,30 +4174,28 @@ export class FanoutTree extends DirectStream {
|
|
|
4004
4174
|
await delay(Math.max(retryMs, 1_000), { signal });
|
|
4005
4175
|
continue;
|
|
4006
4176
|
}
|
|
4007
|
-
// `timeoutMs`
|
|
4008
|
-
//
|
|
4009
|
-
|
|
4010
|
-
|
|
4011
|
-
Date.now() - start > timeoutMs) {
|
|
4012
|
-
const bootstrapsCount = this.getBootstrapsForChannel(ch).length;
|
|
4013
|
-
const rootPeer = this.peers.get(ch.id.root);
|
|
4014
|
-
const rootNeighbor = Boolean(rootPeer && rootPeer.isReadable && rootPeer.isWritable);
|
|
4015
|
-
const bootstrapHint = bootstrapsCount === 0 && !rootNeighbor
|
|
4016
|
-
? " No fanout bootstraps are configured for this channel, and the root is not a direct neighbor. If this peer reached the network via a bootstrap or relay node, initialize it with Peerbit.bootstrap(...) instead of Peerbit.dial(...), or configure FanoutTree.setBootstraps(...) before joining sharded topics."
|
|
4017
|
-
: "";
|
|
4018
|
-
throw new Error(`fanout join timed out after ${timeoutMs}ms (topic=${ch.id.topic} root=${ch.id.root} self=${this.publicKeyHash} rootNeighbor=${rootNeighbor} peers=${this.peers.size} bootstraps=${bootstrapsCount} joinReqSent=${ch.metrics.joinReqSent} joinAcceptReceived=${ch.metrics.joinAcceptReceived} joinRejectReceived=${ch.metrics.joinRejectReceived} peerResets=${ch.metrics.joinPeerResets}).${bootstrapHint}`);
|
|
4019
|
-
}
|
|
4177
|
+
// `timeoutMs` bounds only the initial `joinChannel()` await. Re-parenting
|
|
4178
|
+
// remains unbounded after the first attachment, while every cold-open wait
|
|
4179
|
+
// below is clamped to this same absolute deadline.
|
|
4180
|
+
throwIfInitialJoinTimedOut();
|
|
4020
4181
|
const cooldownMs = ch.rejoinCooldownUntil - Date.now();
|
|
4021
4182
|
if (cooldownMs > 0) {
|
|
4022
|
-
|
|
4183
|
+
const waitMs = clampInitialJoinWait(cooldownMs);
|
|
4184
|
+
if (waitMs <= 0)
|
|
4185
|
+
throwInitialJoinTimeout();
|
|
4186
|
+
await delay(waitMs, { signal });
|
|
4023
4187
|
continue;
|
|
4024
4188
|
}
|
|
4025
4189
|
const bootstraps = this.getBootstrapsForChannel(ch);
|
|
4026
4190
|
let bootstrapPeers = [];
|
|
4027
4191
|
if (bootstraps.length > 0) {
|
|
4028
4192
|
const now = Date.now();
|
|
4029
|
-
const connectedCached = ch.cachedBootstrapPeers.filter((h) =>
|
|
4030
|
-
|
|
4193
|
+
const connectedCached = ch.cachedBootstrapPeers.filter((h) => this.isPeerReadyForJoin(h) &&
|
|
4194
|
+
!unsuccessfulColdBootstrapPeers.has(h));
|
|
4195
|
+
const hasExcludedReadyCached = ch.cachedBootstrapPeers.some((h) => this.isPeerReadyForJoin(h) &&
|
|
4196
|
+
unsuccessfulColdBootstrapPeers.has(h));
|
|
4197
|
+
const due = (hasExcludedReadyCached && now >= bootstrapFallbackRetryAt) ||
|
|
4198
|
+
ch.lastBootstrapEnsureAt === 0 ||
|
|
4031
4199
|
bootstrapEnsureIntervalMs === 0 ||
|
|
4032
4200
|
now - ch.lastBootstrapEnsureAt >= bootstrapEnsureIntervalMs;
|
|
4033
4201
|
const haveEnough = bootstrapMaxPeers > 0
|
|
@@ -4035,12 +4203,34 @@ export class FanoutTree extends DirectStream {
|
|
|
4035
4203
|
: false;
|
|
4036
4204
|
if (due && !haveEnough) {
|
|
4037
4205
|
ch.lastBootstrapEnsureAt = now;
|
|
4038
|
-
const
|
|
4039
|
-
|
|
4206
|
+
const wasFallbackPass = unsuccessfulColdBootstrapPeers.size > 0;
|
|
4207
|
+
const diagnostics = !ch.joinedAtLeastOnce
|
|
4208
|
+
? {
|
|
4209
|
+
metrics: ch.metrics,
|
|
4210
|
+
deadlineAt: initialJoinDeadlineAt,
|
|
4211
|
+
preferConnected: true,
|
|
4212
|
+
excludeReadyPeerHashes: unsuccessfulColdBootstrapPeers,
|
|
4213
|
+
}
|
|
4214
|
+
: undefined;
|
|
4215
|
+
const peers = await this.ensureBootstrapPeers(bootstraps, bootstrapDialTimeoutMs, signal, bootstrapMaxPeers, diagnostics);
|
|
4216
|
+
if (peers.length > 0) {
|
|
4217
|
+
const cohortChanged = peers.length !== ch.cachedBootstrapPeers.length ||
|
|
4218
|
+
peers.some((hash, index) => ch.cachedBootstrapPeers[index] !== hash);
|
|
4040
4219
|
ch.cachedBootstrapPeers = peers;
|
|
4220
|
+
if (cohortChanged) {
|
|
4221
|
+
ch.lastTrackerQueryAt = 0;
|
|
4222
|
+
ch.cachedTrackerCandidates = [];
|
|
4223
|
+
}
|
|
4224
|
+
}
|
|
4225
|
+
else if (wasFallbackPass) {
|
|
4226
|
+
bootstrapFallbackRetryAt =
|
|
4227
|
+
Date.now() + Math.max(1, bootstrapEnsureIntervalMs);
|
|
4228
|
+
}
|
|
4041
4229
|
}
|
|
4042
|
-
bootstrapPeers = ch.cachedBootstrapPeers.filter((h) =>
|
|
4230
|
+
bootstrapPeers = ch.cachedBootstrapPeers.filter((h) => this.isPeerReadyForJoin(h) &&
|
|
4231
|
+
!unsuccessfulColdBootstrapPeers.has(h));
|
|
4043
4232
|
}
|
|
4233
|
+
throwIfInitialJoinTimedOut();
|
|
4044
4234
|
let tracker = [];
|
|
4045
4235
|
if (bootstrapPeers.length > 0 && trackerCandidates > 0) {
|
|
4046
4236
|
const now = Date.now();
|
|
@@ -4049,12 +4239,16 @@ export class FanoutTree extends DirectStream {
|
|
|
4049
4239
|
now - ch.lastTrackerQueryAt >= trackerQueryIntervalMs;
|
|
4050
4240
|
if (due) {
|
|
4051
4241
|
ch.lastTrackerQueryAt = now;
|
|
4052
|
-
const
|
|
4242
|
+
const queryTimeoutMs = clampInitialJoinWait(Math.max(1, trackerQueryTimeoutMs));
|
|
4243
|
+
if (queryTimeoutMs <= 0)
|
|
4244
|
+
throwInitialJoinTimeout();
|
|
4245
|
+
const res = await this.queryTrackers(ch, bootstrapPeers, trackerCandidates, queryTimeoutMs, signal);
|
|
4053
4246
|
if (res.length > 0)
|
|
4054
4247
|
ch.cachedTrackerCandidates = res;
|
|
4055
4248
|
}
|
|
4056
4249
|
tracker = ch.cachedTrackerCandidates;
|
|
4057
4250
|
}
|
|
4251
|
+
throwIfInitialJoinTimedOut();
|
|
4058
4252
|
const candidatesByHash = new Map();
|
|
4059
4253
|
const upsertCandidate = (c) => {
|
|
4060
4254
|
const prev = candidatesByHash.get(c.hash);
|
|
@@ -4072,7 +4266,8 @@ export class FanoutTree extends DirectStream {
|
|
|
4072
4266
|
// Fast path: if the designated root is already a direct neighbor, try it first.
|
|
4073
4267
|
// Without this, large join storms can repeatedly time out on arbitrary peers
|
|
4074
4268
|
// that don't host the channel yet, starving the real root candidate.
|
|
4075
|
-
if (ch.id.root !== this.publicKeyHash &&
|
|
4269
|
+
if (ch.id.root !== this.publicKeyHash &&
|
|
4270
|
+
this.isPeerReadyForJoin(ch.id.root)) {
|
|
4076
4271
|
upsertCandidate({
|
|
4077
4272
|
hash: ch.id.root,
|
|
4078
4273
|
addrs: [],
|
|
@@ -4122,6 +4317,8 @@ export class FanoutTree extends DirectStream {
|
|
|
4122
4317
|
for (const h of this.peers.keys()) {
|
|
4123
4318
|
if (h === this.publicKeyHash)
|
|
4124
4319
|
continue;
|
|
4320
|
+
if (!this.isPeerReadyForJoin(h))
|
|
4321
|
+
continue;
|
|
4125
4322
|
if (bootstrapPeerSet.has(h) && candidatesByHash.has(h))
|
|
4126
4323
|
continue;
|
|
4127
4324
|
upsertCandidate({
|
|
@@ -4167,10 +4364,16 @@ export class FanoutTree extends DirectStream {
|
|
|
4167
4364
|
? Math.max(1, nextAt - now)
|
|
4168
4365
|
: retryMs;
|
|
4169
4366
|
const capMs = Math.max(retryMs, trackerQueryIntervalMs > 0 ? trackerQueryIntervalMs : retryMs);
|
|
4170
|
-
|
|
4367
|
+
const boundedWaitMs = clampInitialJoinWait(Math.max(1, Math.min(waitMs, capMs)));
|
|
4368
|
+
if (boundedWaitMs <= 0)
|
|
4369
|
+
throwInitialJoinTimeout();
|
|
4370
|
+
await delay(boundedWaitMs, { signal });
|
|
4171
4371
|
continue;
|
|
4172
4372
|
}
|
|
4173
|
-
|
|
4373
|
+
const waitMs = clampInitialJoinWait(retryMs);
|
|
4374
|
+
if (waitMs <= 0)
|
|
4375
|
+
throwInitialJoinTimeout();
|
|
4376
|
+
await delay(waitMs, { signal });
|
|
4174
4377
|
continue;
|
|
4175
4378
|
}
|
|
4176
4379
|
let ordered = [...candidates];
|
|
@@ -4185,6 +4388,13 @@ export class FanoutTree extends DirectStream {
|
|
|
4185
4388
|
ordered[j] = tmp;
|
|
4186
4389
|
}
|
|
4187
4390
|
}
|
|
4391
|
+
// Give one usable peer a latency advantage without moving every
|
|
4392
|
+
// connected fallback ahead of better-ranked dialable candidates.
|
|
4393
|
+
const firstReadyIndex = ordered.findIndex((candidate) => this.isPeerReadyForJoin(candidate.hash));
|
|
4394
|
+
if (firstReadyIndex > 0) {
|
|
4395
|
+
const [firstReady] = ordered.splice(firstReadyIndex, 1);
|
|
4396
|
+
ordered.unshift(firstReady);
|
|
4397
|
+
}
|
|
4188
4398
|
}
|
|
4189
4399
|
else if (candidateScoringMode === "weighted") {
|
|
4190
4400
|
const wLevel = Number.isFinite(candidateScoringWeights.level)
|
|
@@ -4266,20 +4476,24 @@ export class FanoutTree extends DirectStream {
|
|
|
4266
4476
|
break;
|
|
4267
4477
|
if (attempts >= joinAttemptsPerRound)
|
|
4268
4478
|
break;
|
|
4479
|
+
throwIfInitialJoinTimedOut();
|
|
4269
4480
|
const c = queue[i];
|
|
4270
4481
|
attempts += 1;
|
|
4271
|
-
const wasConnected =
|
|
4482
|
+
const wasConnected = this.isPeerReadyForJoin(c.hash);
|
|
4483
|
+
if (wasConnected) {
|
|
4484
|
+
ch.metrics.joinConnectedCandidateAttempts += 1;
|
|
4485
|
+
}
|
|
4486
|
+
else {
|
|
4487
|
+
ch.metrics.joinUnconnectedCandidateAttempts += 1;
|
|
4488
|
+
}
|
|
4272
4489
|
let dialOk = wasConnected;
|
|
4273
4490
|
if (!dialOk && c.addrs.length > 0) {
|
|
4274
|
-
dialOk = await this.ensurePeerConnection(c.hash, c.addrs, bootstrapDialTimeoutMs, signal
|
|
4491
|
+
dialOk = await this.ensurePeerConnection(c.hash, c.addrs, bootstrapDialTimeoutMs, signal, !ch.joinedAtLeastOnce
|
|
4492
|
+
? { metrics: ch.metrics, deadlineAt: initialJoinDeadlineAt }
|
|
4493
|
+
: undefined);
|
|
4275
4494
|
}
|
|
4276
4495
|
if (!dialOk) {
|
|
4277
|
-
|
|
4278
|
-
await this.sendTrackerFeedback(ch, bootstrapPeers, c.hash, TRACKER_FEEDBACK_DIAL_FAILED);
|
|
4279
|
-
}
|
|
4280
|
-
catch {
|
|
4281
|
-
// ignore
|
|
4282
|
-
}
|
|
4496
|
+
this.sendTrackerFeedbackBestEffort(ch, bootstrapPeers, c.hash, TRACKER_FEEDBACK_DIAL_FAILED);
|
|
4283
4497
|
if (candidateCooldownMs > 0) {
|
|
4284
4498
|
cooldownUntilByHash.set(c.hash, Date.now() + candidateCooldownMs * 5);
|
|
4285
4499
|
}
|
|
@@ -4289,7 +4503,10 @@ export class FanoutTree extends DirectStream {
|
|
|
4289
4503
|
dialedNew.add(c.hash);
|
|
4290
4504
|
}
|
|
4291
4505
|
const reqId = (this.random() * 0xffffffff) >>> 0;
|
|
4292
|
-
const
|
|
4506
|
+
const requestTimeoutMs = clampInitialJoinWait(Math.max(1, joinReqTimeoutMs));
|
|
4507
|
+
if (requestTimeoutMs <= 0)
|
|
4508
|
+
throwInitialJoinTimeout();
|
|
4509
|
+
const res = await this.tryJoinOnce(ch, c.hash, reqId, requestTimeoutMs, signal);
|
|
4293
4510
|
if (res.redirects && res.redirects.length > 0) {
|
|
4294
4511
|
for (const r of res.redirects) {
|
|
4295
4512
|
if (queue.length >= JOIN_REJECT_REDIRECT_QUEUE_MAX)
|
|
@@ -4315,12 +4532,7 @@ export class FanoutTree extends DirectStream {
|
|
|
4315
4532
|
}
|
|
4316
4533
|
}
|
|
4317
4534
|
if (res.ok) {
|
|
4318
|
-
|
|
4319
|
-
await this.sendTrackerFeedback(ch, bootstrapPeers, c.hash, TRACKER_FEEDBACK_JOINED);
|
|
4320
|
-
}
|
|
4321
|
-
catch {
|
|
4322
|
-
// ignore
|
|
4323
|
-
}
|
|
4535
|
+
this.sendTrackerFeedbackBestEffort(ch, bootstrapPeers, c.hash, TRACKER_FEEDBACK_JOINED);
|
|
4324
4536
|
cooldownUntilByHash.delete(c.hash);
|
|
4325
4537
|
break;
|
|
4326
4538
|
}
|
|
@@ -4336,13 +4548,9 @@ export class FanoutTree extends DirectStream {
|
|
|
4336
4548
|
}
|
|
4337
4549
|
}
|
|
4338
4550
|
if (res.timedOut) {
|
|
4551
|
+
ch.metrics.joinReqTimeouts += 1;
|
|
4339
4552
|
this.noteJoinTimeout(ch, c.hash);
|
|
4340
|
-
|
|
4341
|
-
await this.sendTrackerFeedback(ch, bootstrapPeers, c.hash, TRACKER_FEEDBACK_JOIN_TIMEOUT);
|
|
4342
|
-
}
|
|
4343
|
-
catch {
|
|
4344
|
-
// ignore
|
|
4345
|
-
}
|
|
4553
|
+
this.sendTrackerFeedbackBestEffort(ch, bootstrapPeers, c.hash, TRACKER_FEEDBACK_JOIN_TIMEOUT);
|
|
4346
4554
|
if (candidateCooldownMs > 0) {
|
|
4347
4555
|
cooldownUntilByHash.set(c.hash, Date.now() + candidateCooldownMs * 2);
|
|
4348
4556
|
}
|
|
@@ -4359,12 +4567,7 @@ export class FanoutTree extends DirectStream {
|
|
|
4359
4567
|
}
|
|
4360
4568
|
cooldownUntilByHash.set(c.hash, Date.now() + candidateCooldownMs * factor);
|
|
4361
4569
|
}
|
|
4362
|
-
|
|
4363
|
-
await this.sendTrackerFeedback(ch, bootstrapPeers, c.hash, TRACKER_FEEDBACK_JOIN_REJECT, rejectReason);
|
|
4364
|
-
}
|
|
4365
|
-
catch {
|
|
4366
|
-
// ignore
|
|
4367
|
-
}
|
|
4570
|
+
this.sendTrackerFeedbackBestEffort(ch, bootstrapPeers, c.hash, TRACKER_FEEDBACK_JOIN_REJECT, rejectReason);
|
|
4368
4571
|
}
|
|
4369
4572
|
if (ch.parent) {
|
|
4370
4573
|
// Keep only the selected parent + bootstraps. Everything else we dialed is
|
|
@@ -4383,7 +4586,16 @@ export class FanoutTree extends DirectStream {
|
|
|
4383
4586
|
}
|
|
4384
4587
|
continue;
|
|
4385
4588
|
}
|
|
4386
|
-
|
|
4589
|
+
if (bootstraps.length > 1 &&
|
|
4590
|
+
Date.now() >= bootstrapFallbackRetryAt) {
|
|
4591
|
+
for (const hash of bootstrapPeers) {
|
|
4592
|
+
unsuccessfulColdBootstrapPeers.add(hash);
|
|
4593
|
+
}
|
|
4594
|
+
}
|
|
4595
|
+
const waitMs = clampInitialJoinWait(retryMs);
|
|
4596
|
+
if (waitMs <= 0)
|
|
4597
|
+
throwInitialJoinTimeout();
|
|
4598
|
+
await delay(waitMs, { signal });
|
|
4387
4599
|
}
|
|
4388
4600
|
}
|
|
4389
4601
|
finally {
|
|
@@ -5286,23 +5498,69 @@ export class FanoutTree extends DirectStream {
|
|
|
5286
5498
|
return { ok: true };
|
|
5287
5499
|
if (!this.peers.get(parentHash))
|
|
5288
5500
|
return { ok: false, timedOut: true };
|
|
5289
|
-
const
|
|
5290
|
-
|
|
5291
|
-
|
|
5292
|
-
|
|
5293
|
-
|
|
5501
|
+
const attemptController = new AbortController();
|
|
5502
|
+
let settled = false;
|
|
5503
|
+
let resolveAttempt;
|
|
5504
|
+
let rejectAttempt;
|
|
5505
|
+
const attempt = new Promise((resolve, reject) => {
|
|
5506
|
+
resolveAttempt = resolve;
|
|
5507
|
+
rejectAttempt = reject;
|
|
5294
5508
|
});
|
|
5295
|
-
|
|
5296
|
-
|
|
5297
|
-
|
|
5298
|
-
|
|
5299
|
-
|
|
5300
|
-
|
|
5301
|
-
|
|
5302
|
-
|
|
5303
|
-
|
|
5509
|
+
const settleResult = (result) => {
|
|
5510
|
+
if (settled)
|
|
5511
|
+
return;
|
|
5512
|
+
settled = true;
|
|
5513
|
+
resolveAttempt(result);
|
|
5514
|
+
};
|
|
5515
|
+
const settleError = (error) => {
|
|
5516
|
+
if (settled)
|
|
5517
|
+
return;
|
|
5518
|
+
settled = true;
|
|
5519
|
+
rejectAttempt(error);
|
|
5520
|
+
};
|
|
5521
|
+
ch.pendingJoin.set(reqId, {
|
|
5522
|
+
resolve: settleResult,
|
|
5523
|
+
shadowAttach: options?.shadowAttach === true,
|
|
5524
|
+
});
|
|
5525
|
+
const onAbort = () => {
|
|
5526
|
+
const reason = signal.reason ?? new AbortError("fanout join aborted");
|
|
5527
|
+
if (!attemptController.signal.aborted) {
|
|
5528
|
+
attemptController.abort(reason);
|
|
5529
|
+
}
|
|
5530
|
+
settleError(reason);
|
|
5531
|
+
};
|
|
5532
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
5533
|
+
if (signal.aborted)
|
|
5534
|
+
onAbort();
|
|
5535
|
+
const timer = setTimeout(() => {
|
|
5536
|
+
if (!attemptController.signal.aborted) {
|
|
5537
|
+
attemptController.abort(new AbortError("fanout join attempt timed out"));
|
|
5538
|
+
}
|
|
5539
|
+
settleResult({ ok: false, timedOut: true });
|
|
5540
|
+
}, Math.max(1, timeoutMs));
|
|
5541
|
+
timer.unref?.();
|
|
5542
|
+
const send = this._sendControl(parentHash, this.codec.encodeJoinReq(ch.id.key, reqId, ch.bidPerByte, options?.parentUpgradeReservationToken), attemptController.signal).catch((error) => {
|
|
5543
|
+
if (settled)
|
|
5544
|
+
return;
|
|
5545
|
+
if (signal.aborted) {
|
|
5546
|
+
settleError(signal.reason ?? error);
|
|
5547
|
+
return;
|
|
5548
|
+
}
|
|
5549
|
+
if (!attemptController.signal.aborted)
|
|
5550
|
+
settleError(error);
|
|
5551
|
+
});
|
|
5552
|
+
try {
|
|
5553
|
+
return await attempt;
|
|
5554
|
+
}
|
|
5555
|
+
finally {
|
|
5556
|
+
clearTimeout(timer);
|
|
5557
|
+
signal.removeEventListener("abort", onAbort);
|
|
5304
5558
|
ch.pendingJoin.delete(reqId);
|
|
5305
|
-
|
|
5559
|
+
if (!attemptController.signal.aborted) {
|
|
5560
|
+
attemptController.abort(new AbortError("fanout join attempt settled"));
|
|
5561
|
+
}
|
|
5562
|
+
void send;
|
|
5563
|
+
}
|
|
5306
5564
|
}
|
|
5307
5565
|
async kickChildHashes(ch, children, options) {
|
|
5308
5566
|
const unique = [...new Set(children)].filter((h) => ch.children.has(h));
|