@peerbit/pubsub 5.4.9 → 5.4.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/fanout-tree.d.ts +11 -1
- package/dist/src/fanout-tree.d.ts.map +1 -1
- package/dist/src/fanout-tree.js +228 -38
- package/dist/src/fanout-tree.js.map +1 -1
- package/dist/src/index.d.ts +11 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +612 -127
- package/dist/src/index.js.map +1 -1
- package/package.json +7 -7
- package/src/fanout-tree.ts +301 -42
- package/src/index.ts +857 -177
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@peerbit/pubsub",
|
|
3
|
-
"version": "5.4.
|
|
3
|
+
"version": "5.4.11",
|
|
4
4
|
"description": "Direct streaming for libp2p",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"types": "./dist/src/index.d.ts",
|
|
@@ -67,7 +67,7 @@
|
|
|
67
67
|
},
|
|
68
68
|
"homepage": "https://github.com/dao-xyz/peerbit#readme",
|
|
69
69
|
"devDependencies": {
|
|
70
|
-
"@peerbit/libp2p-test-utils": "3.0.
|
|
70
|
+
"@peerbit/libp2p-test-utils": "3.0.11"
|
|
71
71
|
},
|
|
72
72
|
"dependencies": {
|
|
73
73
|
"@dao-xyz/borsh": "^6.0.0",
|
|
@@ -86,12 +86,12 @@
|
|
|
86
86
|
"uint8arraylist": "^3.0.2",
|
|
87
87
|
"uint8arrays": "^5.1.0",
|
|
88
88
|
"@peerbit/any-store-interface": "1.1.4",
|
|
89
|
-
"@peerbit/logger": "2.0.2",
|
|
90
|
-
"@peerbit/stream": "5.2.3",
|
|
91
|
-
"@peerbit/pubsub-interface": "5.2.2",
|
|
92
89
|
"@peerbit/crypto": "3.1.6",
|
|
93
|
-
"@peerbit/
|
|
94
|
-
"@peerbit/
|
|
90
|
+
"@peerbit/pubsub-interface": "5.2.2",
|
|
91
|
+
"@peerbit/logger": "2.0.2",
|
|
92
|
+
"@peerbit/stream": "5.2.4",
|
|
93
|
+
"@peerbit/stream-interface": "6.0.16",
|
|
94
|
+
"@peerbit/time": "3.0.1"
|
|
95
95
|
},
|
|
96
96
|
"scripts": {
|
|
97
97
|
"bench": "TS_NODE_PROJECT=../../../tsconfig.json node --loader ts-node/esm ./benchmark/index.ts",
|
package/src/fanout-tree.ts
CHANGED
|
@@ -512,7 +512,7 @@ export type FanoutTreeJoinOptions = {
|
|
|
512
512
|
parentUpgradeStaleRootProbeProbability?: number;
|
|
513
513
|
|
|
514
514
|
/**
|
|
515
|
-
* How long to wait for a parent probe reply
|
|
515
|
+
* How long to wait for a candidate or idle-parent probe reply.
|
|
516
516
|
*/
|
|
517
517
|
parentProbeTimeoutMs?: number;
|
|
518
518
|
|
|
@@ -766,6 +766,8 @@ export type FanoutTreeChannelMetrics = {
|
|
|
766
766
|
reparentUpgradeSkipProbeOverloaded: number;
|
|
767
767
|
reparentUpgradeSkipProbeCooldown: number;
|
|
768
768
|
parentProbeReqSent: number;
|
|
769
|
+
/** Subset of parentProbeReqSent used to check the current idle parent. */
|
|
770
|
+
parentHealthProbeReqSent: number;
|
|
769
771
|
parentProbeReqReceived: number;
|
|
770
772
|
parentProbeReplySent: number;
|
|
771
773
|
parentProbeReplyReceived: number;
|
|
@@ -836,6 +838,11 @@ export interface FanoutTreeEvents extends StreamEvents {
|
|
|
836
838
|
"fanout:data": CustomEvent<FanoutTreeDataEvent>;
|
|
837
839
|
"fanout:unicast": CustomEvent<FanoutTreeUnicastEvent>;
|
|
838
840
|
"fanout:joined": CustomEvent<{ topic: string; root: string; parent: string }>;
|
|
841
|
+
"fanout:child-joined": CustomEvent<{
|
|
842
|
+
topic: string;
|
|
843
|
+
root: string;
|
|
844
|
+
child: string;
|
|
845
|
+
}>;
|
|
839
846
|
"fanout:kicked": CustomEvent<{ topic: string; root: string; from: string }>;
|
|
840
847
|
"fanout:provider": CustomEvent<{
|
|
841
848
|
namespace: string;
|
|
@@ -870,6 +877,19 @@ const PROVIDER_DIRECTORY_MAX_NAMESPACES = 4_096;
|
|
|
870
877
|
// large networks can cause unbounded memory growth on the tracker nodes.
|
|
871
878
|
const TRACKER_DIRECTORY_MAX_ENTRIES = 16_384;
|
|
872
879
|
const TRACKER_DIRECTORY_MAX_NAMESPACES = 4_096;
|
|
880
|
+
// Best-effort bound on per-key control metrics for keys without an open channel.
|
|
881
|
+
// Every inbound control frame is recorded under its key, including tracker frames
|
|
882
|
+
// for channels we do not host, so bootstraps/relays see an open-ended key set.
|
|
883
|
+
// Open channels keep their metrics on the channel and are never evicted.
|
|
884
|
+
const DETACHED_METRICS_MAX_KEYS = 4_096;
|
|
885
|
+
// Best-effort bounds for provider watches and namespace names. Provider namespaces
|
|
886
|
+
// are commonly per block (`cid:<cid>`), so anything keyed by them must be bounded.
|
|
887
|
+
const PROVIDER_WATCH_MAX_NAMESPACES = 4_096;
|
|
888
|
+
const PROVIDER_NAMESPACE_NAME_MAX_ENTRIES = 16_384;
|
|
889
|
+
// Per-channel ingress token buckets (proxy-publish/unicast) are kept for recent
|
|
890
|
+
// senders only: a few multiples of the child limit, with a small floor.
|
|
891
|
+
const INGRESS_BUCKETS_MIN_ENTRIES = 64;
|
|
892
|
+
const INGRESS_BUCKETS_PER_CHILD = 4;
|
|
873
893
|
|
|
874
894
|
const FANOUT_PROTOCOLS = ["/peerbit/fanout-tree/0.5.0"];
|
|
875
895
|
|
|
@@ -886,6 +906,7 @@ const DATA_WRITE_FAIL_KICK_COOLDOWN_MS = 2_000;
|
|
|
886
906
|
const DATA_WRITE_FAIL_KICK_MAX_PER_EVENT = 4;
|
|
887
907
|
const PARENT_REPAIR_DEAD_STREAK_THRESHOLD = 16;
|
|
888
908
|
const PARENT_REPAIR_DEAD_MIN_LIVENESS_MS = 15_000;
|
|
909
|
+
const PARENT_IDLE_PROBE_INTERVAL_MS = 5_000;
|
|
889
910
|
const REPAIR_RETRY_MIN_MS = 1_000;
|
|
890
911
|
const REPAIR_RETRY_INTERVAL_FACTOR = 5;
|
|
891
912
|
// A stream can remain locally readable/writable while silently dropping every
|
|
@@ -1162,7 +1183,10 @@ type ChannelState = {
|
|
|
1162
1183
|
number,
|
|
1163
1184
|
{ resolve(entries: TrackerCandidate[]): void }
|
|
1164
1185
|
>;
|
|
1165
|
-
pendingParentProbe: Map<
|
|
1186
|
+
pendingParentProbe: Map<
|
|
1187
|
+
number,
|
|
1188
|
+
{ peer: PeerStreams; resolve(reply?: ParentProbeReply): void }
|
|
1189
|
+
>;
|
|
1166
1190
|
parentUpgradeReservationsByHash: Map<
|
|
1167
1191
|
string,
|
|
1168
1192
|
{ token: number; expiresAt: number; minFreeSlots: number }
|
|
@@ -1300,6 +1324,7 @@ const createEmptyMetrics = (): InternalFanoutTreeChannelMetrics => ({
|
|
|
1300
1324
|
reparentUpgradeSkipProbeOverloaded: 0,
|
|
1301
1325
|
reparentUpgradeSkipProbeCooldown: 0,
|
|
1302
1326
|
parentProbeReqSent: 0,
|
|
1327
|
+
parentHealthProbeReqSent: 0,
|
|
1303
1328
|
parentProbeReqReceived: 0,
|
|
1304
1329
|
parentProbeReplySent: 0,
|
|
1305
1330
|
parentProbeReplyReceived: 0,
|
|
@@ -1358,13 +1383,27 @@ const isDataId = (id: Uint8Array) =>
|
|
|
1358
1383
|
id[2] === ID_PREFIX[2] &&
|
|
1359
1384
|
id[3] === ID_PREFIX[3];
|
|
1360
1385
|
|
|
1386
|
+
const isProviderControlKind = (kind: number) =>
|
|
1387
|
+
kind === MSG_PROVIDER_ANNOUNCE ||
|
|
1388
|
+
kind === MSG_PROVIDER_QUERY ||
|
|
1389
|
+
kind === MSG_PROVIDER_REPLY ||
|
|
1390
|
+
kind === MSG_PROVIDER_SUBSCRIBE ||
|
|
1391
|
+
kind === MSG_PROVIDER_UNSUBSCRIBE ||
|
|
1392
|
+
kind === MSG_PROVIDER_NOTIFY;
|
|
1393
|
+
|
|
1361
1394
|
export class FanoutTree extends DirectStream<FanoutTreeEvents> {
|
|
1362
1395
|
private channelsBySuffixKey = new Map<string, ChannelState>();
|
|
1363
1396
|
private readonly cachedSuffixKey = new WeakMap<Uint8Array, string>();
|
|
1397
|
+
// Metrics for keys without an open channel, in LRU order (oldest first) and
|
|
1398
|
+
// bounded by DETACHED_METRICS_MAX_KEYS. Open channels own `ch.metrics`.
|
|
1364
1399
|
private readonly metricsBySuffixKey = new Map<
|
|
1365
1400
|
string,
|
|
1366
1401
|
InternalFanoutTreeChannelMetrics
|
|
1367
1402
|
>();
|
|
1403
|
+
// Provider control frames are keyed per provider namespace (often per block),
|
|
1404
|
+
// which no channel metrics lookup can address, so they share one aggregate.
|
|
1405
|
+
private readonly providerControlMetrics: InternalFanoutTreeChannelMetrics =
|
|
1406
|
+
createEmptyMetrics();
|
|
1368
1407
|
private readonly joinTimeoutStreakByPeer = new Map<string, number>();
|
|
1369
1408
|
private readonly joinResetCooldownUntilByPeer = new Map<string, number>();
|
|
1370
1409
|
private bootstraps: Multiaddr[] = [];
|
|
@@ -1551,6 +1590,27 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
|
|
|
1551
1590
|
}
|
|
1552
1591
|
}
|
|
1553
1592
|
|
|
1593
|
+
private touchProviderWatchers(
|
|
1594
|
+
suffixKey: string,
|
|
1595
|
+
): Map<string, ProviderWatchRegistration> {
|
|
1596
|
+
// LRU touch
|
|
1597
|
+
const watchers =
|
|
1598
|
+
this.providerWatchersBySuffixKey.get(suffixKey) ??
|
|
1599
|
+
new Map<string, ProviderWatchRegistration>();
|
|
1600
|
+
this.providerWatchersBySuffixKey.delete(suffixKey);
|
|
1601
|
+
this.providerWatchersBySuffixKey.set(suffixKey, watchers);
|
|
1602
|
+
while (
|
|
1603
|
+
this.providerWatchersBySuffixKey.size > PROVIDER_WATCH_MAX_NAMESPACES
|
|
1604
|
+
) {
|
|
1605
|
+
const oldest = this.providerWatchersBySuffixKey.keys().next().value as
|
|
1606
|
+
| string
|
|
1607
|
+
| undefined;
|
|
1608
|
+
if (oldest === undefined) break;
|
|
1609
|
+
this.providerWatchersBySuffixKey.delete(oldest);
|
|
1610
|
+
}
|
|
1611
|
+
return watchers;
|
|
1612
|
+
}
|
|
1613
|
+
|
|
1554
1614
|
private pruneProviderWatchersIfEmpty(suffixKey: string) {
|
|
1555
1615
|
const watchers = this.providerWatchersBySuffixKey.get(suffixKey);
|
|
1556
1616
|
if (watchers && watchers.size === 0) {
|
|
@@ -1686,7 +1746,19 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
|
|
|
1686
1746
|
private getProviderNamespaceId(namespace: string): ProviderNamespaceId {
|
|
1687
1747
|
const key = sha256Sync(textEncoder.encode(`provider|${namespace}`));
|
|
1688
1748
|
const suffixKey = toBase64(key.subarray(0, 24));
|
|
1749
|
+
// LRU touch: every announced block adds a namespace, so keep this bounded.
|
|
1750
|
+
this.providerNamespaceBySuffixKey.delete(suffixKey);
|
|
1689
1751
|
this.providerNamespaceBySuffixKey.set(suffixKey, namespace);
|
|
1752
|
+
while (
|
|
1753
|
+
this.providerNamespaceBySuffixKey.size >
|
|
1754
|
+
PROVIDER_NAMESPACE_NAME_MAX_ENTRIES
|
|
1755
|
+
) {
|
|
1756
|
+
const oldest = this.providerNamespaceBySuffixKey.keys().next().value as
|
|
1757
|
+
| string
|
|
1758
|
+
| undefined;
|
|
1759
|
+
if (oldest === undefined) break;
|
|
1760
|
+
this.providerNamespaceBySuffixKey.delete(oldest);
|
|
1761
|
+
}
|
|
1690
1762
|
return { namespace, key, suffixKey };
|
|
1691
1763
|
}
|
|
1692
1764
|
|
|
@@ -2505,6 +2577,9 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
|
|
|
2505
2577
|
lastTrackerQueryAt: 0,
|
|
2506
2578
|
};
|
|
2507
2579
|
this.channelsBySuffixKey.set(id.suffixKey, ch);
|
|
2580
|
+
// The open channel now owns its metrics; take them out of the detached LRU
|
|
2581
|
+
// so they can never be evicted while the channel is open.
|
|
2582
|
+
this.metricsBySuffixKey.delete(id.suffixKey);
|
|
2508
2583
|
const needsAnnounceLoop = ch.effectiveMaxChildren > 0;
|
|
2509
2584
|
if (needsAnnounceLoop) {
|
|
2510
2585
|
ch.announceLoop = this._announceLoop(ch).catch(() => {});
|
|
@@ -2705,6 +2780,13 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
|
|
|
2705
2780
|
if (!peerHash) return;
|
|
2706
2781
|
this.joinTimeoutStreakByPeer.delete(peerHash);
|
|
2707
2782
|
|
|
2783
|
+
// A disconnected watcher can no longer receive provider notifications.
|
|
2784
|
+
for (const [suffixKey, watchers] of this.providerWatchersBySuffixKey) {
|
|
2785
|
+
if (watchers.delete(peerHash) && watchers.size === 0) {
|
|
2786
|
+
this.providerWatchersBySuffixKey.delete(suffixKey);
|
|
2787
|
+
}
|
|
2788
|
+
}
|
|
2789
|
+
|
|
2708
2790
|
// Detach from a disconnected parent immediately, so children can rejoin.
|
|
2709
2791
|
// This is more reliable than polling `getConnections()` because the underlay
|
|
2710
2792
|
// can flap/reconnect faster than the join loop cadence.
|
|
@@ -2803,6 +2885,8 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
|
|
|
2803
2885
|
|
|
2804
2886
|
this.channelsBySuffixKey.delete(id.suffixKey);
|
|
2805
2887
|
this.trackerBySuffixKey.delete(id.suffixKey);
|
|
2888
|
+
// Keep the counters readable after close (and across a reopen), bounded.
|
|
2889
|
+
this.storeDetachedMetrics(id.suffixKey, ch.metrics);
|
|
2806
2890
|
|
|
2807
2891
|
await Promise.all(pendingSends);
|
|
2808
2892
|
}
|
|
@@ -3985,20 +4069,57 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
|
|
|
3985
4069
|
private getMetricsForSuffixKey(
|
|
3986
4070
|
suffixKey: string,
|
|
3987
4071
|
): InternalFanoutTreeChannelMetrics {
|
|
3988
|
-
|
|
3989
|
-
if (
|
|
3990
|
-
|
|
3991
|
-
|
|
3992
|
-
}
|
|
4072
|
+
const ch = this.channelsBySuffixKey.get(suffixKey);
|
|
4073
|
+
if (ch) return ch.metrics;
|
|
4074
|
+
const m = this.metricsBySuffixKey.get(suffixKey) ?? createEmptyMetrics();
|
|
4075
|
+
this.storeDetachedMetrics(suffixKey, m);
|
|
3993
4076
|
return m;
|
|
3994
4077
|
}
|
|
3995
4078
|
|
|
3996
|
-
private
|
|
4079
|
+
private storeDetachedMetrics(
|
|
4080
|
+
suffixKey: string,
|
|
4081
|
+
metrics: InternalFanoutTreeChannelMetrics,
|
|
4082
|
+
) {
|
|
4083
|
+
// LRU touch
|
|
4084
|
+
this.metricsBySuffixKey.delete(suffixKey);
|
|
4085
|
+
this.metricsBySuffixKey.set(suffixKey, metrics);
|
|
4086
|
+
while (this.metricsBySuffixKey.size > DETACHED_METRICS_MAX_KEYS) {
|
|
4087
|
+
const oldest = this.metricsBySuffixKey.keys().next().value as
|
|
4088
|
+
| string
|
|
4089
|
+
| undefined;
|
|
4090
|
+
if (oldest === undefined) break;
|
|
4091
|
+
this.metricsBySuffixKey.delete(oldest);
|
|
4092
|
+
}
|
|
4093
|
+
}
|
|
4094
|
+
|
|
4095
|
+
private getControlMetrics(
|
|
4096
|
+
suffixKey: string,
|
|
4097
|
+
kind: number,
|
|
4098
|
+
): InternalFanoutTreeChannelMetrics {
|
|
4099
|
+
return isProviderControlKind(kind)
|
|
4100
|
+
? this.providerControlMetrics
|
|
4101
|
+
: this.getMetricsForSuffixKey(suffixKey);
|
|
4102
|
+
}
|
|
4103
|
+
|
|
4104
|
+
/**
|
|
4105
|
+
* Returns aggregate diagnostic counters for provider control frames.
|
|
4106
|
+
*
|
|
4107
|
+
* @internal
|
|
4108
|
+
*/
|
|
4109
|
+
public getProviderControlMetrics(): FanoutTreeChannelMetrics {
|
|
4110
|
+
return this.providerControlMetrics;
|
|
4111
|
+
}
|
|
4112
|
+
|
|
4113
|
+
private recordControlSend(
|
|
4114
|
+
bytes: Uint8Array,
|
|
4115
|
+
transmissions: number,
|
|
4116
|
+
probePurpose: "upgrade" | "health" = "upgrade",
|
|
4117
|
+
) {
|
|
3997
4118
|
if (transmissions <= 0) return;
|
|
3998
4119
|
if (bytes.length < 1 + 24) return;
|
|
3999
4120
|
const kind = bytes[0]!;
|
|
4000
4121
|
const suffixKey = toBase64(bytes.subarray(1, 25));
|
|
4001
|
-
const m = this.
|
|
4122
|
+
const m = this.getControlMetrics(suffixKey, kind);
|
|
4002
4123
|
m.controlSends += transmissions;
|
|
4003
4124
|
const sentBytes = bytes.byteLength * transmissions;
|
|
4004
4125
|
m.controlBytesSent += sentBytes;
|
|
@@ -4053,6 +4174,9 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
|
|
|
4053
4174
|
break;
|
|
4054
4175
|
case MSG_PARENT_PROBE_REQ:
|
|
4055
4176
|
m.parentProbeReqSent += transmissions;
|
|
4177
|
+
if (probePurpose === "health") {
|
|
4178
|
+
m.parentHealthProbeReqSent += transmissions;
|
|
4179
|
+
}
|
|
4056
4180
|
m.controlBytesSentJoin += sentBytes;
|
|
4057
4181
|
break;
|
|
4058
4182
|
case MSG_PARENT_PROBE_REPLY:
|
|
@@ -4077,7 +4201,7 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
|
|
|
4077
4201
|
kind: number,
|
|
4078
4202
|
bytesReceived: number,
|
|
4079
4203
|
) {
|
|
4080
|
-
const m = this.
|
|
4204
|
+
const m = this.getControlMetrics(suffixKey, kind);
|
|
4081
4205
|
m.controlReceives += 1;
|
|
4082
4206
|
m.controlBytesReceived += bytesReceived;
|
|
4083
4207
|
switch (kind) {
|
|
@@ -4205,13 +4329,14 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
|
|
|
4205
4329
|
to: string,
|
|
4206
4330
|
bytes: Uint8Array,
|
|
4207
4331
|
signal?: AbortSignal,
|
|
4332
|
+
probePurpose: "upgrade" | "health" = "upgrade",
|
|
4208
4333
|
) {
|
|
4209
4334
|
if (signal?.aborted) {
|
|
4210
4335
|
throw signal.reason ?? new AbortError("fanout control send aborted");
|
|
4211
4336
|
}
|
|
4212
4337
|
const stream = this.peers.get(to);
|
|
4213
4338
|
if (!stream) return;
|
|
4214
|
-
this.recordControlSend(bytes, 1);
|
|
4339
|
+
this.recordControlSend(bytes, 1, probePurpose);
|
|
4215
4340
|
const message = await this.createMessage(bytes, {
|
|
4216
4341
|
mode: new AnyWhere(),
|
|
4217
4342
|
priority: CONTROL_PRIORITY,
|
|
@@ -4298,17 +4423,35 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
|
|
|
4298
4423
|
tokens = Math.min(capacity, tokens + (elapsedMs * budgetBps) / 1_000);
|
|
4299
4424
|
}
|
|
4300
4425
|
if (tokens < costBytes) {
|
|
4301
|
-
|
|
4302
|
-
byPeer.delete(fromHash);
|
|
4303
|
-
byPeer.set(fromHash, { tokens, lastRefillAt: now });
|
|
4426
|
+
this.setIngressBucket(ch, byPeer, fromHash, { tokens, lastRefillAt: now });
|
|
4304
4427
|
return false;
|
|
4305
4428
|
}
|
|
4306
4429
|
tokens -= costBytes;
|
|
4307
|
-
|
|
4308
|
-
byPeer.set(fromHash, { tokens, lastRefillAt: now });
|
|
4430
|
+
this.setIngressBucket(ch, byPeer, fromHash, { tokens, lastRefillAt: now });
|
|
4309
4431
|
return true;
|
|
4310
4432
|
}
|
|
4311
4433
|
|
|
4434
|
+
private setIngressBucket(
|
|
4435
|
+
ch: ChannelState,
|
|
4436
|
+
byPeer: Map<string, { tokens: number; lastRefillAt: number }>,
|
|
4437
|
+
fromHash: string,
|
|
4438
|
+
bucket: { tokens: number; lastRefillAt: number },
|
|
4439
|
+
) {
|
|
4440
|
+
// LRU touch, capped: long-lived (pinned) roots see an open-ended set of
|
|
4441
|
+
// children over time, and buckets are not dropped when a child leaves.
|
|
4442
|
+
byPeer.delete(fromHash);
|
|
4443
|
+
byPeer.set(fromHash, bucket);
|
|
4444
|
+
const maxBuckets = Math.max(
|
|
4445
|
+
INGRESS_BUCKETS_MIN_ENTRIES,
|
|
4446
|
+
ch.maxChildren * INGRESS_BUCKETS_PER_CHILD,
|
|
4447
|
+
);
|
|
4448
|
+
while (byPeer.size > maxBuckets) {
|
|
4449
|
+
const oldest = byPeer.keys().next().value as string | undefined;
|
|
4450
|
+
if (oldest === undefined) break;
|
|
4451
|
+
byPeer.delete(oldest);
|
|
4452
|
+
}
|
|
4453
|
+
}
|
|
4454
|
+
|
|
4312
4455
|
private async _sendData(
|
|
4313
4456
|
ch: ChannelState,
|
|
4314
4457
|
to: string[],
|
|
@@ -5894,24 +6037,56 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
|
|
|
5894
6037
|
signal: AbortSignal,
|
|
5895
6038
|
minFreeSlots = 0,
|
|
5896
6039
|
reserveRootCapacity = true,
|
|
6040
|
+
purpose: "upgrade" | "health" = "upgrade",
|
|
5897
6041
|
): Promise<ParentProbeReply | undefined> {
|
|
5898
|
-
|
|
5899
|
-
|
|
5900
|
-
const
|
|
5901
|
-
|
|
5902
|
-
|
|
5903
|
-
|
|
5904
|
-
|
|
5905
|
-
|
|
5906
|
-
);
|
|
5907
|
-
|
|
5908
|
-
|
|
5909
|
-
|
|
5910
|
-
|
|
5911
|
-
|
|
5912
|
-
|
|
5913
|
-
|
|
5914
|
-
|
|
6042
|
+
const peer = this.peers.get(parentHash);
|
|
6043
|
+
if (!peer) return;
|
|
6044
|
+
const controller = new AbortController();
|
|
6045
|
+
let reqId = (this.random() * 0xffffffff) >>> 0;
|
|
6046
|
+
while (ch.pendingParentProbe.has(reqId)) reqId = (reqId + 1) >>> 0;
|
|
6047
|
+
let pending:
|
|
6048
|
+
| { peer: PeerStreams; resolve(reply?: ParentProbeReply): void }
|
|
6049
|
+
| undefined;
|
|
6050
|
+
let onAbort!: () => void;
|
|
6051
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
6052
|
+
try {
|
|
6053
|
+
return await new Promise<ParentProbeReply | undefined>((resolve, reject) => {
|
|
6054
|
+
pending = { peer, resolve };
|
|
6055
|
+
ch.pendingParentProbe.set(reqId, pending);
|
|
6056
|
+
onAbort = () => {
|
|
6057
|
+
const reason = signal.reason ?? new AbortError("fanout probe aborted");
|
|
6058
|
+
controller.abort(reason);
|
|
6059
|
+
reject(reason);
|
|
6060
|
+
};
|
|
6061
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
6062
|
+
if (signal.aborted) return onAbort();
|
|
6063
|
+
// Bound signing/queueing as well as the reply; a stuck send must not
|
|
6064
|
+
// prevent the join loop from recovering or closing the channel.
|
|
6065
|
+
timer = setTimeout(() => {
|
|
6066
|
+
controller.abort(new AbortError("fanout probe timed out"));
|
|
6067
|
+
resolve(undefined);
|
|
6068
|
+
}, Math.max(1, timeoutMs));
|
|
6069
|
+
timer.unref?.();
|
|
6070
|
+
void this._sendControl(
|
|
6071
|
+
parentHash,
|
|
6072
|
+
this.codec.encodeParentProbeReq(
|
|
6073
|
+
ch.id.key,
|
|
6074
|
+
reqId,
|
|
6075
|
+
minFreeSlots,
|
|
6076
|
+
reserveRootCapacity,
|
|
6077
|
+
),
|
|
6078
|
+
controller.signal,
|
|
6079
|
+
purpose,
|
|
6080
|
+
).catch(() => resolve(undefined));
|
|
6081
|
+
});
|
|
6082
|
+
} finally {
|
|
6083
|
+
clearTimeout(timer);
|
|
6084
|
+
signal.removeEventListener("abort", onAbort);
|
|
6085
|
+
if (ch.pendingParentProbe.get(reqId) === pending) {
|
|
6086
|
+
ch.pendingParentProbe.delete(reqId);
|
|
6087
|
+
}
|
|
6088
|
+
controller.abort(new AbortError("fanout probe settled"));
|
|
6089
|
+
}
|
|
5915
6090
|
}
|
|
5916
6091
|
|
|
5917
6092
|
private async _joinLoop(
|
|
@@ -6020,6 +6195,10 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
|
|
|
6020
6195
|
if (initialJoinRemainingMs() === 0) throwInitialJoinTimeout();
|
|
6021
6196
|
};
|
|
6022
6197
|
let nextParentUpgradeCheckAt = 0;
|
|
6198
|
+
let nextParentHealthProbeAt = 0;
|
|
6199
|
+
let failedParentProbe:
|
|
6200
|
+
| { peer: PeerStreams; route: ChannelState["routeFromRoot"] }
|
|
6201
|
+
| undefined;
|
|
6023
6202
|
let parentUpgradeCheckSeq = 0;
|
|
6024
6203
|
let parentUpgradeActiveGuardBackoffMs = 0;
|
|
6025
6204
|
const unsuccessfulColdBootstrapPeers = new Set<string>();
|
|
@@ -6110,8 +6289,10 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
|
|
|
6110
6289
|
|
|
6111
6290
|
// Parent disappeared? Rejoin.
|
|
6112
6291
|
if (ch.parent) {
|
|
6113
|
-
const
|
|
6292
|
+
const parentHash = ch.parent;
|
|
6293
|
+
const parentPeer = this.peers.get(parentHash);
|
|
6114
6294
|
let connected = false;
|
|
6295
|
+
let probeFailed = false;
|
|
6115
6296
|
if (parentPeer) {
|
|
6116
6297
|
try {
|
|
6117
6298
|
const conns = this.components.connectionManager.getConnections(
|
|
@@ -6122,8 +6303,61 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
|
|
|
6122
6303
|
connected = parentPeer.isReadable || parentPeer.isWritable;
|
|
6123
6304
|
}
|
|
6124
6305
|
}
|
|
6306
|
+
const now = Date.now();
|
|
6307
|
+
if (now - ch.lastParentDataAt < PARENT_IDLE_PROBE_INTERVAL_MS) {
|
|
6308
|
+
failedParentProbe = undefined;
|
|
6309
|
+
} else if (
|
|
6310
|
+
connected &&
|
|
6311
|
+
parentPeer &&
|
|
6312
|
+
now >= nextParentHealthProbeAt
|
|
6313
|
+
) {
|
|
6314
|
+
const route = ch.routeFromRoot;
|
|
6315
|
+
const lastDataAt = ch.lastParentDataAt;
|
|
6316
|
+
const reply = await this.probeParentCandidate(
|
|
6317
|
+
ch,
|
|
6318
|
+
parentHash,
|
|
6319
|
+
parentUpgrade.probe.timeoutMs,
|
|
6320
|
+
signal,
|
|
6321
|
+
0,
|
|
6322
|
+
false,
|
|
6323
|
+
"health",
|
|
6324
|
+
);
|
|
6325
|
+
if (
|
|
6326
|
+
ch.closed ||
|
|
6327
|
+
signal.aborted ||
|
|
6328
|
+
this.channelsBySuffixKey.get(ch.id.suffixKey) !== ch ||
|
|
6329
|
+
ch.parent !== parentHash ||
|
|
6330
|
+
ch.routeFromRoot !== route ||
|
|
6331
|
+
this.peers.get(parentHash) !== parentPeer
|
|
6332
|
+
) {
|
|
6333
|
+
failedParentProbe = undefined;
|
|
6334
|
+
continue;
|
|
6335
|
+
}
|
|
6336
|
+
nextParentHealthProbeAt = Date.now() + PARENT_IDLE_PROBE_INTERVAL_MS;
|
|
6337
|
+
// A full parent is still healthy. Do not reserve capacity or
|
|
6338
|
+
// refresh lastParentDataAt: probes must not mask stalled data.
|
|
6339
|
+
if (reply || ch.lastParentDataAt !== lastDataAt) {
|
|
6340
|
+
failedParentProbe = undefined;
|
|
6341
|
+
} else if (
|
|
6342
|
+
failedParentProbe?.peer === parentPeer &&
|
|
6343
|
+
failedParentProbe.route === route
|
|
6344
|
+
) {
|
|
6345
|
+
probeFailed = true;
|
|
6346
|
+
connected = false;
|
|
6347
|
+
cooldownUntilByHash.set(
|
|
6348
|
+
parentHash,
|
|
6349
|
+
Date.now() + candidateCooldownMs,
|
|
6350
|
+
);
|
|
6351
|
+
} else {
|
|
6352
|
+
failedParentProbe = { peer: parentPeer, route };
|
|
6353
|
+
nextParentHealthProbeAt = Date.now() + Math.max(retryMs, 1_000);
|
|
6354
|
+
}
|
|
6355
|
+
}
|
|
6125
6356
|
if (!connected) {
|
|
6126
|
-
ch.metrics.
|
|
6357
|
+
if (probeFailed) ch.metrics.reparentStale += 1;
|
|
6358
|
+
else ch.metrics.reparentDisconnect += 1;
|
|
6359
|
+
failedParentProbe = undefined;
|
|
6360
|
+
nextParentHealthProbeAt = 0;
|
|
6127
6361
|
const hadChildren = ch.children.size > 0;
|
|
6128
6362
|
this.detachFromParent(ch);
|
|
6129
6363
|
nextParentUpgradeCheckAt = 0;
|
|
@@ -8424,11 +8658,7 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
|
|
|
8424
8658
|
const want = Math.max(1, decoded.want);
|
|
8425
8659
|
const ttlMs = Math.min(120_000, Math.max(1_000, decoded.ttlMs));
|
|
8426
8660
|
const now = Date.now();
|
|
8427
|
-
|
|
8428
|
-
if (!watchers) {
|
|
8429
|
-
watchers = new Map<string, ProviderWatchRegistration>();
|
|
8430
|
-
this.providerWatchersBySuffixKey.set(suffixKey, watchers);
|
|
8431
|
-
}
|
|
8661
|
+
const watchers = this.touchProviderWatchers(suffixKey);
|
|
8432
8662
|
watchers.set(fromHash, {
|
|
8433
8663
|
hash: fromHash,
|
|
8434
8664
|
want,
|
|
@@ -8566,7 +8796,14 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
|
|
|
8566
8796
|
if (!decoded) return false;
|
|
8567
8797
|
const { reqId, ...reply } = decoded;
|
|
8568
8798
|
const pending = ch.pendingParentProbe.get(reqId);
|
|
8569
|
-
if (
|
|
8799
|
+
if (
|
|
8800
|
+
!pending ||
|
|
8801
|
+
pending.peer !== peerStream ||
|
|
8802
|
+
pending.peer.publicKey.hashcode() !== fromHash ||
|
|
8803
|
+
this.peers.get(fromHash) !== peerStream
|
|
8804
|
+
) {
|
|
8805
|
+
return true;
|
|
8806
|
+
}
|
|
8570
8807
|
ch.pendingParentProbe.delete(reqId);
|
|
8571
8808
|
pending.resolve(reply);
|
|
8572
8809
|
this.touchPeerHint(ch, fromHash);
|
|
@@ -8851,7 +9088,8 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
|
|
|
8851
9088
|
void this.kickChildHashes(ch, [worstChild]).catch(() => {});
|
|
8852
9089
|
}
|
|
8853
9090
|
|
|
8854
|
-
|
|
9091
|
+
const child = { bidPerByte };
|
|
9092
|
+
ch.children.set(fromHash, child);
|
|
8855
9093
|
if (ch.isRoot && consumedParentUpgradeReservation) {
|
|
8856
9094
|
ch.parentUpgradeTrackerNoCapacityUntil = Math.max(
|
|
8857
9095
|
ch.parentUpgradeTrackerNoCapacityUntil,
|
|
@@ -8868,6 +9106,12 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
|
|
|
8868
9106
|
);
|
|
8869
9107
|
void (async () => {
|
|
8870
9108
|
await this._sendControl(fromHash, joinAccept);
|
|
9109
|
+
if (ch.closed || ch.children.get(fromHash) !== child) return;
|
|
9110
|
+
this.dispatchEvent(
|
|
9111
|
+
new CustomEvent("fanout:child-joined", {
|
|
9112
|
+
detail: { topic: ch.id.topic, root: ch.id.root, child: fromHash },
|
|
9113
|
+
}),
|
|
9114
|
+
);
|
|
8871
9115
|
if (ch.endSeqExclusive > 0) {
|
|
8872
9116
|
await this._sendControl(
|
|
8873
9117
|
fromHash,
|
|
@@ -9311,6 +9555,21 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
|
|
|
9311
9555
|
}
|
|
9312
9556
|
|
|
9313
9557
|
if (kind === MSG_KICK) {
|
|
9558
|
+
if (this.peers.get(fromHash) !== peerStream) return true;
|
|
9559
|
+
if (fromHash !== ch.parent) {
|
|
9560
|
+
if (ch.parentUpgradeGrace?.previousParent === fromHash) {
|
|
9561
|
+
// This attachment was evicted; never roll back to it.
|
|
9562
|
+
this.clearParentUpgradeGrace(ch);
|
|
9563
|
+
}
|
|
9564
|
+
if (ch.parentShadow?.hash === fromHash) {
|
|
9565
|
+
ch.parentShadow = undefined;
|
|
9566
|
+
ch.metrics.parentShadowReset += 1;
|
|
9567
|
+
if (this.parentUpgradeShadowInFlightSuffixKey === ch.id.suffixKey) {
|
|
9568
|
+
this.parentUpgradeShadowInFlightSuffixKey = undefined;
|
|
9569
|
+
}
|
|
9570
|
+
}
|
|
9571
|
+
return true;
|
|
9572
|
+
}
|
|
9314
9573
|
ch.metrics.reparentKicked += 1;
|
|
9315
9574
|
ch.parent = undefined;
|
|
9316
9575
|
ch.level = Number.POSITIVE_INFINITY;
|