@peerbit/pubsub 5.4.9 → 5.4.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@peerbit/pubsub",
3
- "version": "5.4.9",
3
+ "version": "5.4.11",
4
4
  "description": "Direct streaming for libp2p",
5
5
  "type": "module",
6
6
  "types": "./dist/src/index.d.ts",
@@ -67,7 +67,7 @@
67
67
  },
68
68
  "homepage": "https://github.com/dao-xyz/peerbit#readme",
69
69
  "devDependencies": {
70
- "@peerbit/libp2p-test-utils": "3.0.10"
70
+ "@peerbit/libp2p-test-utils": "3.0.11"
71
71
  },
72
72
  "dependencies": {
73
73
  "@dao-xyz/borsh": "^6.0.0",
@@ -86,12 +86,12 @@
86
86
  "uint8arraylist": "^3.0.2",
87
87
  "uint8arrays": "^5.1.0",
88
88
  "@peerbit/any-store-interface": "1.1.4",
89
- "@peerbit/logger": "2.0.2",
90
- "@peerbit/stream": "5.2.3",
91
- "@peerbit/pubsub-interface": "5.2.2",
92
89
  "@peerbit/crypto": "3.1.6",
93
- "@peerbit/time": "3.0.1",
94
- "@peerbit/stream-interface": "6.0.16"
90
+ "@peerbit/pubsub-interface": "5.2.2",
91
+ "@peerbit/logger": "2.0.2",
92
+ "@peerbit/stream": "5.2.4",
93
+ "@peerbit/stream-interface": "6.0.16",
94
+ "@peerbit/time": "3.0.1"
95
95
  },
96
96
  "scripts": {
97
97
  "bench": "TS_NODE_PROJECT=../../../tsconfig.json node --loader ts-node/esm ./benchmark/index.ts",
@@ -512,7 +512,7 @@ export type FanoutTreeJoinOptions = {
512
512
  parentUpgradeStaleRootProbeProbability?: number;
513
513
 
514
514
  /**
515
- * How long to wait for a parent probe reply before skipping that candidate.
515
+ * How long to wait for a candidate or idle-parent probe reply.
516
516
  */
517
517
  parentProbeTimeoutMs?: number;
518
518
 
@@ -766,6 +766,8 @@ export type FanoutTreeChannelMetrics = {
766
766
  reparentUpgradeSkipProbeOverloaded: number;
767
767
  reparentUpgradeSkipProbeCooldown: number;
768
768
  parentProbeReqSent: number;
769
+ /** Subset of parentProbeReqSent used to check the current idle parent. */
770
+ parentHealthProbeReqSent: number;
769
771
  parentProbeReqReceived: number;
770
772
  parentProbeReplySent: number;
771
773
  parentProbeReplyReceived: number;
@@ -836,6 +838,11 @@ export interface FanoutTreeEvents extends StreamEvents {
836
838
  "fanout:data": CustomEvent<FanoutTreeDataEvent>;
837
839
  "fanout:unicast": CustomEvent<FanoutTreeUnicastEvent>;
838
840
  "fanout:joined": CustomEvent<{ topic: string; root: string; parent: string }>;
841
+ "fanout:child-joined": CustomEvent<{
842
+ topic: string;
843
+ root: string;
844
+ child: string;
845
+ }>;
839
846
  "fanout:kicked": CustomEvent<{ topic: string; root: string; from: string }>;
840
847
  "fanout:provider": CustomEvent<{
841
848
  namespace: string;
@@ -870,6 +877,19 @@ const PROVIDER_DIRECTORY_MAX_NAMESPACES = 4_096;
870
877
  // large networks can cause unbounded memory growth on the tracker nodes.
871
878
  const TRACKER_DIRECTORY_MAX_ENTRIES = 16_384;
872
879
  const TRACKER_DIRECTORY_MAX_NAMESPACES = 4_096;
880
+ // Best-effort bound on per-key control metrics for keys without an open channel.
881
+ // Every inbound control frame is recorded under its key, including tracker frames
882
+ // for channels we do not host, so bootstraps/relays see an open-ended key set.
883
+ // Open channels keep their metrics on the channel and are never evicted.
884
+ const DETACHED_METRICS_MAX_KEYS = 4_096;
885
+ // Best-effort bounds for provider watches and namespace names. Provider namespaces
886
+ // are commonly per block (`cid:<cid>`), so anything keyed by them must be bounded.
887
+ const PROVIDER_WATCH_MAX_NAMESPACES = 4_096;
888
+ const PROVIDER_NAMESPACE_NAME_MAX_ENTRIES = 16_384;
889
+ // Per-channel ingress token buckets (proxy-publish/unicast) are kept for recent
890
+ // senders only: a few multiples of the child limit, with a small floor.
891
+ const INGRESS_BUCKETS_MIN_ENTRIES = 64;
892
+ const INGRESS_BUCKETS_PER_CHILD = 4;
873
893
 
874
894
  const FANOUT_PROTOCOLS = ["/peerbit/fanout-tree/0.5.0"];
875
895
 
@@ -886,6 +906,7 @@ const DATA_WRITE_FAIL_KICK_COOLDOWN_MS = 2_000;
886
906
  const DATA_WRITE_FAIL_KICK_MAX_PER_EVENT = 4;
887
907
  const PARENT_REPAIR_DEAD_STREAK_THRESHOLD = 16;
888
908
  const PARENT_REPAIR_DEAD_MIN_LIVENESS_MS = 15_000;
909
+ const PARENT_IDLE_PROBE_INTERVAL_MS = 5_000;
889
910
  const REPAIR_RETRY_MIN_MS = 1_000;
890
911
  const REPAIR_RETRY_INTERVAL_FACTOR = 5;
891
912
  // A stream can remain locally readable/writable while silently dropping every
@@ -1162,7 +1183,10 @@ type ChannelState = {
1162
1183
  number,
1163
1184
  { resolve(entries: TrackerCandidate[]): void }
1164
1185
  >;
1165
- pendingParentProbe: Map<number, { resolve(reply?: ParentProbeReply): void }>;
1186
+ pendingParentProbe: Map<
1187
+ number,
1188
+ { peer: PeerStreams; resolve(reply?: ParentProbeReply): void }
1189
+ >;
1166
1190
  parentUpgradeReservationsByHash: Map<
1167
1191
  string,
1168
1192
  { token: number; expiresAt: number; minFreeSlots: number }
@@ -1300,6 +1324,7 @@ const createEmptyMetrics = (): InternalFanoutTreeChannelMetrics => ({
1300
1324
  reparentUpgradeSkipProbeOverloaded: 0,
1301
1325
  reparentUpgradeSkipProbeCooldown: 0,
1302
1326
  parentProbeReqSent: 0,
1327
+ parentHealthProbeReqSent: 0,
1303
1328
  parentProbeReqReceived: 0,
1304
1329
  parentProbeReplySent: 0,
1305
1330
  parentProbeReplyReceived: 0,
@@ -1358,13 +1383,27 @@ const isDataId = (id: Uint8Array) =>
1358
1383
  id[2] === ID_PREFIX[2] &&
1359
1384
  id[3] === ID_PREFIX[3];
1360
1385
 
1386
+ const isProviderControlKind = (kind: number) =>
1387
+ kind === MSG_PROVIDER_ANNOUNCE ||
1388
+ kind === MSG_PROVIDER_QUERY ||
1389
+ kind === MSG_PROVIDER_REPLY ||
1390
+ kind === MSG_PROVIDER_SUBSCRIBE ||
1391
+ kind === MSG_PROVIDER_UNSUBSCRIBE ||
1392
+ kind === MSG_PROVIDER_NOTIFY;
1393
+
1361
1394
  export class FanoutTree extends DirectStream<FanoutTreeEvents> {
1362
1395
  private channelsBySuffixKey = new Map<string, ChannelState>();
1363
1396
  private readonly cachedSuffixKey = new WeakMap<Uint8Array, string>();
1397
+ // Metrics for keys without an open channel, in LRU order (oldest first) and
1398
+ // bounded by DETACHED_METRICS_MAX_KEYS. Open channels own `ch.metrics`.
1364
1399
  private readonly metricsBySuffixKey = new Map<
1365
1400
  string,
1366
1401
  InternalFanoutTreeChannelMetrics
1367
1402
  >();
1403
+ // Provider control frames are keyed per provider namespace (often per block),
1404
+ // which no channel metrics lookup can address, so they share one aggregate.
1405
+ private readonly providerControlMetrics: InternalFanoutTreeChannelMetrics =
1406
+ createEmptyMetrics();
1368
1407
  private readonly joinTimeoutStreakByPeer = new Map<string, number>();
1369
1408
  private readonly joinResetCooldownUntilByPeer = new Map<string, number>();
1370
1409
  private bootstraps: Multiaddr[] = [];
@@ -1551,6 +1590,27 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
1551
1590
  }
1552
1591
  }
1553
1592
 
1593
+ private touchProviderWatchers(
1594
+ suffixKey: string,
1595
+ ): Map<string, ProviderWatchRegistration> {
1596
+ // LRU touch
1597
+ const watchers =
1598
+ this.providerWatchersBySuffixKey.get(suffixKey) ??
1599
+ new Map<string, ProviderWatchRegistration>();
1600
+ this.providerWatchersBySuffixKey.delete(suffixKey);
1601
+ this.providerWatchersBySuffixKey.set(suffixKey, watchers);
1602
+ while (
1603
+ this.providerWatchersBySuffixKey.size > PROVIDER_WATCH_MAX_NAMESPACES
1604
+ ) {
1605
+ const oldest = this.providerWatchersBySuffixKey.keys().next().value as
1606
+ | string
1607
+ | undefined;
1608
+ if (oldest === undefined) break;
1609
+ this.providerWatchersBySuffixKey.delete(oldest);
1610
+ }
1611
+ return watchers;
1612
+ }
1613
+
1554
1614
  private pruneProviderWatchersIfEmpty(suffixKey: string) {
1555
1615
  const watchers = this.providerWatchersBySuffixKey.get(suffixKey);
1556
1616
  if (watchers && watchers.size === 0) {
@@ -1686,7 +1746,19 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
1686
1746
  private getProviderNamespaceId(namespace: string): ProviderNamespaceId {
1687
1747
  const key = sha256Sync(textEncoder.encode(`provider|${namespace}`));
1688
1748
  const suffixKey = toBase64(key.subarray(0, 24));
1749
+ // LRU touch: every announced block adds a namespace, so keep this bounded.
1750
+ this.providerNamespaceBySuffixKey.delete(suffixKey);
1689
1751
  this.providerNamespaceBySuffixKey.set(suffixKey, namespace);
1752
+ while (
1753
+ this.providerNamespaceBySuffixKey.size >
1754
+ PROVIDER_NAMESPACE_NAME_MAX_ENTRIES
1755
+ ) {
1756
+ const oldest = this.providerNamespaceBySuffixKey.keys().next().value as
1757
+ | string
1758
+ | undefined;
1759
+ if (oldest === undefined) break;
1760
+ this.providerNamespaceBySuffixKey.delete(oldest);
1761
+ }
1690
1762
  return { namespace, key, suffixKey };
1691
1763
  }
1692
1764
 
@@ -2505,6 +2577,9 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
2505
2577
  lastTrackerQueryAt: 0,
2506
2578
  };
2507
2579
  this.channelsBySuffixKey.set(id.suffixKey, ch);
2580
+ // The open channel now owns its metrics; take them out of the detached LRU
2581
+ // so they can never be evicted while the channel is open.
2582
+ this.metricsBySuffixKey.delete(id.suffixKey);
2508
2583
  const needsAnnounceLoop = ch.effectiveMaxChildren > 0;
2509
2584
  if (needsAnnounceLoop) {
2510
2585
  ch.announceLoop = this._announceLoop(ch).catch(() => {});
@@ -2705,6 +2780,13 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
2705
2780
  if (!peerHash) return;
2706
2781
  this.joinTimeoutStreakByPeer.delete(peerHash);
2707
2782
 
2783
+ // A disconnected watcher can no longer receive provider notifications.
2784
+ for (const [suffixKey, watchers] of this.providerWatchersBySuffixKey) {
2785
+ if (watchers.delete(peerHash) && watchers.size === 0) {
2786
+ this.providerWatchersBySuffixKey.delete(suffixKey);
2787
+ }
2788
+ }
2789
+
2708
2790
  // Detach from a disconnected parent immediately, so children can rejoin.
2709
2791
  // This is more reliable than polling `getConnections()` because the underlay
2710
2792
  // can flap/reconnect faster than the join loop cadence.
@@ -2803,6 +2885,8 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
2803
2885
 
2804
2886
  this.channelsBySuffixKey.delete(id.suffixKey);
2805
2887
  this.trackerBySuffixKey.delete(id.suffixKey);
2888
+ // Keep the counters readable after close (and across a reopen), bounded.
2889
+ this.storeDetachedMetrics(id.suffixKey, ch.metrics);
2806
2890
 
2807
2891
  await Promise.all(pendingSends);
2808
2892
  }
@@ -3985,20 +4069,57 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
3985
4069
  private getMetricsForSuffixKey(
3986
4070
  suffixKey: string,
3987
4071
  ): InternalFanoutTreeChannelMetrics {
3988
- let m = this.metricsBySuffixKey.get(suffixKey);
3989
- if (!m) {
3990
- m = createEmptyMetrics();
3991
- this.metricsBySuffixKey.set(suffixKey, m);
3992
- }
4072
+ const ch = this.channelsBySuffixKey.get(suffixKey);
4073
+ if (ch) return ch.metrics;
4074
+ const m = this.metricsBySuffixKey.get(suffixKey) ?? createEmptyMetrics();
4075
+ this.storeDetachedMetrics(suffixKey, m);
3993
4076
  return m;
3994
4077
  }
3995
4078
 
3996
- private recordControlSend(bytes: Uint8Array, transmissions: number) {
4079
+ private storeDetachedMetrics(
4080
+ suffixKey: string,
4081
+ metrics: InternalFanoutTreeChannelMetrics,
4082
+ ) {
4083
+ // LRU touch
4084
+ this.metricsBySuffixKey.delete(suffixKey);
4085
+ this.metricsBySuffixKey.set(suffixKey, metrics);
4086
+ while (this.metricsBySuffixKey.size > DETACHED_METRICS_MAX_KEYS) {
4087
+ const oldest = this.metricsBySuffixKey.keys().next().value as
4088
+ | string
4089
+ | undefined;
4090
+ if (oldest === undefined) break;
4091
+ this.metricsBySuffixKey.delete(oldest);
4092
+ }
4093
+ }
4094
+
4095
+ private getControlMetrics(
4096
+ suffixKey: string,
4097
+ kind: number,
4098
+ ): InternalFanoutTreeChannelMetrics {
4099
+ return isProviderControlKind(kind)
4100
+ ? this.providerControlMetrics
4101
+ : this.getMetricsForSuffixKey(suffixKey);
4102
+ }
4103
+
4104
+ /**
4105
+ * Returns aggregate diagnostic counters for provider control frames.
4106
+ *
4107
+ * @internal
4108
+ */
4109
+ public getProviderControlMetrics(): FanoutTreeChannelMetrics {
4110
+ return this.providerControlMetrics;
4111
+ }
4112
+
4113
+ private recordControlSend(
4114
+ bytes: Uint8Array,
4115
+ transmissions: number,
4116
+ probePurpose: "upgrade" | "health" = "upgrade",
4117
+ ) {
3997
4118
  if (transmissions <= 0) return;
3998
4119
  if (bytes.length < 1 + 24) return;
3999
4120
  const kind = bytes[0]!;
4000
4121
  const suffixKey = toBase64(bytes.subarray(1, 25));
4001
- const m = this.getMetricsForSuffixKey(suffixKey);
4122
+ const m = this.getControlMetrics(suffixKey, kind);
4002
4123
  m.controlSends += transmissions;
4003
4124
  const sentBytes = bytes.byteLength * transmissions;
4004
4125
  m.controlBytesSent += sentBytes;
@@ -4053,6 +4174,9 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
4053
4174
  break;
4054
4175
  case MSG_PARENT_PROBE_REQ:
4055
4176
  m.parentProbeReqSent += transmissions;
4177
+ if (probePurpose === "health") {
4178
+ m.parentHealthProbeReqSent += transmissions;
4179
+ }
4056
4180
  m.controlBytesSentJoin += sentBytes;
4057
4181
  break;
4058
4182
  case MSG_PARENT_PROBE_REPLY:
@@ -4077,7 +4201,7 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
4077
4201
  kind: number,
4078
4202
  bytesReceived: number,
4079
4203
  ) {
4080
- const m = this.getMetricsForSuffixKey(suffixKey);
4204
+ const m = this.getControlMetrics(suffixKey, kind);
4081
4205
  m.controlReceives += 1;
4082
4206
  m.controlBytesReceived += bytesReceived;
4083
4207
  switch (kind) {
@@ -4205,13 +4329,14 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
4205
4329
  to: string,
4206
4330
  bytes: Uint8Array,
4207
4331
  signal?: AbortSignal,
4332
+ probePurpose: "upgrade" | "health" = "upgrade",
4208
4333
  ) {
4209
4334
  if (signal?.aborted) {
4210
4335
  throw signal.reason ?? new AbortError("fanout control send aborted");
4211
4336
  }
4212
4337
  const stream = this.peers.get(to);
4213
4338
  if (!stream) return;
4214
- this.recordControlSend(bytes, 1);
4339
+ this.recordControlSend(bytes, 1, probePurpose);
4215
4340
  const message = await this.createMessage(bytes, {
4216
4341
  mode: new AnyWhere(),
4217
4342
  priority: CONTROL_PRIORITY,
@@ -4298,17 +4423,35 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
4298
4423
  tokens = Math.min(capacity, tokens + (elapsedMs * budgetBps) / 1_000);
4299
4424
  }
4300
4425
  if (tokens < costBytes) {
4301
- // LRU-touch on access to keep the bucket map bounded by active children.
4302
- byPeer.delete(fromHash);
4303
- byPeer.set(fromHash, { tokens, lastRefillAt: now });
4426
+ this.setIngressBucket(ch, byPeer, fromHash, { tokens, lastRefillAt: now });
4304
4427
  return false;
4305
4428
  }
4306
4429
  tokens -= costBytes;
4307
- byPeer.delete(fromHash);
4308
- byPeer.set(fromHash, { tokens, lastRefillAt: now });
4430
+ this.setIngressBucket(ch, byPeer, fromHash, { tokens, lastRefillAt: now });
4309
4431
  return true;
4310
4432
  }
4311
4433
 
4434
+ private setIngressBucket(
4435
+ ch: ChannelState,
4436
+ byPeer: Map<string, { tokens: number; lastRefillAt: number }>,
4437
+ fromHash: string,
4438
+ bucket: { tokens: number; lastRefillAt: number },
4439
+ ) {
4440
+ // LRU touch, capped: long-lived (pinned) roots see an open-ended set of
4441
+ // children over time, and buckets are not dropped when a child leaves.
4442
+ byPeer.delete(fromHash);
4443
+ byPeer.set(fromHash, bucket);
4444
+ const maxBuckets = Math.max(
4445
+ INGRESS_BUCKETS_MIN_ENTRIES,
4446
+ ch.maxChildren * INGRESS_BUCKETS_PER_CHILD,
4447
+ );
4448
+ while (byPeer.size > maxBuckets) {
4449
+ const oldest = byPeer.keys().next().value as string | undefined;
4450
+ if (oldest === undefined) break;
4451
+ byPeer.delete(oldest);
4452
+ }
4453
+ }
4454
+
4312
4455
  private async _sendData(
4313
4456
  ch: ChannelState,
4314
4457
  to: string[],
@@ -5894,24 +6037,56 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
5894
6037
  signal: AbortSignal,
5895
6038
  minFreeSlots = 0,
5896
6039
  reserveRootCapacity = true,
6040
+ purpose: "upgrade" | "health" = "upgrade",
5897
6041
  ): Promise<ParentProbeReply | undefined> {
5898
- if (!this.peers.get(parentHash)) return;
5899
- const reqId = (this.random() * 0xffffffff) >>> 0;
5900
- const p = new Promise<ParentProbeReply | undefined>((resolve) => {
5901
- ch.pendingParentProbe.set(reqId, { resolve });
5902
- });
5903
- await this._sendControl(
5904
- parentHash,
5905
- this.codec.encodeParentProbeReq(ch.id.key, reqId, minFreeSlots, reserveRootCapacity),
5906
- );
5907
- const res = await Promise.race([
5908
- p,
5909
- delay(Math.max(1, timeoutMs), { signal }).then(
5910
- (): undefined => undefined,
5911
- ),
5912
- ]);
5913
- if (!res) ch.pendingParentProbe.delete(reqId);
5914
- return res;
6042
+ const peer = this.peers.get(parentHash);
6043
+ if (!peer) return;
6044
+ const controller = new AbortController();
6045
+ let reqId = (this.random() * 0xffffffff) >>> 0;
6046
+ while (ch.pendingParentProbe.has(reqId)) reqId = (reqId + 1) >>> 0;
6047
+ let pending:
6048
+ | { peer: PeerStreams; resolve(reply?: ParentProbeReply): void }
6049
+ | undefined;
6050
+ let onAbort!: () => void;
6051
+ let timer: ReturnType<typeof setTimeout> | undefined;
6052
+ try {
6053
+ return await new Promise<ParentProbeReply | undefined>((resolve, reject) => {
6054
+ pending = { peer, resolve };
6055
+ ch.pendingParentProbe.set(reqId, pending);
6056
+ onAbort = () => {
6057
+ const reason = signal.reason ?? new AbortError("fanout probe aborted");
6058
+ controller.abort(reason);
6059
+ reject(reason);
6060
+ };
6061
+ signal.addEventListener("abort", onAbort, { once: true });
6062
+ if (signal.aborted) return onAbort();
6063
+ // Bound signing/queueing as well as the reply; a stuck send must not
6064
+ // prevent the join loop from recovering or closing the channel.
6065
+ timer = setTimeout(() => {
6066
+ controller.abort(new AbortError("fanout probe timed out"));
6067
+ resolve(undefined);
6068
+ }, Math.max(1, timeoutMs));
6069
+ timer.unref?.();
6070
+ void this._sendControl(
6071
+ parentHash,
6072
+ this.codec.encodeParentProbeReq(
6073
+ ch.id.key,
6074
+ reqId,
6075
+ minFreeSlots,
6076
+ reserveRootCapacity,
6077
+ ),
6078
+ controller.signal,
6079
+ purpose,
6080
+ ).catch(() => resolve(undefined));
6081
+ });
6082
+ } finally {
6083
+ clearTimeout(timer);
6084
+ signal.removeEventListener("abort", onAbort);
6085
+ if (ch.pendingParentProbe.get(reqId) === pending) {
6086
+ ch.pendingParentProbe.delete(reqId);
6087
+ }
6088
+ controller.abort(new AbortError("fanout probe settled"));
6089
+ }
5915
6090
  }
5916
6091
 
5917
6092
  private async _joinLoop(
@@ -6020,6 +6195,10 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
6020
6195
  if (initialJoinRemainingMs() === 0) throwInitialJoinTimeout();
6021
6196
  };
6022
6197
  let nextParentUpgradeCheckAt = 0;
6198
+ let nextParentHealthProbeAt = 0;
6199
+ let failedParentProbe:
6200
+ | { peer: PeerStreams; route: ChannelState["routeFromRoot"] }
6201
+ | undefined;
6023
6202
  let parentUpgradeCheckSeq = 0;
6024
6203
  let parentUpgradeActiveGuardBackoffMs = 0;
6025
6204
  const unsuccessfulColdBootstrapPeers = new Set<string>();
@@ -6110,8 +6289,10 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
6110
6289
 
6111
6290
  // Parent disappeared? Rejoin.
6112
6291
  if (ch.parent) {
6113
- const parentPeer = this.peers.get(ch.parent);
6292
+ const parentHash = ch.parent;
6293
+ const parentPeer = this.peers.get(parentHash);
6114
6294
  let connected = false;
6295
+ let probeFailed = false;
6115
6296
  if (parentPeer) {
6116
6297
  try {
6117
6298
  const conns = this.components.connectionManager.getConnections(
@@ -6122,8 +6303,61 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
6122
6303
  connected = parentPeer.isReadable || parentPeer.isWritable;
6123
6304
  }
6124
6305
  }
6306
+ const now = Date.now();
6307
+ if (now - ch.lastParentDataAt < PARENT_IDLE_PROBE_INTERVAL_MS) {
6308
+ failedParentProbe = undefined;
6309
+ } else if (
6310
+ connected &&
6311
+ parentPeer &&
6312
+ now >= nextParentHealthProbeAt
6313
+ ) {
6314
+ const route = ch.routeFromRoot;
6315
+ const lastDataAt = ch.lastParentDataAt;
6316
+ const reply = await this.probeParentCandidate(
6317
+ ch,
6318
+ parentHash,
6319
+ parentUpgrade.probe.timeoutMs,
6320
+ signal,
6321
+ 0,
6322
+ false,
6323
+ "health",
6324
+ );
6325
+ if (
6326
+ ch.closed ||
6327
+ signal.aborted ||
6328
+ this.channelsBySuffixKey.get(ch.id.suffixKey) !== ch ||
6329
+ ch.parent !== parentHash ||
6330
+ ch.routeFromRoot !== route ||
6331
+ this.peers.get(parentHash) !== parentPeer
6332
+ ) {
6333
+ failedParentProbe = undefined;
6334
+ continue;
6335
+ }
6336
+ nextParentHealthProbeAt = Date.now() + PARENT_IDLE_PROBE_INTERVAL_MS;
6337
+ // A full parent is still healthy. Do not reserve capacity or
6338
+ // refresh lastParentDataAt: probes must not mask stalled data.
6339
+ if (reply || ch.lastParentDataAt !== lastDataAt) {
6340
+ failedParentProbe = undefined;
6341
+ } else if (
6342
+ failedParentProbe?.peer === parentPeer &&
6343
+ failedParentProbe.route === route
6344
+ ) {
6345
+ probeFailed = true;
6346
+ connected = false;
6347
+ cooldownUntilByHash.set(
6348
+ parentHash,
6349
+ Date.now() + candidateCooldownMs,
6350
+ );
6351
+ } else {
6352
+ failedParentProbe = { peer: parentPeer, route };
6353
+ nextParentHealthProbeAt = Date.now() + Math.max(retryMs, 1_000);
6354
+ }
6355
+ }
6125
6356
  if (!connected) {
6126
- ch.metrics.reparentDisconnect += 1;
6357
+ if (probeFailed) ch.metrics.reparentStale += 1;
6358
+ else ch.metrics.reparentDisconnect += 1;
6359
+ failedParentProbe = undefined;
6360
+ nextParentHealthProbeAt = 0;
6127
6361
  const hadChildren = ch.children.size > 0;
6128
6362
  this.detachFromParent(ch);
6129
6363
  nextParentUpgradeCheckAt = 0;
@@ -8424,11 +8658,7 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
8424
8658
  const want = Math.max(1, decoded.want);
8425
8659
  const ttlMs = Math.min(120_000, Math.max(1_000, decoded.ttlMs));
8426
8660
  const now = Date.now();
8427
- let watchers = this.providerWatchersBySuffixKey.get(suffixKey);
8428
- if (!watchers) {
8429
- watchers = new Map<string, ProviderWatchRegistration>();
8430
- this.providerWatchersBySuffixKey.set(suffixKey, watchers);
8431
- }
8661
+ const watchers = this.touchProviderWatchers(suffixKey);
8432
8662
  watchers.set(fromHash, {
8433
8663
  hash: fromHash,
8434
8664
  want,
@@ -8566,7 +8796,14 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
8566
8796
  if (!decoded) return false;
8567
8797
  const { reqId, ...reply } = decoded;
8568
8798
  const pending = ch.pendingParentProbe.get(reqId);
8569
- if (!pending) return true;
8799
+ if (
8800
+ !pending ||
8801
+ pending.peer !== peerStream ||
8802
+ pending.peer.publicKey.hashcode() !== fromHash ||
8803
+ this.peers.get(fromHash) !== peerStream
8804
+ ) {
8805
+ return true;
8806
+ }
8570
8807
  ch.pendingParentProbe.delete(reqId);
8571
8808
  pending.resolve(reply);
8572
8809
  this.touchPeerHint(ch, fromHash);
@@ -8851,7 +9088,8 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
8851
9088
  void this.kickChildHashes(ch, [worstChild]).catch(() => {});
8852
9089
  }
8853
9090
 
8854
- ch.children.set(fromHash, { bidPerByte });
9091
+ const child = { bidPerByte };
9092
+ ch.children.set(fromHash, child);
8855
9093
  if (ch.isRoot && consumedParentUpgradeReservation) {
8856
9094
  ch.parentUpgradeTrackerNoCapacityUntil = Math.max(
8857
9095
  ch.parentUpgradeTrackerNoCapacityUntil,
@@ -8868,6 +9106,12 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
8868
9106
  );
8869
9107
  void (async () => {
8870
9108
  await this._sendControl(fromHash, joinAccept);
9109
+ if (ch.closed || ch.children.get(fromHash) !== child) return;
9110
+ this.dispatchEvent(
9111
+ new CustomEvent("fanout:child-joined", {
9112
+ detail: { topic: ch.id.topic, root: ch.id.root, child: fromHash },
9113
+ }),
9114
+ );
8871
9115
  if (ch.endSeqExclusive > 0) {
8872
9116
  await this._sendControl(
8873
9117
  fromHash,
@@ -9311,6 +9555,21 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
9311
9555
  }
9312
9556
 
9313
9557
  if (kind === MSG_KICK) {
9558
+ if (this.peers.get(fromHash) !== peerStream) return true;
9559
+ if (fromHash !== ch.parent) {
9560
+ if (ch.parentUpgradeGrace?.previousParent === fromHash) {
9561
+ // This attachment was evicted; never roll back to it.
9562
+ this.clearParentUpgradeGrace(ch);
9563
+ }
9564
+ if (ch.parentShadow?.hash === fromHash) {
9565
+ ch.parentShadow = undefined;
9566
+ ch.metrics.parentShadowReset += 1;
9567
+ if (this.parentUpgradeShadowInFlightSuffixKey === ch.id.suffixKey) {
9568
+ this.parentUpgradeShadowInFlightSuffixKey = undefined;
9569
+ }
9570
+ }
9571
+ return true;
9572
+ }
9314
9573
  ch.metrics.reparentKicked += 1;
9315
9574
  ch.parent = undefined;
9316
9575
  ch.level = Number.POSITIVE_INFINITY;