@peerbit/pubsub 5.4.9 → 5.4.12

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@peerbit/pubsub",
-  "version": "5.4.9",
+  "version": "5.4.12",
   "description": "Direct streaming for libp2p",
   "type": "module",
   "types": "./dist/src/index.d.ts",
@@ -67,7 +67,7 @@
   },
   "homepage": "https://github.com/dao-xyz/peerbit#readme",
   "devDependencies": {
-    "@peerbit/libp2p-test-utils": "3.0.10"
+    "@peerbit/libp2p-test-utils": "3.0.11"
   },
   "dependencies": {
     "@dao-xyz/borsh": "^6.0.0",
@@ -86,12 +86,12 @@
     "uint8arraylist": "^3.0.2",
     "uint8arrays": "^5.1.0",
     "@peerbit/any-store-interface": "1.1.4",
+    "@peerbit/crypto": "3.1.6",
     "@peerbit/logger": "2.0.2",
-    "@peerbit/stream": "5.2.3",
     "@peerbit/pubsub-interface": "5.2.2",
-    "@peerbit/crypto": "3.1.6",
-    "@peerbit/time": "3.0.1",
-    "@peerbit/stream-interface": "6.0.16"
+    "@peerbit/stream": "5.2.4",
+    "@peerbit/stream-interface": "6.0.16",
+    "@peerbit/time": "3.0.1"
   },
   "scripts": {
     "bench": "TS_NODE_PROJECT=../../../tsconfig.json node --loader ts-node/esm ./benchmark/index.ts",

package/src/fanout-tree.ts

@@ -512,7 +512,7 @@ export type FanoutTreeJoinOptions = {
 	parentUpgradeStaleRootProbeProbability?: number;
 
 	/**
-	 * How long to wait for a parent probe reply before skipping that candidate.
+	 * How long to wait for a candidate or idle-parent probe reply.
 	 */
 	parentProbeTimeoutMs?: number;
 
@@ -766,6 +766,8 @@ export type FanoutTreeChannelMetrics = {
 	reparentUpgradeSkipProbeOverloaded: number;
 	reparentUpgradeSkipProbeCooldown: number;
 	parentProbeReqSent: number;
+	/** Subset of parentProbeReqSent used to check the current idle parent. */
+	parentHealthProbeReqSent: number;
 	parentProbeReqReceived: number;
 	parentProbeReplySent: number;
 	parentProbeReplyReceived: number;
@@ -836,6 +838,11 @@ export interface FanoutTreeEvents extends StreamEvents {
 	"fanout:data": CustomEvent<FanoutTreeDataEvent>;
 	"fanout:unicast": CustomEvent<FanoutTreeUnicastEvent>;
 	"fanout:joined": CustomEvent<{ topic: string; root: string; parent: string }>;
+	"fanout:child-joined": CustomEvent<{
+		topic: string;
+		root: string;
+		child: string;
+	}>;
 	"fanout:kicked": CustomEvent<{ topic: string; root: string; from: string }>;
 	"fanout:provider": CustomEvent<{
 		namespace: string;
@@ -870,6 +877,19 @@ const PROVIDER_DIRECTORY_MAX_NAMESPACES = 4_096;
 // large networks can cause unbounded memory growth on the tracker nodes.
 const TRACKER_DIRECTORY_MAX_ENTRIES = 16_384;
 const TRACKER_DIRECTORY_MAX_NAMESPACES = 4_096;
+// Best-effort bound on per-key control metrics for keys without an open channel.
+// Every inbound control frame is recorded under its key, including tracker frames
+// for channels we do not host, so bootstraps/relays see an open-ended key set.
+// Open channels keep their metrics on the channel and are never evicted.
+const DETACHED_METRICS_MAX_KEYS = 4_096;
+// Best-effort bounds for provider watches and namespace names. Provider namespaces
+// are commonly per block (`cid:<cid>`), so anything keyed by them must be bounded.
+const PROVIDER_WATCH_MAX_NAMESPACES = 4_096;
+const PROVIDER_NAMESPACE_NAME_MAX_ENTRIES = 16_384;
+// Per-channel ingress token buckets (proxy-publish/unicast) are kept for recent
+// senders only: a few multiples of the child limit, with a small floor.
+const INGRESS_BUCKETS_MIN_ENTRIES = 64;
+const INGRESS_BUCKETS_PER_CHILD = 4;
 
 const FANOUT_PROTOCOLS = ["/peerbit/fanout-tree/0.5.0"];
 
@@ -886,6 +906,7 @@ const DATA_WRITE_FAIL_KICK_COOLDOWN_MS = 2_000;
 const DATA_WRITE_FAIL_KICK_MAX_PER_EVENT = 4;
 const PARENT_REPAIR_DEAD_STREAK_THRESHOLD = 16;
 const PARENT_REPAIR_DEAD_MIN_LIVENESS_MS = 15_000;
+const PARENT_IDLE_PROBE_INTERVAL_MS = 5_000;
 const REPAIR_RETRY_MIN_MS = 1_000;
 const REPAIR_RETRY_INTERVAL_FACTOR = 5;
 // A stream can remain locally readable/writable while silently dropping every
@@ -1162,7 +1183,10 @@ type ChannelState = {
 		number,
 		{ resolve(entries: TrackerCandidate[]): void }
 	>;
-	pendingParentProbe: Map<number, { resolve(reply?: ParentProbeReply): void }>;
+	pendingParentProbe: Map<
+		number,
+		{ peer: PeerStreams; resolve(reply?: ParentProbeReply): void }
+	>;
 	parentUpgradeReservationsByHash: Map<
 		string,
 		{ token: number; expiresAt: number; minFreeSlots: number }
@@ -1300,6 +1324,7 @@ const createEmptyMetrics = (): InternalFanoutTreeChannelMetrics => ({
 	reparentUpgradeSkipProbeOverloaded: 0,
 	reparentUpgradeSkipProbeCooldown: 0,
 	parentProbeReqSent: 0,
+	parentHealthProbeReqSent: 0,
 	parentProbeReqReceived: 0,
 	parentProbeReplySent: 0,
 	parentProbeReplyReceived: 0,
@@ -1358,13 +1383,27 @@ const isDataId = (id: Uint8Array) =>
 	id[2] === ID_PREFIX[2] &&
 	id[3] === ID_PREFIX[3];
 
+const isProviderControlKind = (kind: number) =>
+	kind === MSG_PROVIDER_ANNOUNCE ||
+	kind === MSG_PROVIDER_QUERY ||
+	kind === MSG_PROVIDER_REPLY ||
+	kind === MSG_PROVIDER_SUBSCRIBE ||
+	kind === MSG_PROVIDER_UNSUBSCRIBE ||
+	kind === MSG_PROVIDER_NOTIFY;
+
 export class FanoutTree extends DirectStream<FanoutTreeEvents> {
 	private channelsBySuffixKey = new Map<string, ChannelState>();
 	private readonly cachedSuffixKey = new WeakMap<Uint8Array, string>();
+	// Metrics for keys without an open channel, in LRU order (oldest first) and
+	// bounded by DETACHED_METRICS_MAX_KEYS. Open channels own `ch.metrics`.
 	private readonly metricsBySuffixKey = new Map<
 		string,
 		InternalFanoutTreeChannelMetrics
 	>();
+	// Provider control frames are keyed per provider namespace (often per block),
+	// which no channel metrics lookup can address, so they share one aggregate.
+	private readonly providerControlMetrics: InternalFanoutTreeChannelMetrics =
+		createEmptyMetrics();
 	private readonly joinTimeoutStreakByPeer = new Map<string, number>();
 	private readonly joinResetCooldownUntilByPeer = new Map<string, number>();
 	private bootstraps: Multiaddr[] = [];
@@ -1425,6 +1464,11 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
 		this.topicRootControlPlane =
 			topicRootControlPlane ?? new TopicRootControlPlane();
 
+		this.registerUnderlayPeerDisconnectHandler();
+	}
+
+	private registerUnderlayPeerDisconnectHandler() {
+		if (this.underlayPeerDisconnectHandler) return;
 		const onPeerDisconnect = (ev: any) => {
 			const peerId = ev?.detail;
 			if (!peerId) return;
@@ -1443,6 +1487,15 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
 		);
 	}
 
+	public override async start() {
+		await super.start();
+		// stop() removes the listener, so a restarted instance must add it again
+		// (but not while a stop that raced this start is still tearing down).
+		if (this.started && !this.stopping) {
+			this.registerUnderlayPeerDisconnectHandler();
+		}
+	}
+
 	private createLoopCloseController(): AbortController {
 		const controller = new AbortController();
 		const stopSignal = this.closeController.signal;
@@ -1551,6 +1604,27 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
 		}
 	}
 
+	private touchProviderWatchers(
+		suffixKey: string,
+	): Map<string, ProviderWatchRegistration> {
+		// LRU touch
+		const watchers =
+			this.providerWatchersBySuffixKey.get(suffixKey) ??
+			new Map<string, ProviderWatchRegistration>();
+		this.providerWatchersBySuffixKey.delete(suffixKey);
+		this.providerWatchersBySuffixKey.set(suffixKey, watchers);
+		while (
+			this.providerWatchersBySuffixKey.size > PROVIDER_WATCH_MAX_NAMESPACES
+		) {
+			const oldest = this.providerWatchersBySuffixKey.keys().next().value as
+				| string
+				| undefined;
+			if (oldest === undefined) break;
+			this.providerWatchersBySuffixKey.delete(oldest);
+		}
+		return watchers;
+	}
+
 	private pruneProviderWatchersIfEmpty(suffixKey: string) {
 		const watchers = this.providerWatchersBySuffixKey.get(suffixKey);
 		if (watchers && watchers.size === 0) {
@@ -1686,7 +1760,19 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
 	private getProviderNamespaceId(namespace: string): ProviderNamespaceId {
 		const key = sha256Sync(textEncoder.encode(`provider|${namespace}`));
 		const suffixKey = toBase64(key.subarray(0, 24));
+		// LRU touch: every announced block adds a namespace, so keep this bounded.
+		this.providerNamespaceBySuffixKey.delete(suffixKey);
 		this.providerNamespaceBySuffixKey.set(suffixKey, namespace);
+		while (
+			this.providerNamespaceBySuffixKey.size >
+			PROVIDER_NAMESPACE_NAME_MAX_ENTRIES
+		) {
+			const oldest = this.providerNamespaceBySuffixKey.keys().next().value as
+				| string
+				| undefined;
+			if (oldest === undefined) break;
+			this.providerNamespaceBySuffixKey.delete(oldest);
+		}
 		return { namespace, key, suffixKey };
 	}
 
@@ -2505,6 +2591,9 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
 			lastTrackerQueryAt: 0,
 		};
 		this.channelsBySuffixKey.set(id.suffixKey, ch);
+		// The open channel now owns its metrics; take them out of the detached LRU
+		// so they can never be evicted while the channel is open.
+		this.metricsBySuffixKey.delete(id.suffixKey);
 		const needsAnnounceLoop = ch.effectiveMaxChildren > 0;
 		if (needsAnnounceLoop) {
 			ch.announceLoop = this._announceLoop(ch).catch(() => {});
@@ -2705,6 +2794,13 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
 		if (!peerHash) return;
 		this.joinTimeoutStreakByPeer.delete(peerHash);
 
+		// A disconnected watcher can no longer receive provider notifications.
+		for (const [suffixKey, watchers] of this.providerWatchersBySuffixKey) {
+			if (watchers.delete(peerHash) && watchers.size === 0) {
+				this.providerWatchersBySuffixKey.delete(suffixKey);
+			}
+		}
+
 		// Detach from a disconnected parent immediately, so children can rejoin.
 		// This is more reliable than polling `getConnections()` because the underlay
 		// can flap/reconnect faster than the join loop cadence.
@@ -2803,6 +2899,8 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
 
 		this.channelsBySuffixKey.delete(id.suffixKey);
 		this.trackerBySuffixKey.delete(id.suffixKey);
+		// Keep the counters readable after close (and across a reopen), bounded.
+		this.storeDetachedMetrics(id.suffixKey, ch.metrics);
 
 		await Promise.all(pendingSends);
 	}
@@ -3985,20 +4083,57 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
 	private getMetricsForSuffixKey(
 		suffixKey: string,
 	): InternalFanoutTreeChannelMetrics {
-		let m = this.metricsBySuffixKey.get(suffixKey);
-		if (!m) {
-			m = createEmptyMetrics();
-			this.metricsBySuffixKey.set(suffixKey, m);
-		}
+		const ch = this.channelsBySuffixKey.get(suffixKey);
+		if (ch) return ch.metrics;
+		const m = this.metricsBySuffixKey.get(suffixKey) ?? createEmptyMetrics();
+		this.storeDetachedMetrics(suffixKey, m);
 		return m;
 	}
 
-	private recordControlSend(bytes: Uint8Array, transmissions: number) {
+	private storeDetachedMetrics(
+		suffixKey: string,
+		metrics: InternalFanoutTreeChannelMetrics,
+	) {
+		// LRU touch
+		this.metricsBySuffixKey.delete(suffixKey);
+		this.metricsBySuffixKey.set(suffixKey, metrics);
+		while (this.metricsBySuffixKey.size > DETACHED_METRICS_MAX_KEYS) {
+			const oldest = this.metricsBySuffixKey.keys().next().value as
+				| string
+				| undefined;
+			if (oldest === undefined) break;
+			this.metricsBySuffixKey.delete(oldest);
+		}
+	}
+
+	private getControlMetrics(
+		suffixKey: string,
+		kind: number,
+	): InternalFanoutTreeChannelMetrics {
+		return isProviderControlKind(kind)
+			? this.providerControlMetrics
+			: this.getMetricsForSuffixKey(suffixKey);
+	}
+
+	/**
+	 * Returns aggregate diagnostic counters for provider control frames.
+	 *
+	 * @internal
+	 */
+	public getProviderControlMetrics(): FanoutTreeChannelMetrics {
+		return this.providerControlMetrics;
+	}
+
+	private recordControlSend(
+		bytes: Uint8Array,
+		transmissions: number,
+		probePurpose: "upgrade" | "health" = "upgrade",
+	) {
 		if (transmissions <= 0) return;
 		if (bytes.length < 1 + 24) return;
 		const kind = bytes[0]!;
 		const suffixKey = toBase64(bytes.subarray(1, 25));
-		const m = this.getMetricsForSuffixKey(suffixKey);
+		const m = this.getControlMetrics(suffixKey, kind);
 		m.controlSends += transmissions;
 		const sentBytes = bytes.byteLength * transmissions;
 		m.controlBytesSent += sentBytes;
@@ -4053,6 +4188,9 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
 				break;
 			case MSG_PARENT_PROBE_REQ:
 				m.parentProbeReqSent += transmissions;
+				if (probePurpose === "health") {
+					m.parentHealthProbeReqSent += transmissions;
+				}
 				m.controlBytesSentJoin += sentBytes;
 				break;
 			case MSG_PARENT_PROBE_REPLY:
@@ -4077,7 +4215,7 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
 		kind: number,
 		bytesReceived: number,
 	) {
-		const m = this.getMetricsForSuffixKey(suffixKey);
+		const m = this.getControlMetrics(suffixKey, kind);
 		m.controlReceives += 1;
 		m.controlBytesReceived += bytesReceived;
 		switch (kind) {
@@ -4205,13 +4343,14 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
 		to: string,
 		bytes: Uint8Array,
 		signal?: AbortSignal,
+		probePurpose: "upgrade" | "health" = "upgrade",
 	) {
 		if (signal?.aborted) {
 			throw signal.reason ?? new AbortError("fanout control send aborted");
 		}
 		const stream = this.peers.get(to);
 		if (!stream) return;
-		this.recordControlSend(bytes, 1);
+		this.recordControlSend(bytes, 1, probePurpose);
 		const message = await this.createMessage(bytes, {
 			mode: new AnyWhere(),
 			priority: CONTROL_PRIORITY,
@@ -4298,17 +4437,35 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
 			tokens = Math.min(capacity, tokens + (elapsedMs * budgetBps) / 1_000);
 		}
 		if (tokens < costBytes) {
-			// LRU-touch on access to keep the bucket map bounded by active children.
-			byPeer.delete(fromHash);
-			byPeer.set(fromHash, { tokens, lastRefillAt: now });
+			this.setIngressBucket(ch, byPeer, fromHash, { tokens, lastRefillAt: now });
 			return false;
 		}
 		tokens -= costBytes;
-		byPeer.delete(fromHash);
-		byPeer.set(fromHash, { tokens, lastRefillAt: now });
+		this.setIngressBucket(ch, byPeer, fromHash, { tokens, lastRefillAt: now });
 		return true;
 	}
 
+	private setIngressBucket(
+		ch: ChannelState,
+		byPeer: Map<string, { tokens: number; lastRefillAt: number }>,
+		fromHash: string,
+		bucket: { tokens: number; lastRefillAt: number },
+	) {
+		// LRU touch, capped: long-lived (pinned) roots see an open-ended set of
+		// children over time, and buckets are not dropped when a child leaves.
+		byPeer.delete(fromHash);
+		byPeer.set(fromHash, bucket);
+		const maxBuckets = Math.max(
+			INGRESS_BUCKETS_MIN_ENTRIES,
+			ch.maxChildren * INGRESS_BUCKETS_PER_CHILD,
+		);
+		while (byPeer.size > maxBuckets) {
+			const oldest = byPeer.keys().next().value as string | undefined;
+			if (oldest === undefined) break;
+			byPeer.delete(oldest);
+		}
+	}
+
 	private async _sendData(
 		ch: ChannelState,
 		to: string[],
@@ -5894,24 +6051,56 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
 		signal: AbortSignal,
 		minFreeSlots = 0,
 		reserveRootCapacity = true,
+		purpose: "upgrade" | "health" = "upgrade",
 	): Promise<ParentProbeReply | undefined> {
-		if (!this.peers.get(parentHash)) return;
-		const reqId = (this.random() * 0xffffffff) >>> 0;
-		const p = new Promise<ParentProbeReply | undefined>((resolve) => {
-			ch.pendingParentProbe.set(reqId, { resolve });
-		});
-		await this._sendControl(
-			parentHash,
-			this.codec.encodeParentProbeReq(ch.id.key, reqId, minFreeSlots, reserveRootCapacity),
-		);
-		const res = await Promise.race([
-			p,
-			delay(Math.max(1, timeoutMs), { signal }).then(
-				(): undefined => undefined,
-			),
-		]);
-		if (!res) ch.pendingParentProbe.delete(reqId);
-		return res;
+		const peer = this.peers.get(parentHash);
+		if (!peer) return;
+		const controller = new AbortController();
+		let reqId = (this.random() * 0xffffffff) >>> 0;
+		while (ch.pendingParentProbe.has(reqId)) reqId = (reqId + 1) >>> 0;
+		let pending:
+			| { peer: PeerStreams; resolve(reply?: ParentProbeReply): void }
+			| undefined;
+		let onAbort!: () => void;
+		let timer: ReturnType<typeof setTimeout> | undefined;
+		try {
+			return await new Promise<ParentProbeReply | undefined>((resolve, reject) => {
+				pending = { peer, resolve };
+				ch.pendingParentProbe.set(reqId, pending);
+				onAbort = () => {
+					const reason = signal.reason ?? new AbortError("fanout probe aborted");
+					controller.abort(reason);
+					reject(reason);
+				};
+				signal.addEventListener("abort", onAbort, { once: true });
+				if (signal.aborted) return onAbort();
+				// Bound signing/queueing as well as the reply; a stuck send must not
+				// prevent the join loop from recovering or closing the channel.
+				timer = setTimeout(() => {
+					controller.abort(new AbortError("fanout probe timed out"));
+					resolve(undefined);
+				}, Math.max(1, timeoutMs));
+				timer.unref?.();
+				void this._sendControl(
+					parentHash,
+					this.codec.encodeParentProbeReq(
+						ch.id.key,
+						reqId,
+						minFreeSlots,
+						reserveRootCapacity,
+					),
+					controller.signal,
+					purpose,
+				).catch(() => resolve(undefined));
+			});
+		} finally {
+			clearTimeout(timer);
+			signal.removeEventListener("abort", onAbort);
+			if (ch.pendingParentProbe.get(reqId) === pending) {
+				ch.pendingParentProbe.delete(reqId);
+			}
+			controller.abort(new AbortError("fanout probe settled"));
+		}
 	}
 
 	private async _joinLoop(
@@ -6020,6 +6209,10 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
 			if (initialJoinRemainingMs() === 0) throwInitialJoinTimeout();
 		};
 		let nextParentUpgradeCheckAt = 0;
+		let nextParentHealthProbeAt = 0;
+		let failedParentProbe:
+			| { peer: PeerStreams; route: ChannelState["routeFromRoot"] }
+			| undefined;
 		let parentUpgradeCheckSeq = 0;
 		let parentUpgradeActiveGuardBackoffMs = 0;
 		const unsuccessfulColdBootstrapPeers = new Set<string>();
@@ -6110,8 +6303,10 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
 
 				// Parent disappeared? Rejoin.
 				if (ch.parent) {
-					const parentPeer = this.peers.get(ch.parent);
+					const parentHash = ch.parent;
+					const parentPeer = this.peers.get(parentHash);
 					let connected = false;
+					let probeFailed = false;
 					if (parentPeer) {
 						try {
 							const conns = this.components.connectionManager.getConnections(
@@ -6122,8 +6317,61 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
 							connected = parentPeer.isReadable || parentPeer.isWritable;
 						}
 					}
+					const now = Date.now();
+					if (now - ch.lastParentDataAt < PARENT_IDLE_PROBE_INTERVAL_MS) {
+						failedParentProbe = undefined;
+					} else if (
+						connected &&
+						parentPeer &&
+						now >= nextParentHealthProbeAt
+					) {
+						const route = ch.routeFromRoot;
+						const lastDataAt = ch.lastParentDataAt;
+						const reply = await this.probeParentCandidate(
+							ch,
+							parentHash,
+							parentUpgrade.probe.timeoutMs,
+							signal,
+							0,
+							false,
+							"health",
+						);
+						if (
+							ch.closed ||
+							signal.aborted ||
+							this.channelsBySuffixKey.get(ch.id.suffixKey) !== ch ||
+							ch.parent !== parentHash ||
+							ch.routeFromRoot !== route ||
+							this.peers.get(parentHash) !== parentPeer
+						) {
+							failedParentProbe = undefined;
+							continue;
+						}
+						nextParentHealthProbeAt = Date.now() + PARENT_IDLE_PROBE_INTERVAL_MS;
+						// A full parent is still healthy. Do not reserve capacity or
+						// refresh lastParentDataAt: probes must not mask stalled data.
+						if (reply || ch.lastParentDataAt !== lastDataAt) {
+							failedParentProbe = undefined;
+						} else if (
+							failedParentProbe?.peer === parentPeer &&
+							failedParentProbe.route === route
+						) {
+							probeFailed = true;
+							connected = false;
+							cooldownUntilByHash.set(
+								parentHash,
+								Date.now() + candidateCooldownMs,
+							);
+						} else {
+							failedParentProbe = { peer: parentPeer, route };
+							nextParentHealthProbeAt = Date.now() + Math.max(retryMs, 1_000);
+						}
+					}
 					if (!connected) {
-						ch.metrics.reparentDisconnect += 1;
+						if (probeFailed) ch.metrics.reparentStale += 1;
+						else ch.metrics.reparentDisconnect += 1;
+						failedParentProbe = undefined;
+						nextParentHealthProbeAt = 0;
 						const hadChildren = ch.children.size > 0;
 						this.detachFromParent(ch);
 						nextParentUpgradeCheckAt = 0;
@@ -8424,11 +8672,7 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
 				const want = Math.max(1, decoded.want);
 				const ttlMs = Math.min(120_000, Math.max(1_000, decoded.ttlMs));
 				const now = Date.now();
-				let watchers = this.providerWatchersBySuffixKey.get(suffixKey);
-				if (!watchers) {
-					watchers = new Map<string, ProviderWatchRegistration>();
-					this.providerWatchersBySuffixKey.set(suffixKey, watchers);
-				}
+				const watchers = this.touchProviderWatchers(suffixKey);
 				watchers.set(fromHash, {
 					hash: fromHash,
 					want,
@@ -8566,7 +8810,14 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
 				if (!decoded) return false;
 				const { reqId, ...reply } = decoded;
 				const pending = ch.pendingParentProbe.get(reqId);
-				if (!pending) return true;
+				if (
+					!pending ||
+					pending.peer !== peerStream ||
+					pending.peer.publicKey.hashcode() !== fromHash ||
+					this.peers.get(fromHash) !== peerStream
+				) {
+					return true;
+				}
 				ch.pendingParentProbe.delete(reqId);
 				pending.resolve(reply);
 				this.touchPeerHint(ch, fromHash);
@@ -8851,7 +9102,8 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
 					void this.kickChildHashes(ch, [worstChild]).catch(() => {});
 				}
 
-				ch.children.set(fromHash, { bidPerByte });
+				const child = { bidPerByte };
+				ch.children.set(fromHash, child);
 				if (ch.isRoot && consumedParentUpgradeReservation) {
 					ch.parentUpgradeTrackerNoCapacityUntil = Math.max(
 						ch.parentUpgradeTrackerNoCapacityUntil,
@@ -8868,6 +9120,12 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
 				);
 				void (async () => {
 					await this._sendControl(fromHash, joinAccept);
+					if (ch.closed || ch.children.get(fromHash) !== child) return;
+					this.dispatchEvent(
+						new CustomEvent("fanout:child-joined", {
+							detail: { topic: ch.id.topic, root: ch.id.root, child: fromHash },
+						}),
+					);
 					if (ch.endSeqExclusive > 0) {
 						await this._sendControl(
 							fromHash,
@@ -9311,6 +9569,21 @@ export class FanoutTree extends DirectStream<FanoutTreeEvents> {
 			}
 
 			if (kind === MSG_KICK) {
+				if (this.peers.get(fromHash) !== peerStream) return true;
+				if (fromHash !== ch.parent) {
+					if (ch.parentUpgradeGrace?.previousParent === fromHash) {
+						// This attachment was evicted; never roll back to it.
+						this.clearParentUpgradeGrace(ch);
+					}
+					if (ch.parentShadow?.hash === fromHash) {
+						ch.parentShadow = undefined;
+						ch.metrics.parentShadowReset += 1;
+						if (this.parentUpgradeShadowInFlightSuffixKey === ch.id.suffixKey) {
+							this.parentUpgradeShadowInFlightSuffixKey = undefined;
+						}
+					}
+					return true;
+				}
 				ch.metrics.reparentKicked += 1;
 				ch.parent = undefined;
 				ch.level = Number.POSITIVE_INFINITY;