@optimystic/db-p2p 0.14.1 → 0.16.3

package/src/libp2p-node-base.ts

@@ -1,7 +1,7 @@
 import { createLibp2p, type Libp2p } from 'libp2p';
 import { noise } from '@chainsafe/libp2p-noise';
 import { yamux } from '@chainsafe/libp2p-yamux';
-import { identify } from '@libp2p/identify';
+import { identify, identifyPush } from '@libp2p/identify';
 import { ping } from '@libp2p/ping';
 import { dcutr } from '@libp2p/dcutr';
 import { autoNAT } from '@libp2p/autonat';
@@ -14,26 +14,31 @@ import type { ConnectionGater, PrivateKey } from '@libp2p/interface';
 import { clusterService } from './cluster/service.js';
 import { blockTransferService } from './cluster/block-transfer-service.js';
 import { repoService } from './repo/service.js';
-import { StorageRepo } from './storage/storage-repo.js';
+import { StorageRepo, withBlockCommitLatch } from './storage/storage-repo.js';
 import { BlockStorage } from './storage/block-storage.js';
 import { MemoryRawStorage } from './storage/memory-storage.js';
 import type { IRawStorage } from './storage/i-raw-storage.js';
-import { clusterMember, type ReconcileBlockCallback, type CommitCertificateSink } from './cluster/cluster-repo.js';
+import { seedOwnedBlocksFromStorage } from './owned-block-seed.js';
+import { clusterMember, type ReconcileBlockCallback, type CommitCertificateSink, type DeriveExpectedClusterCallback } from './cluster/cluster-repo.js';
+import { selectQuorumRev, selectQuorumBlock, canonicalBlockHash, type RevClaim, type BlockHashCandidate } from './cluster/quorum-restore.js';
 import { createCommitCertStore, makeClusterCommitCertExtractor, type CommitCertStore } from './cluster/commit-cert.js';
 import { coordinatorRepo } from './repo/coordinator-repo.js';
 import { Libp2pKeyPeerNetwork, type NetworkMode, type NetworkStatePersistence } from './libp2p-key-network.js';
 import { ClusterClient } from './cluster/client.js';
-import type { IRepo, ICluster, ITransactionValidator, BlockId, ActionRev, IBlock, IBlockChangeNotifier } from '@optimystic/db-core';
+import type { IRepo, ICluster, ITransactionValidator, BlockId, IBlock, IBlockChangeNotifier } from '@optimystic/db-core';
 import type { ITransactionStateStore } from './cluster/i-transaction-state-store.js';
 import { networkManagerService, type NetworkManagerService } from './network/network-manager-service.js';
 import type { SpreadOnChurnConfig, SpreadOnChurnMonitor } from './cluster/spread-on-churn.js';
+import { BlockTransferCoordinator } from './cluster/block-transfer.js';
+import type { RebalanceMonitorConfig } from './cluster/rebalance-monitor.js';
 import { fretService, Libp2pFretService } from 'p2p-fret';
 import { syncService } from './sync/service.js';
 import { SyncClient } from './sync/client.js';
 import type { SyncResponse } from './sync/protocol.js';
 import type { ClusterLatestCallback } from './repo/coordinator-repo.js';
-import { RestorationCoordinator } from './storage/restoration-coordinator-v2.js';
+import { RestorationCoordinator } from './storage/restoration-coordinator.js';
 import { RingSelector } from './storage/ring-selector.js';
+import { RingShiftCoordinator } from './storage/ring-shift-coordinator.js';
 import { StorageMonitor } from './storage/storage-monitor.js';
 import type { StorageMonitorConfig } from './storage/storage-monitor.js';
 import { ArachnodeFretAdapter } from './storage/arachnode-fret-adapter.js';
@@ -65,6 +70,7 @@ import {
 	reactivityNodePolicy,
 	createTierAddressing,
 	createRingHash,
+	DEFAULT_SUPER_MAJORITY_THRESHOLD,
 	Tier,
 	b64urlToBytes,
 	bytesToB64url,
@@ -75,18 +81,57 @@ import {
 	type NotificationVerifier,
 } from '@optimystic/db-core';
 import { PartitionDetector } from './cluster/partition-detector.js';
+import { assertSuperMajorityCoupling } from './cluster/supermajority-coupling.js';
 import { createLogger } from './logger.js';
 import { PeerReputationService } from './reputation/peer-reputation.js';
+import type { IPeerReputation } from './reputation/types.js';
+import { PenaltyReason } from './reputation/types.js';
 import { DisputeService } from './dispute/dispute-service.js';
 import { DisputeClient } from './dispute/client.js';
+import { sampleArbitrators } from './dispute/arbitrator-selection.js';
 import type { DisputeConfig } from './dispute/types.js';
 
 type Libp2pInit = NonNullable<Parameters<typeof createLibp2p>[0]>;
 export type Libp2pTransports = NonNullable<Libp2pInit['transports']>;
 
+/** A service that accepts post-construction injection of the running libp2p node. */
+interface SetLibp2pCapable {
+	setLibp2p(libp2p: Libp2p): void;
+}
+
+/** A service that accepts post-start injection of the peer-reputation view. */
+interface SetReputationCapable {
+	setReputation(reputation: IPeerReputation): void;
+}
+
+/**
+ * The custom services that receive post-assembly dependency injection. Reaching them through this
+ * typed record (rather than `(node as any).services?.fret?.setLibp2p?.(...)`) removes the silent
+ * optional-chaining skips: a call like `wired.fret.setLibp2p(node)` is checked against these
+ * interfaces at build time (a caller-side typo or wrong-arity call fails tsc), and if the service is
+ * absent at runtime the property access throws (fail-fast) instead of being quietly no-op'd. Note the
+ * config-side `services` map is itself cast (see the comment at its declaration), so a service RENAME
+ * is caught here at runtime, not by tsc, and a signature change on the real service is caught only at
+ * that service's own definition. All three services are unconditionally present in that config, so a
+ * throw here is a genuine wiring bug, not a missing service.
+ */
+type WiredServices = {
+	fret: SetLibp2pCapable;
+	networkManager: SetLibp2pCapable & SetReputationCapable;
+	repo: SetLibp2pCapable;
+};
+
 /** Logger for the reactivity node-wiring (origination/forwarder/recover/rotation composition). */
 const reactivityWiringLog = createLogger('reactivity-node-wiring');
 
+/**
+ * Logger for the best-effort in-factory service wiring. These injections run during `createLibp2p`
+ * internals against the unreliable `components.libp2p` proxy; the real node is re-injected
+ * post-construction (see the load-bearing block after `createLibp2p`), so a failure here is logged,
+ * not fatal.
+ */
+const wiringLog = createLogger('node-wiring');
+
 /** Factory function or instance for creating raw storage */
 export type RawStorageProvider = IRawStorage | (() => IRawStorage);
 
@@ -133,7 +178,7 @@ export type NodeOptions = {
 	clusterPolicy?: {
 		allowDownsize?: boolean;
 		sizeTolerance?: number; // acceptable relative difference (e.g. 0.5 = +/-50%)
-		superMajorityThreshold?: number; // fraction of peers needed for super-majority (default: 0.67)
+		superMajorityThreshold?: number; // fraction of peers needed for super-majority (default: DEFAULT_SUPER_MAJORITY_THRESHOLD = 0.75)
 	};
 
 	/** Override libp2p listen multiaddrs. */
@@ -165,6 +210,16 @@ export type NodeOptions = {
 	 */
 	spreadOnChurn?: Partial<SpreadOnChurnConfig>;
 
+	/**
+	 * Rebalance reaction tuning. Drives the RebalanceMonitor + BlockTransferCoordinator pull-gained/
+	 * push-lost path when arachnode/FRET are available (the only place fretAdapter + restoration
+	 * coordinator exist). Absent -> enabled with defaults (see RebalanceMonitorConfig). Set
+	 * { enabled: false } to disable the rebalance reaction on this node. When arachnode is disabled
+	 * or FRET is absent the rebalance path stays inert regardless of this flag (rebalance is a
+	 * resilience optimization, not a correctness requirement).
+	 */
+	rebalance?: Partial<RebalanceMonitorConfig> & { enabled?: boolean };
+
 	/** Transaction validator for cluster consensus */
 	validator?: ITransactionValidator;
 
@@ -288,6 +343,15 @@ export async function createLibp2pNodeBase(
 		new BlockStorage(blockId, rawStorage, restoreCallback)
 	);
 
+	// Per-block commit-latch runner, ready to thread into the invalidation-apply sink (`onInvalidate`)
+	// passed to `clusterMember(...)` and into each cascade `CollectionEnv`, the instant either is wired
+	// here. Sharing the `StorageRepo.commit:<blockId>` latch makes a compensating saveReplica/saveDeletion
+	// RMW of `meta.latest` mutually exclusive with a concurrent commit on the same block. It is unused
+	// today only because no `onInvalidate`/cascade driver is wired in the live node (see review handoff);
+	// it is bound here so that wiring is a one-liner and cannot reach for a divergent latch key.
+	const blockCommitLatch = withBlockCommitLatch;
+	void blockCommitLatch;
+
 	let clusterImpl: ICluster | undefined;
 	let coordinatedRepo: IRepo | undefined;
 
@@ -341,26 +405,60 @@ export async function createLibp2pNodeBase(
 		? (actionId, cert): void => { certStore.put(actionId, cert); options.onCommitCertificate?.(actionId, cert); }
 		: options.onCommitCertificate;
 
-	const libp2pOptions: unknown = {
+	const libp2pOptions: Libp2pInit = {
 		start: false,
 		privateKey: nodePrivateKey,
 		addresses: {
 			listen: listenAddrs
 		},
 		connectionManager: {
-			autoDial: true,
-			minConnections: 1,
+			// `autoDial`, `minConnections`, and `dialQueue` were stale libp2p option keys silently
+			// ignored under the former `libp2pOptions as any` (removed with this change). This libp2p
+			// version has no such keys — auto-dial is now default connection-manager behavior with no
+			// direct replacement — so they are dropped rather than re-cast. See review handoff.
 			maxConnections: 16,
-			inboundConnectionUpgradeTimeout: 10_000,
-			dialQueue: { concurrency: 2, attempts: 2 }
+			// Renamed from the stale `inboundConnectionUpgradeTimeout`. 10_000 equals this version's
+			// default, so surfacing (and correcting) the key is behavior-preserving; the old key was a no-op.
+			inboundUpgradeTimeout: 10_000
 		},
 		...(options.connectionGater ? { connectionGater: options.connectionGater } : {}),
 		transports,
 		connectionEncrypters: [noise()],
 		streamMuxers: [yamux()],
-		services: {
+		// Narrow cast confined to the `services` field: the built-in factories (identify/dcutr/…) are
+		// typed against a SECOND copy of `@libp2p/interface` pulled in transitively (via `@libp2p/crypto`),
+		// whose `Uint8Array<ArrayBuffer>` vs `<ArrayBufferLike>` PeerId/key shapes are structurally
+		// incompatible with the top-level copy — a dependency-dedup artifact, not a real mismatch. The cast
+		// stays on this field alone so the rest of `libp2pOptions` remains fully typed as `Libp2pInit`.
+		// NOTE: this cast exists ONLY because of the duplicate @libp2p/interface install; if that dedups
+		// (or on a libp2p bump) drop `as unknown as NonNullable<Libp2pInit['services']>` and type the map directly.
+		services: ({
+			// `@libp2p/identify` is the ONE service here whose protocol id it builds itself:
+			// `Identify`/`IdentifyPush` both emit `/${protocolPrefix}/id[/push]/1.0.0`, always
+			// prepending the leading slash (its own default is the BARE `'ipfs'`). So this
+			// prefix must stay slash-LESS — passing `/optimystic/...` yields the malformed
+			// double-slash `//optimystic/<net>/id/1.0.0`. Every other service below
+			// (cluster/repo/sync/blockTransfer) concatenates its own template literal and so
+			// takes the slash-PREFIXED `protocolPrefix` form; do not unify the two.
+			// Locked by `identify-protocol-id.spec.ts`.
 			identify: identify({
-				protocolPrefix: `/optimystic/${options.networkName}`
+				protocolPrefix: `optimystic/${options.networkName}`
+			}),
+			// identify/push propagates *later* address/protocol changes (relay reservation,
+			// AutoNAT-learned observed addr, a service registered post-start) to already-connected
+			// peers. Without it those peers keep the stale snapshot from the initial identify.
+			// Two consequences, both now covered by tests rather than asserted here:
+			//  - Addresses: a relay-only peer's reservation completes AFTER its first connection to
+			//    the relay, so the circuit address is exactly the one identify cannot have carried.
+			//    The relay's peerStore entry stays empty and a later dial by peer id alone fails
+			//    with NoValidAddressesError against a reachable peer — `relay-address-propagation.spec.ts`
+			//    (its gated control reproduces that failure with push removed).
+			//  - Protocols: `membershipOf` in `libp2p-key-network.ts` classifies a peer serves/
+			//    foreign/unknown purely from the peerStore protocol list, so a cluster/repo handler
+			//    registered post-start never flips an already-connected peer to `serves` —
+			//    `identify-push-propagation.spec.ts`.
+			identifyPush: identifyPush({
+				protocolPrefix: `optimystic/${options.networkName}`
 			}),
 			ping: ping(),
 			// DCUtR (hole-punch) upgrades relayed node↔node connections to direct
@@ -457,7 +555,9 @@ export async function createLibp2pNodeBase(
 					clusterSizeTolerance: options.clusterPolicy?.sizeTolerance ?? 0.5
 				});
 				const svc = svcFactory(components);
-				try { (svc as any).setLibp2p?.(components.libp2p); } catch { }
+				// Best-effort proxy-time injection; the real node is re-injected post-construction below.
+				try { (svc as SetLibp2pCapable).setLibp2p(components.libp2p); }
+				catch (err) { wiringLog('networkManager in-factory setLibp2p failed (proxy); real node injected post-construction: %o', err); }
 				return svc;
 			},
 			fret: (components: any) => {
@@ -470,26 +570,39 @@ export async function createLibp2pNodeBase(
 					bootstraps: options.bootstrapNodes ?? []
 				});
 				const svc = svcFactory(components) as Libp2pFretService;
-				try { svc.setLibp2p(components.libp2p); } catch { }
+				// Best-effort proxy-time injection; the real node is re-injected post-construction below.
+				try { (svc as SetLibp2pCapable).setLibp2p(components.libp2p); }
+				catch (err) { wiringLog('fret in-factory setLibp2p failed (proxy); real node injected post-construction: %o', err); }
 				return svc;
 			}
-		},
+
+			// [dispute-subsystem-dormant] The /optimystic/<network>/dispute/1.0.0 handler
+			// (disputeProtocolService / DisputeProtocolService) is intentionally NOT registered here.
+			// The subsystem is staged dormant pending arbitrator-set anchoring — without it, a peer
+			// minting throwaway keypairs can forge a synthetic super-majority and pass resolution.
+			// Gate: tickets/backlog/hardening/invalidation-live-wiring-requires-arbitrator-set-anchoring
+			// Wiring plan: tickets/backlog/feat-dispute-subsystem-live-activation
+		}) as unknown as NonNullable<Libp2pInit['services']>,
 		// Add bootstrap nodes as needed
 		peerDiscovery: [
 			...(options.bootstrapNodes?.length ? [bootstrap({ list: options.bootstrapNodes })] : [])
 		],
 	};
 
-	const node = await createLibp2p(libp2pOptions as any);
+	const node = await createLibp2p(libp2pOptions);
 
-	// Inject libp2p reference into services that need it before start
-	try { ((node as any).services?.fret as any)?.setLibp2p?.(node); } catch { }
-	try { ((node as any).services?.networkManager as any)?.setLibp2p?.(node); } catch { }
+	// Inject the REAL libp2p node into the services that need it, before start(). These are
+	// load-bearing and the node has NOT started yet, so any throw fails fast and rejects node
+	// creation (nothing started leaks) — far better than the service silently falling back to the
+	// unreliable `components.libp2p` proxy and surfacing later as routing/consensus failures.
+	const wired = node.services as unknown as WiredServices;
+	wired.fret.setLibp2p(node);
+	wired.networkManager.setLibp2p(node);
 	// RepoService.checkRedirect resolves the network manager / self id / connection
 	// addrs through this injected node (the components.libp2p proxy is unreliable
 	// from inside a service at request time). Done before start() so the protocol
 	// handler is live with a resolvable node from its first request.
-	try { ((node as any).services?.repo as any)?.setLibp2p?.(node); } catch { }
+	wired.repo.setLibp2p(node);
 
 	await node.start();
 
@@ -498,25 +611,42 @@ export async function createLibp2pNodeBase(
 
 	// Initialize cluster coordination components
 	const networkMode: NetworkMode = (options.bootstrapNodes?.length ?? 0) > 0 ? 'joining' : 'forming';
-	const keyNetwork = new Libp2pKeyPeerNetwork(node, options.clusterSize, undefined, networkMode, options.persistence, reputation);
-	await keyNetwork.initFromPersistedState();
+	// Network-namespaced protocol prefix, threaded into the key network so coordinator/
+	// cohort selection is scoped to peers that serve THIS network's cluster/repo protocol.
+	// A peer that only belongs to another network sharing the same physical nodes/
+	// bootstraps registers a different (network-namespaced) identify protocol, so it is
+	// never selected and can't drag this network's super-majority below quorum.
 	const protocolPrefix = `/optimystic/${options.networkName}`;
+	const keyNetwork = new Libp2pKeyPeerNetwork(node, options.clusterSize, undefined, networkMode, options.persistence, reputation, protocolPrefix);
+	await keyNetwork.initFromPersistedState();
 	const createClusterClient = (peerId: any) => ClusterClient.create(peerId, keyNetwork, protocolPrefix);
 
-	// Inject reputation into NetworkManagerService
-	try { ((node as any).services?.networkManager as any)?.setReputation?.(reputation); } catch { }
+	// Inject reputation into NetworkManagerService. Load-bearing and non-optional: the service is
+	// unconditionally present, so a throw is a real wiring bug. Unlike the pre-start injections above
+	// the node has already started here, so stop it before rethrowing rather than leaking a started
+	// node + open transports (mirrors the cohortTopic hard-fail blocks below).
+	try {
+		wired.networkManager.setReputation(reputation);
+	} catch (err) {
+		await node.stop();
+		throw err;
+	}
 
 	// Create partition detector and get FRET service
 	const partitionDetector = new PartitionDetector();
 	const fretSvc = (node as any).services?.fret as FretService | undefined;
 
 	const consensusConfig = {
-		superMajorityThreshold: options.clusterPolicy?.superMajorityThreshold ?? 0.67,
+		superMajorityThreshold: options.clusterPolicy?.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD,
 		simpleMajorityThreshold: 0.51,
 		minAbsoluteClusterSize: 2,
 		allowClusterDownsize: options.clusterPolicy?.allowDownsize ?? true,
 		clusterSizeTolerance: options.clusterPolicy?.sizeTolerance ?? 0.5,
-		partitionDetectionWindow: 60000
+		partitionDetectionWindow: 60000,
+		// Configured full cluster size — the member's own reference for "full size" in the membership
+		// admission gate (a below-full-size declared set under low FRET confidence is refused as a possible
+		// self-shrink). Matches the size threaded into the coordinator below.
+		clusterSize: options.clusterSize ?? 10
 	};
 
 	// Fetch a block archive from one cohort peer over the sync protocol, bounded by a
@@ -552,25 +682,78 @@ export async function createLibp2pNodeBase(
 		const targets = cohortPeerIds.filter(id => id !== node.peerId.toString());
 		if (targets.length === 0) return;
 
-		const archives = await Promise.all(targets.map(peerIdStr => fetchArchiveFromPeer(peerIdStr, blockId)));
+		const fetched = await Promise.all(
+			targets.map(async peerIdStr => ({ peerIdStr, archive: await fetchArchiveFromPeer(peerIdStr, blockId) }))
+		);
 
-		let best: { block: IBlock; source: ActionRev } | undefined;
-		for (const archive of archives) {
+		// Each cohort archive contributes one (rev, actionId) claim from its max
+		// revision (>= the rev we committed). Pick the target rev by quorum
+		// corroboration rather than raw Math.max — a lone peer inflating its rev
+		// cannot steer reconciliation. Keep the serving peer + block per candidate
+		// so we can then verify content agreement.
+		// NOTE: this quorum is corroboration-of-a-claim, NOT Sybil-resistant cohort
+		// membership — deferred to backlog `debt-read-repair-commit-cert-verification`.
+		const candidates: { peerIdStr: string; rev: number; actionId: string; block?: IBlock }[] = [];
+		for (const { peerIdStr, archive } of fetched) {
 			if (!archive) continue;
 			const revs = Object.keys(archive.revisions).map(Number);
 			if (revs.length === 0) continue;
 			const maxRev = Math.max(...revs);
 			if (maxRev < committed.rev) continue;
 			const data = archive.revisions[maxRev];
-			if (!data?.block) continue;
-			if (!best || maxRev > best.source.rev) {
-				best = { block: data.block, source: { actionId: data.action.actionId, rev: maxRev } };
-			}
+			if (!data?.action) continue;
+			candidates.push({ peerIdStr, rev: maxRev, actionId: data.action.actionId, block: data.block });
 		}
 
-		if (best) {
-			await storageRepo.saveReplicatedBlock(blockId, best.block, best.source);
+		const revClaims: RevClaim[] = candidates.map(c => ({ peerId: c.peerIdStr, rev: c.rev, actionId: c.actionId }));
+		const selected = selectQuorumRev(revClaims, consensusConfig.simpleMajorityThreshold);
+		if (!selected) return; // no rev corroborated by a quorum → leave block, churn/rebalance retries later
+
+		// Content agreement: among archives corroborating the chosen (rev, actionId)
+		// and actually carrying the block, the content must be byte-identical across
+		// a quorum. A cohort member serving content that hashes differently is rejected.
+		// NOTE: selectQuorumBlock recomputes its quorum over only the block-CARRYING
+		// corroborators, not the full rev-responder set. If most peers corroborate the
+		// rev but few carry block bytes (e.g. mid-prune), the content quorum can shrink
+		// to 2. Harmless with honest peers; if a colluding pair ever becomes the only
+		// block-servers for an agreed rev, that is the Sybil regime already deferred to
+		// backlog `debt-read-repair-commit-cert-verification`.
+		const corroborating = candidates.filter(c => c.rev === selected.rev && c.actionId === selected.actionId && c.block);
+		const hashCandidates: BlockHashCandidate[] = await Promise.all(
+			corroborating.map(async c => ({ peerId: c.peerIdStr, hash: await canonicalBlockHash(c.block!), block: c.block! }))
+		);
+		const agreed = selectQuorumBlock(hashCandidates, consensusConfig.simpleMajorityThreshold);
+		if (!agreed) return; // no content quorum → skip persist
+
+		// Best-effort: penalize cohort members that served content contradicting the
+		// agreed hash for the same committed (rev, actionId). Never let this throw.
+		try {
+			for (const c of hashCandidates) {
+				if (c.hash !== agreed.hash) {
+					reputation.reportPeer(c.peerId, PenaltyReason.InvalidRestoration, `reconcile:${blockId}`);
+				}
+			}
+		} catch { /* reputation write must never block restoration */ }
+
+		await storageRepo.saveReplicatedBlock(blockId, agreed.block, { actionId: selected.actionId, rev: selected.rev });
+	};
+
+	// Member-side membership derivation for the admission gate: independently re-derive this block's
+	// responsible cluster from the SAME source the coordinator uses (IKeyNetwork.findCluster), plus FRET's
+	// network-size confidence. A member gates a coordinator-declared peer set against this view before
+	// voting, so a self-shrunk minority-partition set cannot be voted into super-majority (see cluster-repo
+	// admitMembership). No FRET ⇒ confidence 0 ⇒ the gate fails closed for any downsize.
+	const deriveExpectedCluster: DeriveExpectedClusterCallback = async (blockId) => {
+		const peers = await keyNetwork.findCluster(new TextEncoder().encode(blockId));
+		let confidence = 0;
+		if (fretSvc) {
+			try {
+				confidence = fretSvc.getNetworkSizeEstimate().confidence;
+			} catch {
+				// Leave confidence 0 → fail closed for downsizing.
+			}
 		}
+		return { peers: peers ?? {}, confidence };
 	};
 
 	clusterImpl = clusterMember({
@@ -586,7 +769,8 @@ export async function createLibp2pNodeBase(
 		consensusConfig,
 		stateStore: options.transactionStateStore,
 		reconcileBlock,
-		onCommitCertificate
+		onCommitCertificate,
+		deriveExpectedCluster
 		// `recomputeArbitratorSet` (invalidation layer-2) is intentionally NOT wired here yet: a live FRET
 		// recompute needs a churn-tolerance window so it does not false-reject legitimate certificates from
 		// late-joiners (a liveness regression). Until that is tuned against live topology — and the
@@ -599,7 +783,7 @@ export async function createLibp2pNodeBase(
 		keyNetwork,
 		createClusterClient,
 		{
-			clusterSize: options.clusterSize ?? 10,
+			// clusterSize is now part of consensusConfig (member + coordinator share one reference).
 			...consensusConfig
 		},
 		fretSvc,
@@ -647,20 +831,66 @@ export async function createLibp2pNodeBase(
 		clusterLatestCallback
 	});
 
+	// Fail-fast coupling: the cluster member (what accepts a super-majority as sufficient) and the
+	// coordinator (what declares a transaction committed on that super-majority) MUST run the same
+	// threshold, or the node would come up able to disagree with itself mid-consensus. Both are fed from
+	// the single `consensusConfig` above; this asserts on their RESOLVED values so any future drift throws
+	// HERE at construction. See `assertSuperMajorityCoupling`.
+	assertSuperMajorityCoupling(
+		clusterImpl as import('./cluster/cluster-repo.js').ClusterMember,
+		coordinatedRepo as import('./repo/coordinator-repo.js').CoordinatorRepo
+	);
+
 	// Recover persisted transaction state before accepting new requests
 	if (options.transactionStateStore) {
 		await (clusterImpl as import('./cluster/cluster-repo.js').ClusterMember).recoverTransactions();
 		await (coordinatedRepo as import('./repo/coordinator-repo.js').CoordinatorRepo).recoverTransactions();
 	}
 
+	// --- Shared owned-block set for the resilience monitors ---
+	// SpreadOnChurnMonitor (sender) and RebalanceMonitor (responsibility tracker) both act on "the
+	// blocks this node physically holds". They share ONE Set so the two can never drift: a single
+	// owned-block feed populates it, and the rebalance responsibility-loss signal evicts from it
+	// (in the rebalance block below). Both monitors take this exact instance via deps.trackedBlocks.
+	const networkManager = (node as any).services?.networkManager as NetworkManagerService | undefined;
+	const ownedBlocks = new Set<string>();
+	// Single owned-block feed: every block this node commits OR receives as a replica fires
+	// storageRepo.onAnyCollectionChange. Subscribe to storageRepo DIRECTLY (not
+	// node.blockChangeNotifier): the cohort-topic activation block below may replace
+	// blockChangeNotifier with a decorating bridge, but storageRepo keeps emitting on its own
+	// surface regardless of that opt-in. NOTE: this feed does NOT re-emit blocks already durable
+	// from a previous run; those are seeded once at startup by the storage-enumeration scan wired
+	// below (seedOwnedBlocksFromStorage), so a restarted node protects on-disk data without waiting
+	// for each block to be touched again. Registered lazily the first time a
+	// monitor that reads ownedBlocks is wired, so when BOTH monitors are disabled no subscription
+	// leaks; torn down exactly once in the stop wrapper below.
+	let offOwnedBlockFeed: (() => void) | undefined;
+	const ensureOwnedBlockFeed = (): void => {
+		if (offOwnedBlockFeed) return;
+		offOwnedBlockFeed = storageRepo.onAnyCollectionChange((e) => {
+			for (const blockId of e.blockIds) ownedBlocks.add(blockId);
+		});
+	};
+	// Single owned-block-feed teardown. Registered up front (before either monitor's own stop
+	// wrapper) so it runs regardless of WHICH monitor subscribed the feed - including the
+	// spread-disabled / rebalance-only case. Idempotent: offOwnedBlockFeed is undefined-guarded.
+	{
+		const previousStop = node.stop.bind(node);
+		node.stop = async () => {
+			try {
+				offOwnedBlockFeed?.();
+			} finally {
+				await previousStop();
+			}
+		};
+	}
+
 	// --- Churn-resilient spread: drive SpreadOnChurnMonitor on a live node ---
 	// Nothing previously activated the SENDING side of the churn-resilient spread protocol on a
-	// real node. Here we init + start the monitor and feed it the blocks this node physically
-	// holds, so a debounced connection:close re-pushes them to expansion-cohort peers (the
-	// receiver durably persists each push via saveReplicatedBlock).
-	const networkManager = (node as any).services?.networkManager as NetworkManagerService | undefined;
+	// real node. Here we init + start the monitor (sharing ownedBlocks) and ensure the single
+	// owned-block feed is live, so a debounced connection:close re-pushes the node's blocks to
+	// expansion-cohort peers (the receiver durably persists each push via saveReplicatedBlock).
 	let spreadMonitor: SpreadOnChurnMonitor | undefined;
-	let offOwnedBlockFeed: (() => void) | undefined;
 	if (networkManager && (options.spreadOnChurn?.enabled ?? true) !== false) {
 		try {
 			spreadMonitor = networkManager.initSpreadOnChurnMonitor(
@@ -669,21 +899,11 @@ export async function createLibp2pNodeBase(
 				keyNetwork,
 				options.clusterSize ?? 10,
 				protocolPrefix,
+				ownedBlocks,
 				options.spreadOnChurn,
 			);
 			await spreadMonitor.start();
-			// Feed owned blocks: every block this node commits OR receives as a replica fires
-			// storageRepo.onAnyCollectionChange. Subscribe to storageRepo DIRECTLY (not
-			// node.blockChangeNotifier): the cohort-topic activation block below may replace
-			// blockChangeNotifier with a decorating bridge, but storageRepo keeps emitting on its
-			// own surface regardless of that opt-in. NOTE: blocks already durable from a previous
-			// run are NOT re-emitted on startup, so they are not tracked until next touched -
-			// acceptable here (churn re-replication re-derives over time); an initial-scan is a
-			// follow-on enhancement.
-			const monitor = spreadMonitor;
-			offOwnedBlockFeed = storageRepo.onAnyCollectionChange((e) => {
-				for (const blockId of e.blockIds) monitor.trackBlock(blockId);
-			});
+			ensureOwnedBlockFeed();
 		} catch (err) {
 			// Spread is a resilience optimization, not a correctness requirement - a wiring
 			// failure (e.g. FRET briefly unavailable) must NOT hard-fail node startup, unlike the
@@ -695,16 +915,15 @@ export async function createLibp2pNodeBase(
 	// Expose for tests/diagnostics (mirrors node.keyNetwork / node.reputation).
 	(node as any).spreadOnChurnMonitor = spreadMonitor;
 
-	// Disposal: release the owned-block subscription + stop the monitor deterministically
-	// before the transports close. Composes with the arachnode / clusterMember / cohort-topic
-	// stop wrappers (each calls its captured previousStop last). Both steps are idempotent (the
-	// unsubscribe is flag-guarded; SpreadOnChurnMonitor.stop early-returns when not running), so
-	// a double node.stop() does not throw.
+	// Disposal: stop the spread monitor deterministically before the transports close. Composes
+	// with the arachnode / clusterMember / cohort-topic stop wrappers (each calls its captured
+	// previousStop last). Idempotent (SpreadOnChurnMonitor.stop early-returns when not running), so
+	// a double node.stop() does not throw. The owned-block feed teardown is the separate up-front
+	// wrapper above (shared across both monitors).
 	{
 		const previousStop = node.stop.bind(node);
 		node.stop = async () => {
 			try {
-				offOwnedBlockFeed?.();
 				if (spreadMonitor) await spreadMonitor.stop();
 			} finally {
 				await previousStop();
@@ -719,7 +938,24 @@ export async function createLibp2pNodeBase(
 		const fret = (node as any).services?.fret as any;
 
 		if (fret) {
-			const fretAdapter = new ArachnodeFretAdapter(fret);
+			const fretAdapter = new ArachnodeFretAdapter(fret, node.peerId.toString());
+
+			// Blocks whose shed range has been RELEASED (Phase C of a ring shift, or a confirmed
+			// rebalance release). This is the GC-eligibility signal the future storage sweep
+			// (`st-storage-sweep-archival-and-capacity-estimate`) must consult: a block's local bytes may
+			// be reclaimed ONLY once it appears here, so an unconfirmed / still-served range is never
+			// swept. Populated strictly after replication is confirmed. See
+			// docs/arachnode-ring-handoff.md § Part 2 (Local bytes vs. tracking).
+			// NOTE: no sweep consumes this set yet; it is the coordinated eligibility handoff the sweep
+			// ticket will read. Until then it grows unbounded — bound it when the sweep lands.
+			const gcEligible = new Set<string>();
+			(node as any).gcEligibleBlocks = gcEligible;
+
+			// The ring-shift state machine (advertise→confirm→release). Wired inside the rebalance block
+			// below (it needs the BlockTransferCoordinator confirmer + the cohort-size floor); left
+			// undefined when the rebalance reaction is not wired, in which case ring shifts stay inert —
+			// a move-out is unsafe without the confirm/release path.
+			let ringShift: RingShiftCoordinator | undefined;
 
 			const storageMonitor = new StorageMonitor(rawStorage, options.arachnode?.storage ?? {});
 			const ringSelector = new RingSelector(fretAdapter, storageMonitor, {
@@ -727,7 +963,12 @@ export async function createLibp2pNodeBase(
 				thresholds: {
 					moveOut: 0.85,
 					moveIn: 0.40
-				}
+				},
+				// Damping so the ring decision cannot thrash near a boundary
+				// (docs/arachnode-ring-handoff.md § Part 1).
+				smoothingAlpha: 0.2,
+				deadband: 0.5,
+				minDwellMs: 10 * 60 * 1000
 			});
 
 			// Determine and announce ring membership
@@ -754,15 +995,154 @@ export async function createLibp2pNodeBase(
 			(storageRepo as any).createBlockStorage = (blockId: string) =>
 				new BlockStorage(blockId, rawStorage, newRestoreCallback);
 
-			// Monitor capacity and adjust ring periodically
+			// --- Rebalance reaction: drive RebalanceMonitor + react via BlockTransferCoordinator ---
+			// Nothing previously activated the rebalance path on a real node: initRebalanceMonitor was
+			// never called, the monitor was never start()ed, and BlockTransferCoordinator (the
+			// pull-gained / push-lost reaction primitive) was never constructed in src. This block lives
+			// inside the arachnode `if (fret)` gate because both dependencies only exist here — the
+			// fretAdapter and the RestorationCoordinator. When arachnode is disabled or FRET is absent the
+			// rebalance path stays inert (acceptable: rebalance is a resilience optimization). A wiring
+			// failure here is non-fatal (log + continue), unlike the operator-opted-in cohortTopic block.
+			if (networkManager && (options.rebalance?.enabled ?? true) !== false) {
+				try {
+					// repo → the LOCAL storageRepo (not repoProxy/coordinatedRepo): a pulled/pushed replica
+					// must land in / be read from this node's own storage, same reasoning as the
+					// blockTransfer service handler registration. protocolPrefix (/optimystic/<networkName>)
+					// MUST match the prefix the node registers its block-transfer handler under, or every
+					// lost-block push dials the wrong protocol and fails to connect.
+					const coordinator = new BlockTransferCoordinator(
+						storageRepo,
+						keyNetwork,
+						restorationCoordinatorV2,
+						partitionDetector,
+						protocolPrefix,
+					);
+
+					const rebalanceMonitor = networkManager.initRebalanceMonitor(
+						partitionDetector,
+						fretAdapter,
+						ownedBlocks,
+						options.rebalance,
+					);
+					await rebalanceMonitor.start();
+
+					// onRebalance fires synchronously from the monitor's debounced check; the coordinator's
+					// reaction (pull gained / push lost, each partition-guarded) is async, so hop it off the
+					// handler rather than blocking the monitor's emit loop. handleRebalanceEvent can REJECT
+					// (e.g. RestorationCoordinator.restore() throws while pulling a gained block) and a bare
+					// `void` would surface that as an unhandled rejection (process-fatal on Node >=15); the
+					// reaction is a resilience optimization, so swallow + log instead.
+					//
+					// ALONGSIDE dispatching to the coordinator, drive the shared owned-block set off this
+					// authoritative responsibility signal. A GAINED block is added immediately so it is
+					// tracked even before its next commit/replica touches the feed.
+					//
+					// A LOST block is NO LONGER released synchronously: doing so stopped spreading a block
+					// whose push to the new owners might fail, drop it below the replication floor, and let a
+					// later sweep reclaim it (docs/arachnode-ring-handoff.md § Why the current code violates
+					// it #2). Instead the release is GATED on confirmation — the coordinator returns the lost
+					// blocks it confirmed replicated to ≥ floor new owners, and ONLY those are untracked
+					// (authoritative eviction from the shared set — complements spread's lazy self-prune) and
+					// marked GC-eligible. A lost block whose push failed / was partition-skipped stays
+					// tracked and served, and is retried on the next rebalance.
+					//
+					// Best-effort iteration safety: this eviction can mutate ownedBlocks while
+					// SpreadOnChurnMonitor (or this monitor) is mid for...of over the same Set inside an
+					// async loop. Adding/deleting a Set entry during iteration does not throw in JS — entries
+					// are visited best-effort — which is acceptable for a resilience mechanism, so we
+					// document it here rather than add locking.
+					rebalanceMonitor.onRebalance((event) => {
+						for (const blockId of event.gained) ownedBlocks.add(blockId);
+						coordinator.handleRebalanceEvent(event).then((result) => {
+							for (const blockId of result.released) {
+								rebalanceMonitor.untrackBlock(blockId); // also evicts from the shared ownedBlocks set
+								gcEligible.add(blockId);                 // confirmed replicated → safe to sweep
+							}
+						}).catch((err) => {
+							log?.('rebalance reaction failed: %o', err);
+						});
+					});
+
+					// Ring-shift handoff (advertise→confirm→release). It needs the confirmer (this
+					// coordinator) and the cohort-size floor (this monitor), so it is wired here. The
+					// `onRelease` callback runs Phase C's local effect: stop serving/spreading the shed
+					// range and mark it GC-eligible — the same authoritative eviction the confirmed-rebalance
+					// release performs.
+					ringShift = new RingShiftCoordinator({
+						fretAdapter,
+						ringSelector,
+						fret,
+						partitionDetector,
+						confirmer: coordinator,
+						ownedBlocks,
+						selfPeerId: peerId,
+						getFloor: () => rebalanceMonitor.getCohortSize(),
+						onRelease: (blockIds) => {
+							for (const blockId of blockIds) {
+								rebalanceMonitor.untrackBlock(blockId);
+								gcEligible.add(blockId);
+							}
+						}
+					});
+					// Reconcile any stale `moving` advertisement left by a crash mid-handoff (no-op unless
+					// arachnode metadata survived a restart still marked `moving`).
+					ringShift.reconcileOnStart();
+
+					// Feed owned blocks via the SINGLE shared feed (idempotent — already live if the spread
+					// block above wired it). Both monitors read the same ownedBlocks set this populates.
+					ensureOwnedBlockFeed();
+
+					// Expose for tests/diagnostics (mirrors node.spreadOnChurnMonitor).
+					(node as any).rebalanceMonitor = rebalanceMonitor;
+					(node as any).blockTransferCoordinator = coordinator;
+					(node as any).ringShiftCoordinator = ringShift;
+
+					// Disposal: stop the monitor before transports close. Composes with the other stop
+					// wrappers (each calls its captured previousStop last). Idempotent — RebalanceMonitor.stop()
+					// early-returns when not running (NetworkManagerService.stop() also stops it). The shared
+					// owned-block feed teardown is the separate up-front wrapper (not duplicated here).
+					const previousStop = node.stop.bind(node);
+					node.stop = async () => {
+						try {
+							await rebalanceMonitor.stop();
+						} finally {
+							await previousStop();
+						}
+					};
+				} catch (err) {
+					// Rebalance is a resilience optimization, not a correctness requirement - a wiring
+					// failure (e.g. FRET briefly unavailable) must NOT hard-fail node startup.
+					log?.('rebalance wiring init failed: %o', err);
+				}
+			}
+
+			// Monitor capacity and adjust ring periodically. The damped `shouldTransition()` decides
+			// WHETHER/where to move (docs/arachnode-ring-handoff.md § Part 1); the RingShiftCoordinator
+			// carries the move out through the advertise→confirm→release handoff (§ Part 2) so a shift
+			// never drops a key below its replication floor. The old unilateral `setArachnodeInfo` flip —
+			// which changed advertised responsibility instantly with no data handoff — is gone.
+			//
+			// Ring shifts run ONLY when `ringShift` is wired (i.e. the rebalance reaction is enabled): a
+			// move-out is unsafe without the confirm/release path, so a node with the rebalance reaction
+			// disabled stays at its bootstrap ring rather than flipping unsafely.
 			const monitorInterval = setInterval(async () => {
+				if (!ringShift) return;
 				const transition = await ringSelector.shouldTransition();
-				if (transition.shouldMove) {
+				if (transition.shouldMove && transition.direction && transition.newRingDepth !== undefined) {
 					log?.('Ring transition needed: moving %s to Ring %d', transition.direction, transition.newRingDepth);
-
-					// Update Arachnode info with new ring
-					const updatedInfo = await ringSelector.createArachnodeInfo(peerId);
-					fretAdapter.setArachnodeInfo(updatedInfo);
+					try {
+						const outcome = await ringShift.executeShift({
+							direction: transition.direction,
+							newRingDepth: transition.newRingDepth
+						});
+						log?.('Ring shift outcome: %o', outcome);
+					} catch (err) {
+						log?.('Ring shift failed: %o', err);
+					} finally {
+						// Measure the minimum dwell from the SETTLED shift (completed or rolled back), not
+						// just the trigger stamped inside shouldTransition (docs/arachnode-ring-handoff.md §1.3).
+						ringSelector.recordShiftSettled();
+					}
 				}
 			}, 60_000);
 
@@ -777,6 +1157,45 @@ export async function createLibp2pNodeBase(
 		}
 	}
 
+	// --- Seed the shared owned-block set from already-durable storage ---
+	// Blocks durable from a previous run are otherwise untracked until next touched (see the
+	// onAnyCollectionChange comment above where ownedBlocks is declared). Placed here, AFTER both
+	// monitor-wiring blocks (spread ~line 862, rebalance ~line 974) have had their chance to call
+	// ensureOwnedBlockFeed():
+	//   - Gate on offOwnedBlockFeed: only seed when a monitor actually consumes ownedBlocks; if both
+	//     are disabled the set is unused and the scan (plus the background task) is wasted work.
+	//   - Feed-before-scan ordering is load-bearing: because the feed is already live, a block
+	//     committed/replicated DURING the scan is caught by the feed; Set.add is idempotent so the
+	//     overlap is harmless. Scanning before subscribing would drop a block committed in the gap.
+	//   - Fire-and-forget so a large store never blocks startup; the .catch keeps a scan rejection
+	//     from becoming an unhandled rejection.
+	//   - Cancellable: a stop wrapper flips seedStopping so the scan loop breaks against a
+	//     stopping/closing backend rather than running the enumeration to completion.
+	// NOTE: a concurrent rebalance release can untrackBlock (delete from ownedBlocks) a confirmed-
+	// released block while this scan is still running, and the scan could then re-add that id. Benign
+	// transient: the block is still in the metadata store (no sweep reclaims metadata yet), so a
+	// re-added released block is simply re-evaluated and re-released on the next rebalance tick. Right
+	// after a restart, responsibility-loss detection lags this fast metadata scan, so the window is
+	// small. Accepted rather than synchronized.
+	if (offOwnedBlockFeed && typeof rawStorage.listBlockIds === 'function') {
+		let seedStopping = false;
+		const previousStop = node.stop.bind(node);
+		node.stop = async () => {
+			seedStopping = true;
+			await previousStop();
+		};
+		void seedOwnedBlocksFromStorage(rawStorage, ownedBlocks, () => seedStopping)
+			.catch((err) => ((node as any).logger?.forComponent?.('db-p2p:owned-block-seed'))?.('seed failed: %o', err));
+	}
+
+	// [dispute-subsystem-dormant] The DisputeService object is constructed below so tests and
+	// getDisputeStatus() work, but it is unreachable from the live network path:
+	//   - No inbound handler: disputeProtocolService is NOT in the services map above.
+	//   - onInvalidation is deliberately unset: maybeInvalidate() is a no-op on live nodes.
+	//   - revalidate is deliberately unset: handleChallenge always votes inconclusive on live nodes.
+	// Full activation requires arbitrator-set anchoring before a forged synthetic cohort can pass resolution.
+	// Gate: tickets/backlog/hardening/invalidation-live-wiring-requires-arbitrator-set-anchoring
+	// Wiring plan: tickets/backlog/feat-dispute-subsystem-live-activation
 	// Initialize dispute service if enabled
 	let disputeServiceInstance: DisputeService | undefined;
 	if (options.dispute?.disputeEnabled) {
@@ -789,23 +1208,28 @@ export async function createLibp2pNodeBase(
 			reputation,
 			validator: options.validator,
 			config: options.dispute,
-			selectArbitrators: async (blockId: string, excludePeers: string[], count: number) => {
+			selectArbitrators: async (blockId: string, excludePeers: string[], count: number, round: number, epoch: Uint8Array) => {
 				const { hashKey: fretHashKey } = await import('p2p-fret');
-				const blockIdBytes = new TextEncoder().encode(blockId);
 				const fret = (node as any).services?.fret as FretService | undefined;
 				if (!fret) return [];
-				// Get a larger cohort and exclude the original cluster peers
-				const cohortSize = count + excludePeers.length + 1;
-				const hashedCoord = await fretHashKey(blockIdBytes);
-				const allPeerIdStrs = fret.assembleCohort(hashedCoord, cohortSize) as string[];
-				// Filter out original cluster peers and self, convert to PeerId
+				// Dispersed sampling: draw `count` peers from coordinates spread across the whole keyspace
+				// (hash(blockId ‖ round ‖ epoch ‖ i)) rather than the block's XOR neighborhood, so an attacker
+				// who owns the block's locale does not thereby own the arbitrators. `assembleCohort` already
+				// filters to known members; excluding the original cluster + self keeps arbitrators independent.
 				const excludeSet = new Set(excludePeers);
+				// NOTE: adding the local node's own id to `exclude` makes the draw node-relative. Cross-node
+				// determinism (the verifiable-recompute property) holds today only because the dissent
+				// coordinator running this is itself a member of the original cluster, so `self` is already in
+				// `excludePeers` — the add is a no-op and every honest node excludes the identical set. When a
+				// verify-path recompute lands, it MUST reconstruct `exclude` from the challenger's identity
+				// (`proof.challengerPeerId`) + original cluster, never the verifier's own id, or re-derivation diverges.
 				excludeSet.add(node.peerId.toString());
-				const arbitratorPeerIds = allPeerIdStrs
-					.filter(pid => !excludeSet.has(pid))
-					.slice(0, count)
-					.map(pid => peerIdFromString(pid));
-				return arbitratorPeerIds;
+				const picks = await sampleArbitrators(
+					{ blockId: new TextEncoder().encode(blockId), round, epoch, count, exclude: excludeSet },
+					(coord, wants) => fret.assembleCohort(coord, wants) as string[],
+					fretHashKey,
+				);
+				return picks.map(pid => peerIdFromString(pid));
 			},
 		});
 	}
@@ -830,6 +1254,12 @@ export async function createLibp2pNodeBase(
 	(node as any).keyNetwork = keyNetwork;
 	(node as any).reputation = reputation;
 	(node as any).disputeService = disputeServiceInstance;
+	// The node's libp2p Ed25519 identity key. Exposed on the same `(node as any).*` surface as
+	// coordinatedRepo/keyNetwork so a host can bind a client-transaction signer to it (the Quereus
+	// collection-factory's getSigner reuses this via signPeer). libp2p does not surface the private
+	// key on its public `Libp2p` interface, so this attachment is the sanctioned in-process handle.
+	// Ed25519 by construction (options.privateKey defaults to generateKeyPair('Ed25519')).
+	(node as any).peerPrivateKey = nodePrivateKey;
 
 	// --- Cohort-topic origination activation (post-node: consumes the fully-assembled node + FRET) ---
 	// This is the only place that is after the node + FRET are assembled (node.start() done, fretSvc