@optimystic/db-p2p 0.14.0 → 0.16.2

package/dist/src/libp2p-node-base.js

@@ -13,20 +13,24 @@ import { generateKeyPair } from '@libp2p/crypto/keys';
 import { clusterService } from './cluster/service.js';
 import { blockTransferService } from './cluster/block-transfer-service.js';
 import { repoService } from './repo/service.js';
-import { StorageRepo } from './storage/storage-repo.js';
+import { StorageRepo, withBlockCommitLatch } from './storage/storage-repo.js';
 import { BlockStorage } from './storage/block-storage.js';
 import { MemoryRawStorage } from './storage/memory-storage.js';
+import { seedOwnedBlocksFromStorage } from './owned-block-seed.js';
 import { clusterMember } from './cluster/cluster-repo.js';
+import { selectQuorumRev, selectQuorumBlock, canonicalBlockHash } from './cluster/quorum-restore.js';
 import { createCommitCertStore, makeClusterCommitCertExtractor } from './cluster/commit-cert.js';
 import { coordinatorRepo } from './repo/coordinator-repo.js';
 import { Libp2pKeyPeerNetwork } from './libp2p-key-network.js';
 import { ClusterClient } from './cluster/client.js';
 import { networkManagerService } from './network/network-manager-service.js';
-import { fretService } from 'p2p-fret';
+import { BlockTransferCoordinator } from './cluster/block-transfer.js';
+import { fretService, Libp2pFretService } from 'p2p-fret';
 import { syncService } from './sync/service.js';
 import { SyncClient } from './sync/client.js';
-import { RestorationCoordinator } from './storage/restoration-coordinator-v2.js';
+import { RestorationCoordinator } from './storage/restoration-coordinator.js';
 import { RingSelector } from './storage/ring-selector.js';
+import { RingShiftCoordinator } from './storage/ring-shift-coordinator.js';
 import { StorageMonitor } from './storage/storage-monitor.js';
 import { ArachnodeFretAdapter } from './storage/arachnode-fret-adapter.js';
 import { createCohortTopicHost } from './cohort-topic/host.js';
@@ -43,14 +47,24 @@ import { DEFAULT_REACTIVITY_PROTOCOLS, reactivityProtocolList } from './reactivi
 import { registerMatchmakingQueryHandler } from './matchmaking/query-transport.js';
 import { DEFAULT_MATCHMAKING_PROTOCOLS, matchmakingProtocolList } from './matchmaking/protocols.js';
 import { signPeer } from './cohort-topic/peer-sig.js';
-import { createNotificationVerifier, createCorrelationReplayGuard, createStickyCohortHintCache, reactivityNodePolicy, createTierAddressing, createRingHash, Tier, b64urlToBytes, bytesToB64url, } from '@optimystic/db-core';
+import { createNotificationVerifier, createCorrelationReplayGuard, createStickyCohortHintCache, reactivityNodePolicy, createTierAddressing, createRingHash, DEFAULT_SUPER_MAJORITY_THRESHOLD, Tier, b64urlToBytes, bytesToB64url, } from '@optimystic/db-core';
 import { PartitionDetector } from './cluster/partition-detector.js';
+import { assertSuperMajorityCoupling } from './cluster/supermajority-coupling.js';
 import { createLogger } from './logger.js';
 import { PeerReputationService } from './reputation/peer-reputation.js';
+import { PenaltyReason } from './reputation/types.js';
 import { DisputeService } from './dispute/dispute-service.js';
 import { DisputeClient } from './dispute/client.js';
+import { sampleArbitrators } from './dispute/arbitrator-selection.js';
 /** Logger for the reactivity node-wiring (origination/forwarder/recover/rotation composition). */
 const reactivityWiringLog = createLogger('reactivity-node-wiring');
+/**
+ * Logger for the best-effort in-factory service wiring. These injections run during `createLibp2p`
+ * internals against the unreliable `components.libp2p` proxy; the real node is re-injected
+ * post-construction (see the load-bearing block after `createLibp2p`), so a failure here is logged,
+ * not fatal.
+ */
+const wiringLog = createLogger('node-wiring');
 function resolveStorage(provider) {
     if (!provider) {
         return new MemoryRawStorage();
@@ -96,6 +110,14 @@ export async function createLibp2pNodeBase(options, defaults) {
     };
     // Create shared storage layers with restoration callback
     const storageRepo = new StorageRepo((blockId) => new BlockStorage(blockId, rawStorage, restoreCallback));
+    // Per-block commit-latch runner, ready to thread into the invalidation-apply sink (`onInvalidate`)
+    // passed to `clusterMember(...)` and into each cascade `CollectionEnv`, the instant either is wired
+    // here. Sharing the `StorageRepo.commit:<blockId>` latch makes a compensating saveReplica/saveDeletion
+    // RMW of `meta.latest` mutually exclusive with a concurrent commit on the same block. It is unused
+    // today only because no `onInvalidate`/cascade driver is wired in the live node (see review handoff);
+    // it is bound here so that wiring is a one-liner and cannot reach for a divergent latch key.
+    const blockCommitLatch = withBlockCommitLatch;
+    void blockCommitLatch;
     let clusterImpl;
     let coordinatedRepo;
     const clusterProxy = {
@@ -150,17 +172,27 @@ export async function createLibp2pNodeBase(options, defaults) {
             listen: listenAddrs
         },
         connectionManager: {
-            autoDial: true,
-            minConnections: 1,
+            // `autoDial`, `minConnections`, and `dialQueue` were stale libp2p option keys silently
+            // ignored under the former `libp2pOptions as any` (removed with this change). This libp2p
+            // version has no such keys — auto-dial is now default connection-manager behavior with no
+            // direct replacement — so they are dropped rather than re-cast. See review handoff.
             maxConnections: 16,
-            inboundConnectionUpgradeTimeout: 10_000,
-            dialQueue: { concurrency: 2, attempts: 2 }
+            // Renamed from the stale `inboundConnectionUpgradeTimeout`. 10_000 equals this version's
+            // default, so surfacing (and correcting) the key is behavior-preserving; the old key was a no-op.
+            inboundUpgradeTimeout: 10_000
         },
         ...(options.connectionGater ? { connectionGater: options.connectionGater } : {}),
         transports,
         connectionEncrypters: [noise()],
         streamMuxers: [yamux()],
-        services: {
+        // Narrow cast confined to the `services` field: the built-in factories (identify/dcutr/…) are
+        // typed against a SECOND copy of `@libp2p/interface` pulled in transitively (via `@libp2p/crypto`),
+        // whose `Uint8Array<ArrayBuffer>` vs `<ArrayBufferLike>` PeerId/key shapes are structurally
+        // incompatible with the top-level copy — a dependency-dedup artifact, not a real mismatch. The cast
+        // stays on this field alone so the rest of `libp2pOptions` remains fully typed as `Libp2pInit`.
+        // NOTE: this cast exists ONLY because of the duplicate @libp2p/interface install; if that dedups
+        // (or on a libp2p bump) drop `as unknown as NonNullable<Libp2pInit['services']>` and type the map directly.
+        services: ({
             identify: identify({
                 protocolPrefix: `/optimystic/${options.networkName}`
             }),
@@ -255,10 +287,13 @@ export async function createLibp2pNodeBase(options, defaults) {
                     clusterSizeTolerance: options.clusterPolicy?.sizeTolerance ?? 0.5
                 });
                 const svc = svcFactory(components);
+                // Best-effort proxy-time injection; the real node is re-injected post-construction below.
                 try {
-                    svc.setLibp2p?.(components.libp2p);
+                    svc.setLibp2p(components.libp2p);
+                }
+                catch (err) {
+                    wiringLog('networkManager in-factory setLibp2p failed (proxy); real node injected post-construction: %o', err);
                 }
-                catch { }
                 return svc;
             },
             fret: (components) => {
@@ -271,60 +306,79 @@ export async function createLibp2pNodeBase(options, defaults) {
                     bootstraps: options.bootstrapNodes ?? []
                 });
                 const svc = svcFactory(components);
+                // Best-effort proxy-time injection; the real node is re-injected post-construction below.
                 try {
                     svc.setLibp2p(components.libp2p);
                 }
-                catch { }
+                catch (err) {
+                    wiringLog('fret in-factory setLibp2p failed (proxy); real node injected post-construction: %o', err);
+                }
                 return svc;
             }
-        },
+            // [dispute-subsystem-dormant] The /optimystic/<network>/dispute/1.0.0 handler
+            // (disputeProtocolService / DisputeProtocolService) is intentionally NOT registered here.
+            // The subsystem is staged dormant pending arbitrator-set anchoring — without it, a peer
+            // minting throwaway keypairs can forge a synthetic super-majority and pass resolution.
+            // Gate: tickets/backlog/hardening/invalidation-live-wiring-requires-arbitrator-set-anchoring
+            // Wiring plan: tickets/backlog/feat-dispute-subsystem-live-activation
+        }),
         // Add bootstrap nodes as needed
         peerDiscovery: [
             ...(options.bootstrapNodes?.length ? [bootstrap({ list: options.bootstrapNodes })] : [])
         ],
     };
     const node = await createLibp2p(libp2pOptions);
-    // Inject libp2p reference into services that need it before start
-    try {
-        node.services?.fret?.setLibp2p?.(node);
-    }
-    catch { }
-    try {
-        node.services?.networkManager?.setLibp2p?.(node);
-    }
-    catch { }
+    // Inject the REAL libp2p node into the services that need it, before start(). These are
+    // load-bearing and the node has NOT started yet, so any throw fails fast and rejects node
+    // creation (nothing started leaks) — far better than the service silently falling back to the
+    // unreliable `components.libp2p` proxy and surfacing later as routing/consensus failures.
+    const wired = node.services;
+    wired.fret.setLibp2p(node);
+    wired.networkManager.setLibp2p(node);
     // RepoService.checkRedirect resolves the network manager / self id / connection
     // addrs through this injected node (the components.libp2p proxy is unreliable
     // from inside a service at request time). Done before start() so the protocol
     // handler is live with a resolvable node from its first request.
-    try {
-        node.services?.repo?.setLibp2p?.(node);
-    }
-    catch { }
+    wired.repo.setLibp2p(node);
     await node.start();
     // Initialize peer reputation service
     const reputation = new PeerReputationService();
     // Initialize cluster coordination components
     const networkMode = (options.bootstrapNodes?.length ?? 0) > 0 ? 'joining' : 'forming';
-    const keyNetwork = new Libp2pKeyPeerNetwork(node, options.clusterSize, undefined, networkMode, options.persistence, reputation);
-    await keyNetwork.initFromPersistedState();
+    // Network-namespaced protocol prefix, threaded into the key network so coordinator/
+    // cohort selection is scoped to peers that serve THIS network's cluster/repo protocol.
+    // A peer that only belongs to another network sharing the same physical nodes/
+    // bootstraps registers a different (network-namespaced) identify protocol, so it is
+    // never selected and can't drag this network's super-majority below quorum.
     const protocolPrefix = `/optimystic/${options.networkName}`;
+    const keyNetwork = new Libp2pKeyPeerNetwork(node, options.clusterSize, undefined, networkMode, options.persistence, reputation, protocolPrefix);
+    await keyNetwork.initFromPersistedState();
     const createClusterClient = (peerId) => ClusterClient.create(peerId, keyNetwork, protocolPrefix);
-    // Inject reputation into NetworkManagerService
+    // Inject reputation into NetworkManagerService. Load-bearing and non-optional: the service is
+    // unconditionally present, so a throw is a real wiring bug. Unlike the pre-start injections above
+    // the node has already started here, so stop it before rethrowing rather than leaking a started
+    // node + open transports (mirrors the cohortTopic hard-fail blocks below).
     try {
-        node.services?.networkManager?.setReputation?.(reputation);
+        wired.networkManager.setReputation(reputation);
+    }
+    catch (err) {
+        await node.stop();
+        throw err;
     }
-    catch { }
     // Create partition detector and get FRET service
     const partitionDetector = new PartitionDetector();
     const fretSvc = node.services?.fret;
     const consensusConfig = {
-        superMajorityThreshold: options.clusterPolicy?.superMajorityThreshold ?? 0.67,
+        superMajorityThreshold: options.clusterPolicy?.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD,
         simpleMajorityThreshold: 0.51,
         minAbsoluteClusterSize: 2,
         allowClusterDownsize: options.clusterPolicy?.allowDownsize ?? true,
         clusterSizeTolerance: options.clusterPolicy?.sizeTolerance ?? 0.5,
-        partitionDetectionWindow: 60000
+        partitionDetectionWindow: 60000,
+        // Configured full cluster size — the member's own reference for "full size" in the membership
+        // admission gate (a below-full-size declared set under low FRET confidence is refused as a possible
+        // self-shrink). Matches the size threaded into the coordinator below.
+        clusterSize: options.clusterSize ?? 10
     };
     // Fetch a block archive from one cohort peer over the sync protocol, bounded by a
     // per-peer timeout so an unreachable peer can't stall reconciliation. Mirrors the
@@ -361,9 +415,16 @@ export async function createLibp2pNodeBase(options, defaults) {
         const targets = cohortPeerIds.filter(id => id !== node.peerId.toString());
         if (targets.length === 0)
             return;
-        const archives = await Promise.all(targets.map(peerIdStr => fetchArchiveFromPeer(peerIdStr, blockId)));
-        let best;
-        for (const archive of archives) {
+        const fetched = await Promise.all(targets.map(async (peerIdStr) => ({ peerIdStr, archive: await fetchArchiveFromPeer(peerIdStr, blockId) })));
+        // Each cohort archive contributes one (rev, actionId) claim from its max
+        // revision (>= the rev we committed). Pick the target rev by quorum
+        // corroboration rather than raw Math.max — a lone peer inflating its rev
+        // cannot steer reconciliation. Keep the serving peer + block per candidate
+        // so we can then verify content agreement.
+        // NOTE: this quorum is corroboration-of-a-claim, NOT Sybil-resistant cohort
+        // membership — deferred to backlog `debt-read-repair-commit-cert-verification`.
+        const candidates = [];
+        for (const { peerIdStr, archive } of fetched) {
             if (!archive)
                 continue;
             const revs = Object.keys(archive.revisions).map(Number);
@@ -373,15 +434,57 @@ export async function createLibp2pNodeBase(options, defaults) {
             if (maxRev < committed.rev)
                 continue;
             const data = archive.revisions[maxRev];
-            if (!data?.block)
+            if (!data?.action)
                 continue;
-            if (!best || maxRev > best.source.rev) {
-                best = { block: data.block, source: { actionId: data.action.actionId, rev: maxRev } };
+            candidates.push({ peerIdStr, rev: maxRev, actionId: data.action.actionId, block: data.block });
+        }
+        const revClaims = candidates.map(c => ({ peerId: c.peerIdStr, rev: c.rev, actionId: c.actionId }));
+        const selected = selectQuorumRev(revClaims, consensusConfig.simpleMajorityThreshold);
+        if (!selected)
+            return; // no rev corroborated by a quorum → leave block, churn/rebalance retries later
+        // Content agreement: among archives corroborating the chosen (rev, actionId)
+        // and actually carrying the block, the content must be byte-identical across
+        // a quorum. A cohort member serving content that hashes differently is rejected.
+        // NOTE: selectQuorumBlock recomputes its quorum over only the block-CARRYING
+        // corroborators, not the full rev-responder set. If most peers corroborate the
+        // rev but few carry block bytes (e.g. mid-prune), the content quorum can shrink
+        // to 2. Harmless with honest peers; if a colluding pair ever becomes the only
+        // block-servers for an agreed rev, that is the Sybil regime already deferred to
+        // backlog `debt-read-repair-commit-cert-verification`.
+        const corroborating = candidates.filter(c => c.rev === selected.rev && c.actionId === selected.actionId && c.block);
+        const hashCandidates = await Promise.all(corroborating.map(async (c) => ({ peerId: c.peerIdStr, hash: await canonicalBlockHash(c.block), block: c.block })));
+        const agreed = selectQuorumBlock(hashCandidates, consensusConfig.simpleMajorityThreshold);
+        if (!agreed)
+            return; // no content quorum → skip persist
+        // Best-effort: penalize cohort members that served content contradicting the
+        // agreed hash for the same committed (rev, actionId). Never let this throw.
+        try {
+            for (const c of hashCandidates) {
+                if (c.hash !== agreed.hash) {
+                    reputation.reportPeer(c.peerId, PenaltyReason.InvalidRestoration, `reconcile:${blockId}`);
+                }
             }
         }
-        if (best) {
-            await storageRepo.saveReplicatedBlock(blockId, best.block, best.source);
+        catch { /* reputation write must never block restoration */ }
+        await storageRepo.saveReplicatedBlock(blockId, agreed.block, { actionId: selected.actionId, rev: selected.rev });
+    };
+    // Member-side membership derivation for the admission gate: independently re-derive this block's
+    // responsible cluster from the SAME source the coordinator uses (IKeyNetwork.findCluster), plus FRET's
+    // network-size confidence. A member gates a coordinator-declared peer set against this view before
+    // voting, so a self-shrunk minority-partition set cannot be voted into super-majority (see cluster-repo
+    // admitMembership). No FRET ⇒ confidence 0 ⇒ the gate fails closed for any downsize.
+    const deriveExpectedCluster = async (blockId) => {
+        const peers = await keyNetwork.findCluster(new TextEncoder().encode(blockId));
+        let confidence = 0;
+        if (fretSvc) {
+            try {
+                confidence = fretSvc.getNetworkSizeEstimate().confidence;
+            }
+            catch {
+                // Leave confidence 0 → fail closed for downsizing.
+            }
         }
+        return { peers: peers ?? {}, confidence };
     };
     clusterImpl = clusterMember({
         storageRepo,
@@ -396,7 +499,8 @@ export async function createLibp2pNodeBase(options, defaults) {
         consensusConfig,
         stateStore: options.transactionStateStore,
         reconcileBlock,
-        onCommitCertificate
+        onCommitCertificate,
+        deriveExpectedCluster
         // `recomputeArbitratorSet` (invalidation layer-2) is intentionally NOT wired here yet: a live FRET
         // recompute needs a churn-tolerance window so it does not false-reject legitimate certificates from
         // late-joiners (a liveness regression). Until that is tuned against live topology — and the
@@ -405,7 +509,7 @@ export async function createLibp2pNodeBase(options, defaults) {
         // `verifyInvalidationCertificate` and `tickets/plan/cohort-topic-membership-cert-trust-anchoring.md`.
     });
     const coordinatorRepoFactory = coordinatorRepo(keyNetwork, createClusterClient, {
-        clusterSize: options.clusterSize ?? 10,
+        // clusterSize is now part of consensusConfig (member + coordinator share one reference).
         ...consensusConfig
     }, fretSvc, reputation, options.transactionStateStore);
     // Create callback for querying cluster peers for their latest block revision
@@ -448,36 +552,68 @@ export async function createLibp2pNodeBase(options, defaults) {
         localPeerId: node.peerId,
         clusterLatestCallback
     });
+    // Fail-fast coupling: the cluster member (what accepts a super-majority as sufficient) and the
+    // coordinator (what declares a transaction committed on that super-majority) MUST run the same
+    // threshold, or the node would come up able to disagree with itself mid-consensus. Both are fed from
+    // the single `consensusConfig` above; this asserts on their RESOLVED values so any future drift throws
+    // HERE at construction. See `assertSuperMajorityCoupling`.
+    assertSuperMajorityCoupling(clusterImpl, coordinatedRepo);
     // Recover persisted transaction state before accepting new requests
     if (options.transactionStateStore) {
         await clusterImpl.recoverTransactions();
         await coordinatedRepo.recoverTransactions();
     }
+    // --- Shared owned-block set for the resilience monitors ---
+    // SpreadOnChurnMonitor (sender) and RebalanceMonitor (responsibility tracker) both act on "the
+    // blocks this node physically holds". They share ONE Set so the two can never drift: a single
+    // owned-block feed populates it, and the rebalance responsibility-loss signal evicts from it
+    // (in the rebalance block below). Both monitors take this exact instance via deps.trackedBlocks.
+    const networkManager = node.services?.networkManager;
+    const ownedBlocks = new Set();
+    // Single owned-block feed: every block this node commits OR receives as a replica fires
+    // storageRepo.onAnyCollectionChange. Subscribe to storageRepo DIRECTLY (not
+    // node.blockChangeNotifier): the cohort-topic activation block below may replace
+    // blockChangeNotifier with a decorating bridge, but storageRepo keeps emitting on its own
+    // surface regardless of that opt-in. NOTE: this feed does NOT re-emit blocks already durable
+    // from a previous run; those are seeded once at startup by the storage-enumeration scan wired
+    // below (seedOwnedBlocksFromStorage), so a restarted node protects on-disk data without waiting
+    // for each block to be touched again. Registered lazily the first time a
+    // monitor that reads ownedBlocks is wired, so when BOTH monitors are disabled no subscription
+    // leaks; torn down exactly once in the stop wrapper below.
+    let offOwnedBlockFeed;
+    const ensureOwnedBlockFeed = () => {
+        if (offOwnedBlockFeed)
+            return;
+        offOwnedBlockFeed = storageRepo.onAnyCollectionChange((e) => {
+            for (const blockId of e.blockIds)
+                ownedBlocks.add(blockId);
+        });
+    };
+    // Single owned-block-feed teardown. Registered up front (before either monitor's own stop
+    // wrapper) so it runs regardless of WHICH monitor subscribed the feed - including the
+    // spread-disabled / rebalance-only case. Idempotent: offOwnedBlockFeed is undefined-guarded.
+    {
+        const previousStop = node.stop.bind(node);
+        node.stop = async () => {
+            try {
+                offOwnedBlockFeed?.();
+            }
+            finally {
+                await previousStop();
+            }
+        };
+    }
     // --- Churn-resilient spread: drive SpreadOnChurnMonitor on a live node ---
     // Nothing previously activated the SENDING side of the churn-resilient spread protocol on a
-    // real node. Here we init + start the monitor and feed it the blocks this node physically
-    // holds, so a debounced connection:close re-pushes them to expansion-cohort peers (the
-    // receiver durably persists each push via saveReplicatedBlock).
-    const networkManager = node.services?.networkManager;
+    // real node. Here we init + start the monitor (sharing ownedBlocks) and ensure the single
+    // owned-block feed is live, so a debounced connection:close re-pushes the node's blocks to
+    // expansion-cohort peers (the receiver durably persists each push via saveReplicatedBlock).
     let spreadMonitor;
-    let offOwnedBlockFeed;
     if (networkManager && (options.spreadOnChurn?.enabled ?? true) !== false) {
         try {
-            spreadMonitor = networkManager.initSpreadOnChurnMonitor(partitionDetector, storageRepo, keyNetwork, options.clusterSize ?? 10, protocolPrefix, options.spreadOnChurn);
+            spreadMonitor = networkManager.initSpreadOnChurnMonitor(partitionDetector, storageRepo, keyNetwork, options.clusterSize ?? 10, protocolPrefix, ownedBlocks, options.spreadOnChurn);
             await spreadMonitor.start();
-            // Feed owned blocks: every block this node commits OR receives as a replica fires
-            // storageRepo.onAnyCollectionChange. Subscribe to storageRepo DIRECTLY (not
-            // node.blockChangeNotifier): the cohort-topic activation block below may replace
-            // blockChangeNotifier with a decorating bridge, but storageRepo keeps emitting on its
-            // own surface regardless of that opt-in. NOTE: blocks already durable from a previous
-            // run are NOT re-emitted on startup, so they are not tracked until next touched -
-            // acceptable here (churn re-replication re-derives over time); an initial-scan is a
-            // follow-on enhancement.
-            const monitor = spreadMonitor;
-            offOwnedBlockFeed = storageRepo.onAnyCollectionChange((e) => {
-                for (const blockId of e.blockIds)
-                    monitor.trackBlock(blockId);
-            });
+            ensureOwnedBlockFeed();
         }
         catch (err) {
             // Spread is a resilience optimization, not a correctness requirement - a wiring
@@ -488,16 +624,15 @@ export async function createLibp2pNodeBase(options, defaults) {
     }
     // Expose for tests/diagnostics (mirrors node.keyNetwork / node.reputation).
     node.spreadOnChurnMonitor = spreadMonitor;
-    // Disposal: release the owned-block subscription + stop the monitor deterministically
-    // before the transports close. Composes with the arachnode / clusterMember / cohort-topic
-    // stop wrappers (each calls its captured previousStop last). Both steps are idempotent (the
-    // unsubscribe is flag-guarded; SpreadOnChurnMonitor.stop early-returns when not running), so
-    // a double node.stop() does not throw.
+    // Disposal: stop the spread monitor deterministically before the transports close. Composes
+    // with the arachnode / clusterMember / cohort-topic stop wrappers (each calls its captured
+    // previousStop last). Idempotent (SpreadOnChurnMonitor.stop early-returns when not running), so
+    // a double node.stop() does not throw. The owned-block feed teardown is the separate up-front
+    // wrapper above (shared across both monitors).
     {
         const previousStop = node.stop.bind(node);
         node.stop = async () => {
             try {
-                offOwnedBlockFeed?.();
                 if (spreadMonitor)
                     await spreadMonitor.stop();
             }
@@ -512,14 +647,34 @@ export async function createLibp2pNodeBase(options, defaults) {
         const log = node.logger?.forComponent?.('db-p2p:arachnode');
         const fret = node.services?.fret;
         if (fret) {
-            const fretAdapter = new ArachnodeFretAdapter(fret);
+            const fretAdapter = new ArachnodeFretAdapter(fret, node.peerId.toString());
+            // Blocks whose shed range has been RELEASED (Phase C of a ring shift, or a confirmed
+            // rebalance release). This is the GC-eligibility signal the future storage sweep
+            // (`st-storage-sweep-archival-and-capacity-estimate`) must consult: a block's local bytes may
+            // be reclaimed ONLY once it appears here, so an unconfirmed / still-served range is never
+            // swept. Populated strictly after replication is confirmed. See
+            // docs/arachnode-ring-handoff.md § Part 2 (Local bytes vs. tracking).
+            // NOTE: no sweep consumes this set yet; it is the coordinated eligibility handoff the sweep
+            // ticket will read. Until then it grows unbounded — bound it when the sweep lands.
+            const gcEligible = new Set();
+            node.gcEligibleBlocks = gcEligible;
+            // The ring-shift state machine (advertise→confirm→release). Wired inside the rebalance block
+            // below (it needs the BlockTransferCoordinator confirmer + the cohort-size floor); left
+            // undefined when the rebalance reaction is not wired, in which case ring shifts stay inert —
+            // a move-out is unsafe without the confirm/release path.
+            let ringShift;
             const storageMonitor = new StorageMonitor(rawStorage, options.arachnode?.storage ?? {});
             const ringSelector = new RingSelector(fretAdapter, storageMonitor, {
                 minCapacity: 100 * 1024 * 1024,
                 thresholds: {
                     moveOut: 0.85,
                     moveIn: 0.40
-                }
+                },
+                // Damping so the ring decision cannot thrash near a boundary
+                // (docs/arachnode-ring-handoff.md § Part 1).
+                smoothingAlpha: 0.2,
+                deadband: 0.5,
+                minDwellMs: 10 * 60 * 1000
             });
             // Determine and announce ring membership
             const peerId = node.peerId.toString();
@@ -534,14 +689,142 @@ export async function createLibp2pNodeBase(options, defaults) {
             };
             // Replace the restore callback (this is a bit hacky, but works for now)
             storageRepo.createBlockStorage = (blockId) => new BlockStorage(blockId, rawStorage, newRestoreCallback);
-            // Monitor capacity and adjust ring periodically
+            // --- Rebalance reaction: drive RebalanceMonitor + react via BlockTransferCoordinator ---
+            // Nothing previously activated the rebalance path on a real node: initRebalanceMonitor was
+            // never called, the monitor was never start()ed, and BlockTransferCoordinator (the
+            // pull-gained / push-lost reaction primitive) was never constructed in src. This block lives
+            // inside the arachnode `if (fret)` gate because both dependencies only exist here — the
+            // fretAdapter and the RestorationCoordinator. When arachnode is disabled or FRET is absent the
+            // rebalance path stays inert (acceptable: rebalance is a resilience optimization). A wiring
+            // failure here is non-fatal (log + continue), unlike the operator-opted-in cohortTopic block.
+            if (networkManager && (options.rebalance?.enabled ?? true) !== false) {
+                try {
+                    // repo → the LOCAL storageRepo (not repoProxy/coordinatedRepo): a pulled/pushed replica
+                    // must land in / be read from this node's own storage, same reasoning as the
+                    // blockTransfer service handler registration. protocolPrefix (/optimystic/<networkName>)
+                    // MUST match the prefix the node registers its block-transfer handler under, or every
+                    // lost-block push dials the wrong protocol and fails to connect.
+                    const coordinator = new BlockTransferCoordinator(storageRepo, keyNetwork, restorationCoordinatorV2, partitionDetector, protocolPrefix);
+                    const rebalanceMonitor = networkManager.initRebalanceMonitor(partitionDetector, fretAdapter, ownedBlocks, options.rebalance);
+                    await rebalanceMonitor.start();
+                    // onRebalance fires synchronously from the monitor's debounced check; the coordinator's
+                    // reaction (pull gained / push lost, each partition-guarded) is async, so hop it off the
+                    // handler rather than blocking the monitor's emit loop. handleRebalanceEvent can REJECT
+                    // (e.g. RestorationCoordinator.restore() throws while pulling a gained block) and a bare
+                    // `void` would surface that as an unhandled rejection (process-fatal on Node >=15); the
+                    // reaction is a resilience optimization, so swallow + log instead.
+                    //
+                    // ALONGSIDE dispatching to the coordinator, drive the shared owned-block set off this
+                    // authoritative responsibility signal. A GAINED block is added immediately so it is
+                    // tracked even before its next commit/replica touches the feed.
+                    //
+                    // A LOST block is NO LONGER released synchronously: doing so stopped spreading a block
+                    // whose push to the new owners might fail, drop it below the replication floor, and let a
+                    // later sweep reclaim it (docs/arachnode-ring-handoff.md § Why the current code violates
+                    // it #2). Instead the release is GATED on confirmation — the coordinator returns the lost
+                    // blocks it confirmed replicated to ≥ floor new owners, and ONLY those are untracked
+                    // (authoritative eviction from the shared set — complements spread's lazy self-prune) and
+                    // marked GC-eligible. A lost block whose push failed / was partition-skipped stays
+                    // tracked and served, and is retried on the next rebalance.
+                    //
+                    // Best-effort iteration safety: this eviction can mutate ownedBlocks while
+                    // SpreadOnChurnMonitor (or this monitor) is mid for...of over the same Set inside an
+                    // async loop. Adding/deleting a Set entry during iteration does not throw in JS — entries
+                    // are visited best-effort — which is acceptable for a resilience mechanism, so we
+                    // document it here rather than add locking.
+                    rebalanceMonitor.onRebalance((event) => {
+                        for (const blockId of event.gained)
+                            ownedBlocks.add(blockId);
+                        coordinator.handleRebalanceEvent(event).then((result) => {
+                            for (const blockId of result.released) {
+                                rebalanceMonitor.untrackBlock(blockId); // also evicts from the shared ownedBlocks set
+                                gcEligible.add(blockId); // confirmed replicated → safe to sweep
+                            }
+                        }).catch((err) => {
+                            log?.('rebalance reaction failed: %o', err);
+                        });
+                    });
+                    // Ring-shift handoff (advertise→confirm→release). It needs the confirmer (this
+                    // coordinator) and the cohort-size floor (this monitor), so it is wired here. The
+                    // `onRelease` callback runs Phase C's local effect: stop serving/spreading the shed
+                    // range and mark it GC-eligible — the same authoritative eviction the confirmed-rebalance
+                    // release performs.
+                    ringShift = new RingShiftCoordinator({
+                        fretAdapter,
+                        ringSelector,
+                        fret,
+                        partitionDetector,
+                        confirmer: coordinator,
+                        ownedBlocks,
+                        selfPeerId: peerId,
+                        getFloor: () => rebalanceMonitor.getCohortSize(),
+                        onRelease: (blockIds) => {
+                            for (const blockId of blockIds) {
+                                rebalanceMonitor.untrackBlock(blockId);
+                                gcEligible.add(blockId);
+                            }
+                        }
+                    });
+                    // Reconcile any stale `moving` advertisement left by a crash mid-handoff (no-op unless
+                    // arachnode metadata survived a restart still marked `moving`).
+                    ringShift.reconcileOnStart();
+                    // Feed owned blocks via the SINGLE shared feed (idempotent — already live if the spread
+                    // block above wired it). Both monitors read the same ownedBlocks set this populates.
+                    ensureOwnedBlockFeed();
+                    // Expose for tests/diagnostics (mirrors node.spreadOnChurnMonitor).
+                    node.rebalanceMonitor = rebalanceMonitor;
+                    node.blockTransferCoordinator = coordinator;
+                    node.ringShiftCoordinator = ringShift;
+                    // Disposal: stop the monitor before transports close. Composes with the other stop
+                    // wrappers (each calls its captured previousStop last). Idempotent — RebalanceMonitor.stop()
+                    // early-returns when not running (NetworkManagerService.stop() also stops it). The shared
+                    // owned-block feed teardown is the separate up-front wrapper (not duplicated here).
+                    const previousStop = node.stop.bind(node);
+                    node.stop = async () => {
+                        try {
+                            await rebalanceMonitor.stop();
+                        }
+                        finally {
+                            await previousStop();
+                        }
+                    };
+                }
+                catch (err) {
+                    // Rebalance is a resilience optimization, not a correctness requirement - a wiring
+                    // failure (e.g. FRET briefly unavailable) must NOT hard-fail node startup.
+                    log?.('rebalance wiring init failed: %o', err);
+                }
+            }
+            // Monitor capacity and adjust ring periodically. The damped `shouldTransition()` decides
+            // WHETHER/where to move (docs/arachnode-ring-handoff.md § Part 1); the RingShiftCoordinator
+            // carries the move out through the advertise→confirm→release handoff (§ Part 2) so a shift
+            // never drops a key below its replication floor. The old unilateral `setArachnodeInfo` flip —
+            // which changed advertised responsibility instantly with no data handoff — is gone.
+            //
+            // Ring shifts run ONLY when `ringShift` is wired (i.e. the rebalance reaction is enabled): a
+            // move-out is unsafe without the confirm/release path, so a node with the rebalance reaction
+            // disabled stays at its bootstrap ring rather than flipping unsafely.
             const monitorInterval = setInterval(async () => {
+                if (!ringShift)
+                    return;
                 const transition = await ringSelector.shouldTransition();
-                if (transition.shouldMove) {
+                if (transition.shouldMove && transition.direction && transition.newRingDepth !== undefined) {
                     log?.('Ring transition needed: moving %s to Ring %d', transition.direction, transition.newRingDepth);
-                    // Update Arachnode info with new ring
-                    const updatedInfo = await ringSelector.createArachnodeInfo(peerId);
-                    fretAdapter.setArachnodeInfo(updatedInfo);
+                    try {
+                        const outcome = await ringShift.executeShift({
+                            direction: transition.direction,
+                            newRingDepth: transition.newRingDepth
+                        });
+                        log?.('Ring shift outcome: %o', outcome);
+                    }
+                    catch (err) {
+                        log?.('Ring shift failed: %o', err);
+                    }
+                    finally {
+                        // Measure the minimum dwell from the SETTLED shift (completed or rolled back), not
+                        // just the trigger stamped inside shouldTransition (docs/arachnode-ring-handoff.md §1.3).
+                        ringSelector.recordShiftSettled();
+                    }
                 }
             }, 60_000);
             // Cleanup on node stop
@@ -555,6 +838,44 @@ export async function createLibp2pNodeBase(options, defaults) {
             log?.('FRET service not available, Arachnode disabled');
         }
     }
+    // --- Seed the shared owned-block set from already-durable storage ---
+    // Blocks durable from a previous run are otherwise untracked until next touched (see the
+    // onAnyCollectionChange comment above where ownedBlocks is declared). Placed here, AFTER both
+    // monitor-wiring blocks (spread ~line 862, rebalance ~line 974) have had their chance to call
+    // ensureOwnedBlockFeed():
+    //   - Gate on offOwnedBlockFeed: only seed when a monitor actually consumes ownedBlocks; if both
+    //     are disabled the set is unused and the scan (plus the background task) is wasted work.
+    //   - Feed-before-scan ordering is load-bearing: because the feed is already live, a block
+    //     committed/replicated DURING the scan is caught by the feed; Set.add is idempotent so the
+    //     overlap is harmless. Scanning before subscribing would drop a block committed in the gap.
+    //   - Fire-and-forget so a large store never blocks startup; the .catch keeps a scan rejection
+    //     from becoming an unhandled rejection.
+    //   - Cancellable: a stop wrapper flips seedStopping so the scan loop breaks against a
+    //     stopping/closing backend rather than running the enumeration to completion.
+    // NOTE: a concurrent rebalance release can untrackBlock (delete from ownedBlocks) a confirmed-
+    // released block while this scan is still running, and the scan could then re-add that id. Benign
+    // transient: the block is still in the metadata store (no sweep reclaims metadata yet), so a
+    // re-added released block is simply re-evaluated and re-released on the next rebalance tick. Right
+    // after a restart, responsibility-loss detection lags this fast metadata scan, so the window is
+    // small. Accepted rather than synchronized.
+    if (offOwnedBlockFeed && typeof rawStorage.listBlockIds === 'function') {
+        let seedStopping = false;
+        const previousStop = node.stop.bind(node);
+        node.stop = async () => {
+            seedStopping = true;
+            await previousStop();
+        };
+        void seedOwnedBlocksFromStorage(rawStorage, ownedBlocks, () => seedStopping)
+            .catch((err) => (node.logger?.forComponent?.('db-p2p:owned-block-seed'))?.('seed failed: %o', err));
+    }
+    // [dispute-subsystem-dormant] The DisputeService object is constructed below so tests and
+    // getDisputeStatus() work, but it is unreachable from the live network path:
+    //   - No inbound handler: disputeProtocolService is NOT in the services map above.
+    //   - onInvalidation is deliberately unset: maybeInvalidate() is a no-op on live nodes.
+    //   - revalidate is deliberately unset: handleChallenge always votes inconclusive on live nodes.
+    // Full activation requires arbitrator-set anchoring before a forged synthetic cohort can pass resolution.
+    // Gate: tickets/backlog/hardening/invalidation-live-wiring-requires-arbitrator-set-anchoring
+    // Wiring plan: tickets/backlog/feat-dispute-subsystem-live-activation
     // Initialize dispute service if enabled
     let disputeServiceInstance;
     if (options.dispute?.disputeEnabled) {
@@ -567,24 +888,25 @@ export async function createLibp2pNodeBase(options, defaults) {
             reputation,
             validator: options.validator,
             config: options.dispute,
-            selectArbitrators: async (blockId, excludePeers, count) => {
+            selectArbitrators: async (blockId, excludePeers, count, round, epoch) => {
                 const { hashKey: fretHashKey } = await import('p2p-fret');
-                const blockIdBytes = new TextEncoder().encode(blockId);
                 const fret = node.services?.fret;
                 if (!fret)
                     return [];
-                // Get a larger cohort and exclude the original cluster peers
-                const cohortSize = count + excludePeers.length + 1;
-                const hashedCoord = await fretHashKey(blockIdBytes);
-                const allPeerIdStrs = fret.assembleCohort(hashedCoord, cohortSize);
-                // Filter out original cluster peers and self, convert to PeerId
+                // Dispersed sampling: draw `count` peers from coordinates spread across the whole keyspace
+                // (hash(blockId ‖ round ‖ epoch ‖ i)) rather than the block's XOR neighborhood, so an attacker
+                // who owns the block's locale does not thereby own the arbitrators. `assembleCohort` already
+                // filters to known members; excluding the original cluster + self keeps arbitrators independent.
                 const excludeSet = new Set(excludePeers);
+                // NOTE: adding the local node's own id to `exclude` makes the draw node-relative. Cross-node
+                // determinism (the verifiable-recompute property) holds today only because the dissent
+                // coordinator running this is itself a member of the original cluster, so `self` is already in
+                // `excludePeers` — the add is a no-op and every honest node excludes the identical set. When a
+                // verify-path recompute lands, it MUST reconstruct `exclude` from the challenger's identity
+                // (`proof.challengerPeerId`) + original cluster, never the verifier's own id, or re-derivation diverges.
                 excludeSet.add(node.peerId.toString());
-                const arbitratorPeerIds = allPeerIdStrs
-                    .filter(pid => !excludeSet.has(pid))
-                    .slice(0, count)
-                    .map(pid => peerIdFromString(pid));
-                return arbitratorPeerIds;
+                const picks = await sampleArbitrators({ blockId: new TextEncoder().encode(blockId), round, epoch, count, exclude: excludeSet }, (coord, wants) => fret.assembleCohort(coord, wants), fretHashKey);
+                return picks.map(pid => peerIdFromString(pid));
             },
         });
     }
@@ -607,6 +929,12 @@ export async function createLibp2pNodeBase(options, defaults) {
     node.keyNetwork = keyNetwork;
     node.reputation = reputation;
     node.disputeService = disputeServiceInstance;
+    // The node's libp2p Ed25519 identity key. Exposed on the same `(node as any).*` surface as
+    // coordinatedRepo/keyNetwork so a host can bind a client-transaction signer to it (the Quereus
+    // collection-factory's getSigner reuses this via signPeer). libp2p does not surface the private
+    // key on its public `Libp2p` interface, so this attachment is the sanctioned in-process handle.
+    // Ed25519 by construction (options.privateKey defaults to generateKeyPair('Ed25519')).
+    node.peerPrivateKey = nodePrivateKey;
     // --- Cohort-topic origination activation (post-node: consumes the fully-assembled node + FRET) ---
     // This is the only place that is after the node + FRET are assembled (node.start() done, fretSvc
     // available) yet before any caller can capture `blockChangeNotifier` — the Quereus collection-factory