@optimystic/db-p2p 0.16.3 → 0.18.0

package/src/cluster/cluster-policy.ts

@@ -0,0 +1,152 @@
+import { DEFAULT_SUPER_MAJORITY_THRESHOLD, type ClusterConsensusConfig } from "@optimystic/db-core";
+
+/**
+ * Resolves the operator-facing cluster knobs (`clusterSize`, `clusterPolicy.*`) into the concrete
+ * numbers the consensus and block-restoration paths run on.
+ *
+ * Extracted from `createLibp2pNodeBase` rather than left inline so the composition root's defaults
+ * are assertable without booting a libp2p node — the layer a real deployment actually uses, and
+ * therefore the layer where a default that relaxed the repair corroboration floor to a single voter
+ * survived unnoticed (see `test/cluster-policy.spec.ts`).
+ *
+ * ## Why two size yardsticks, not one
+ *
+ * One operator field — `clusterPolicy.assumedClusterSize`, "the smallest cohort this deployment can
+ * genuinely field" — feeds two consumers whose failure modes point in opposite directions, so its
+ * *default* cannot serve both:
+ *
+ * - **Membership admission gate** (`cluster/cluster-repo.ts`, `admitMembership`) reads it only on its
+ *   fallback path, when this node has no confident network-size estimate. Too small: a
+ *   partition-induced downsize slips past while the node is unconfident. Too large: the node refuses
+ *   legitimate writes — unavailability. It wants a *permissive* default, because an unconfigured
+ *   two-node mesh must still be able to transact. It gets {@link minAbsoluteClusterSize} (2).
+ * - **Repair corroboration floor** (`corroboratorCapacity` in `cluster/quorum-restore.ts`, called by
+ *   `CoordinatorRepo.queryClusterForLatest` and `createReconcileBlock`) reads it on *every* repair,
+ *   unconditionally. Too small: a shrunken — and always unauthenticated — cohort view buys a lone
+ *   peer full trust. Too large: a block stays unrepaired, degraded rather than dead. It wants a
+ *   *strict* default. It gets {@link ResolvedClusterPolicy.repairCorroborationClusterSize}, which
+ *   falls back to `clusterSize` (the configured replication factor).
+ *
+ * A single explicit `clusterPolicy.assumedClusterSize` still sets BOTH — an operator declaring their
+ * real cohort size means it for both consumers. Only the unconfigured case diverges.
+ *
+ * So a genuine two-node mesh needs exactly one setting to self-repair: either
+ * `clusterPolicy.assumedClusterSize: 2` (which does not lower the replication factor) or an honest
+ * `clusterSize: 2`. Writes and voting still work with zero configuration.
+ *
+ * ## Future
+ *
+ * Deriving the yardstick from observation (the largest peer group this node has ever seen for the
+ * key) would remove the trade entirely and subsume both values. Filed as backlog
+ * `feat-admission-floor-from-observed-cohort-high-water-mark`; do not build it here.
+ */
+
+/**
+ * Absolute floor below which no cohort is safe, whatever the size references say. Named rather than
+ * inlined because the admission gate's `assumedClusterSize` defaults to exactly this value — the two
+ * must not drift.
+ */
+export const minAbsoluteClusterSize = 2;
+
+/**
+ * The operator-facing cluster knobs. `NodeOptions` (`libp2p-node-base.ts`) intersects this rather
+ * than restating it, so a knob added here is one `resolveClusterPolicy` is guaranteed to see — a
+ * second declaration would compile fine and be silently dropped.
+ */
+export interface ClusterPolicyOptions {
+	/**
+	 * Desired cluster size per key (default 10) — the replication factor / target cohort breadth
+	 * the coordinator aims for. NOT a statement about how many peers actually exist, so the
+	 * membership admission gate is never measured against it (see `cluster/cluster-repo.ts`).
+	 *
+	 * The read-repair/reconcile corroboration floor DOES fall back to it when
+	 * `clusterPolicy.assumedClusterSize` is absent — the strict direction, so an unconfigured node
+	 * cannot have its floor talked down by a shrunken cohort view. A deployment that genuinely runs
+	 * fewer peers than this should declare `clusterPolicy.assumedClusterSize`.
+	 */
+	clusterSize?: number;
+	clusterPolicy?: {
+		allowDownsize?: boolean;
+		/** Acceptable relative difference (e.g. 0.5 = +/-50%). */
+		sizeTolerance?: number;
+		/** Fraction of peers needed for super-majority (default {@link DEFAULT_SUPER_MAJORITY_THRESHOLD}). */
+		superMajorityThreshold?: number;
+		/**
+		 * Opt in to transacting below the safe cluster-size floor when FRET has no confident
+		 * network-size estimate — the membership-admission and coordinator small-cluster gates both
+		 * fail closed without it. Default false. Turn on only for single-node / local dev meshes that
+		 * knowingly run undersized.
+		 */
+		allowUnvalidatedSmallCluster?: boolean;
+		/**
+		 * The smallest cohort this deployment can genuinely field — normally the number of nodes you
+		 * actually run, capped at `clusterSize`. Two consumers read it: the membership admission gate,
+		 * on its fallback path when the node has no confident network-size estimate; and the
+		 * read-repair/reconcile corroboration floor (`corroboratorCapacity`), unconditionally.
+		 *
+		 * Declaring it sets BOTH. Leaving it unset does NOT — see the module doc for why the two
+		 * defaults point in opposite directions. A large deployment should still set this to its real
+		 * cohort size, otherwise the admission gate cannot police a partition-induced downsize while
+		 * its size estimate is unconfident; a genuine two-node mesh needs it (or an honest
+		 * `clusterSize: 2`) to self-repair.
+		 */
+		assumedClusterSize?: number;
+	};
+}
+
+/** Everything a node's consensus + restoration paths need, with every default already applied. */
+export type ResolvedClusterPolicy = ClusterConsensusConfig & {
+	/** Replication factor / target cohort breadth. Always concrete after resolution. */
+	clusterSize: number;
+	/**
+	 * Yardstick the repair corroboration floor measures a (possibly shrunken, always unauthenticated)
+	 * cohort view against — see `corroboratorCapacity` in `cluster/quorum-restore.ts`.
+	 *
+	 * Deliberately distinct from {@link ClusterConsensusConfig.assumedClusterSize}, which the
+	 * membership admission gate reads: the two share an operator field but not a default, because
+	 * over- and under-stating them cost opposite things. See the module doc.
+	 */
+	repairCorroborationClusterSize: number;
+};
+
+/**
+ * Apply every cluster-policy default a node needs. Pure — same options in, same numbers out — so the
+ * composition root's behavior is unit-testable (`test/cluster-policy.spec.ts`).
+ */
+export function resolveClusterPolicy(options: ClusterPolicyOptions): ResolvedClusterPolicy {
+	// undefined here means "the operator said nothing", which is the only case where the two
+	// yardsticks below diverge.
+	//
+	// NOTE: a declared value is passed through unvalidated. The admission gate floors a degenerate one
+	// (0, negative, NaN, Infinity) itself — see `cluster-repo.admissionFloor` and its specs — but
+	// `corroboratorCapacity` does not: NaN there makes every quorum comparison false, so repair
+	// silently declines forever. Fail-safe, and unreachable through the reference-peer CLI (which
+	// rejects non-positive integers). If another composition root starts accepting unvalidated config,
+	// clamp here rather than in each consumer.
+	const declaredCohortSize = options.clusterPolicy?.assumedClusterSize;
+	const clusterSize = options.clusterSize ?? 10;
+
+	return {
+		superMajorityThreshold: options.clusterPolicy?.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD,
+		simpleMajorityThreshold: 0.51,
+		minAbsoluteClusterSize,
+		allowClusterDownsize: options.clusterPolicy?.allowDownsize ?? true,
+		clusterSizeTolerance: options.clusterPolicy?.sizeTolerance ?? 0.5,
+		// Fail closed by default (an undersized cluster with no confident network-size estimate is
+		// rejected); embedders running knowingly-small meshes opt in through clusterPolicy.
+		allowUnvalidatedSmallCluster: options.clusterPolicy?.allowUnvalidatedSmallCluster ?? false,
+		partitionDetectionWindow: 60000,
+		// Replication factor / target cohort breadth — what the coordinator aims for when selecting a
+		// cohort. Deliberately NOT the membership admission gate's yardstick: it says nothing about how
+		// many peers actually exist, so an unconfigured small mesh would refuse every write.
+		clusterSize,
+		// Membership admission gate, fallback path only (no confident network-size estimate). Defaults
+		// permissive so a two- or three-node mesh transacts unconfigured; the cost of that default is
+		// bounded to the gate, since the repair floor no longer reads this field.
+		assumedClusterSize: declaredCohortSize ?? minAbsoluteClusterSize,
+		// Repair corroboration floor, every repair. Defaults strict — to the replication factor — so an
+		// unconfigured node cannot have its floor talked down to a single voter by a shrunken cohort
+		// view. A genuinely small mesh declares its size (either field) to regain self-repair.
+		repairCorroborationClusterSize: declaredCohortSize ?? clusterSize
+	};
+}

package/src/cluster/cluster-repo.ts

@@ -17,6 +17,8 @@ import type { FretService } from "p2p-fret";
 import type { IPeerReputation } from "../reputation/types.js";
 import { PenaltyReason } from "../reputation/types.js";
 import type { ITransactionStateStore } from "./i-transaction-state-store.js";
+import { isMissingBaseRevisionFailure } from "../storage/storage-repo.js";
+import { RECONCILE_TIMEOUT_MS } from "./reconcile-block.js";
 
 const log = createLogger('cluster-member')
 
@@ -115,9 +117,10 @@ export type ExpectedClusterView = {
  * Independently derive this member's own view of a block's responsible cluster. Injected so
  * {@link ClusterMember} stays transport-agnostic — the composition root supplies it from
  * `IKeyNetwork.findCluster` + FRET (mirroring how the coordinator derives the cluster). Absent on nodes
- * that cannot derive a view (no FRET, unit tests): with no derived view AND no configured full-size
- * reference the gate preserves legacy approve behavior, but a configured `clusterSize` still lets the gate
- * fail closed on an unjustified downsize. See {@link ClusterMember} admission gate.
+ * that cannot derive a view (no FRET, unit tests): with no derived view AND no asserted
+ * {@link ClusterConsensusConfig.assumedClusterSize} the gate preserves legacy approve behavior, but an
+ * asserted size still lets the gate fail closed on an unjustified downsize. See {@link ClusterMember}
+ * admission gate.
  */
 export type DeriveExpectedClusterCallback = (blockId: BlockId) => Promise<ExpectedClusterView>;
 
@@ -176,7 +179,8 @@ const ExecutedTransactionTtlMs = 10 * 60 * 1000;
 // Upper bound on an awaited active reconciliation of a divergent commit. Bounds the
 // consensus path so a slow/unreachable cohort peer can't stall the cluster stream;
 // a timeout is logged and tolerated (never thrown — that would reset the stream).
-const ReconcileTimeoutMs = 5000;
+// Shared with the read path's acquisition (see RECONCILE_TIMEOUT_MS) — same operation, same bound.
+const ReconcileTimeoutMs = RECONCILE_TIMEOUT_MS;
 
 /**
  * True when a thrown storage error reports a missing pending action — i.e. this
@@ -229,8 +233,8 @@ export class ClusterMember implements ICluster {
 	private readonly minAbsoluteClusterSize: number;
 	private readonly clusterSizeTolerance: number;
 	private readonly membershipAdmissionFraction: number;
-	/** Configured full cluster size, or undefined when unknown (then the gate cannot judge a downsize). */
-	private readonly configuredClusterSize: number | undefined;
+	/** Operator-asserted smallest genuine cohort size, or undefined when unknown. */
+	private readonly assumedClusterSize: number | undefined;
 	private readonly allowUnvalidatedSmallCluster: boolean;
 
 	constructor(
@@ -256,8 +260,17 @@ export class ClusterMember implements ICluster {
 		this.minAbsoluteClusterSize = consensusConfig?.minAbsoluteClusterSize ?? 3;
 		this.clusterSizeTolerance = consensusConfig?.clusterSizeTolerance ?? 0.5;
 		this.membershipAdmissionFraction = consensusConfig?.membershipAdmissionFraction ?? 0.75;
-		this.configuredClusterSize = consensusConfig?.clusterSize;
+		this.assumedClusterSize = consensusConfig?.assumedClusterSize;
 		this.allowUnvalidatedSmallCluster = consensusConfig?.allowUnvalidatedSmallCluster ?? false;
+		// State the resolved gate parameters once, so an operator diagnosing a membership rejection can see
+		// what this node actually resolved. A fact, not a warning: `assumedClusterSize < clusterSize` is the
+		// normal default state, so warning on it would fire for every node and be ignored.
+		log('cluster-member:admission-config', {
+			assumedClusterSize: this.assumedClusterSize,
+			minAbsoluteClusterSize: this.minAbsoluteClusterSize,
+			membershipAdmissionFraction: this.membershipAdmissionFraction,
+			allowUnvalidatedSmallCluster: this.allowUnvalidatedSmallCluster
+		});
 		// Periodically clean up expired transactions (.unref() so tests/short-lived processes can exit)
 		this.expirationInterval = setInterval(() => this.queueExpiredTransactions(), 60000);
 		this.expirationInterval.unref();
@@ -855,12 +868,18 @@ export class ClusterMember implements ICluster {
 	 *     of a peer or two is absorbed, a wholesale-disjoint or half-size set is not.
 	 *
 	 * **Fail-closed posture.** When the member cannot confidently derive `E` (no capability, low FRET
-	 * confidence — exactly what a partition induces), it must refuse any *downsizing* decision: a
-	 * below-full-size `D` is rejected against the configured full `clusterSize`. With NEITHER a confident
-	 * view NOR a configured full size the gate cannot judge a downsize at all, so it preserves the legacy
-	 * approve behavior (backward-compatible for nodes/tests with no derivation wired). `allowUnvalidatedSmallCluster`
-	 * is the explicit opt-in (single-node / local dev knowingly below the safe floor), matching the
-	 * coordinator's `validateSmallCluster` semantics.
+	 * confidence — exactly what a partition induces), it must refuse any *downsizing* decision — but it
+	 * needs a size reference to judge "downsize" against, and it may NOT borrow `clusterSize` for that:
+	 * `clusterSize` is the replication factor (what a cohort should aim for), not a claim about how many
+	 * peers exist, so a small deployment configured with the default 10 would refuse every write. The
+	 * fallback yardstick is instead {@link ClusterConsensusConfig.assumedClusterSize} — the operator's own
+	 * assertion of the smallest cohort this deployment can genuinely field — run through the SAME
+	 * {@link admissionFloor} as the confident path, so the fallback can never be stricter than the measured
+	 * path (it was: it demanded the full configured size, with no fraction and no slack for churn or a peer
+	 * not yet discovered). With NEITHER a confident view NOR an asserted size the gate cannot judge a
+	 * downsize at all, so it preserves the legacy approve behavior (backward-compatible for nodes/tests with
+	 * no derivation wired). `allowUnvalidatedSmallCluster` is the explicit opt-in (single-node / local dev
+	 * knowingly below the safe floor), matching the coordinator's `validateSmallCluster` semantics.
 	 */
 	private async admitMembership(record: ClusterRecord): Promise<{ admit: boolean; reason?: string }> {
 		const ourId = this.peerId.toString();
@@ -891,30 +910,42 @@ export class ClusterMember implements ICluster {
 			&& derivedSize > 0;
 
 		if (!confident) {
-			// Fail closed for downsizing under low/absent confidence. A full-size (or larger) declared set is
-			// still admitted — there is nothing to shrink. Without a configured full-size reference we cannot
-			// tell a downsize from a legitimate small cluster, so we preserve legacy approve behavior.
-			if (this.configuredClusterSize === undefined) {
+			// Fail closed for downsizing under low/absent confidence, measured against the operator's asserted
+			// cohort size rather than the replication factor. With no asserted size the gate cannot tell a
+			// downsize from a legitimately small cluster at all, so it preserves legacy approve behavior.
+			if (this.assumedClusterSize === undefined) {
 				return { admit: true };
 			}
-			if (declared.length >= this.configuredClusterSize) {
+			const floor = this.admissionFloor(this.assumedClusterSize);
+			if (declared.length >= floor) {
 				return { admit: true };
 			}
 			log('cluster-member:admission-reject', {
 				messageHash: record.messageHash,
 				reason: 'low-confidence-downsize',
 				declaredSize: declared.length,
-				configuredClusterSize: this.configuredClusterSize,
+				floor,
+				assumedClusterSize: this.assumedClusterSize,
 				confidence: derived?.confidence
 			});
-			return { admit: false, reason: `${MEMBERSHIP_NOT_ADMITTED}:low-confidence-downsize` };
+			// The numbers ride along in the reason: this rejection is caused by *local* configuration, and
+			// without them a coordinator (or an operator reading a dispute record) has no hint which knob did it.
+			// NOTE: two honest members with different local config now emit *different* reason strings for the
+			// same record. Nothing compares reasons across peers today (`disputeEvidence.rejectReasons` is a
+			// per-peer map, and the signed payload hashes the string each vote carries); if anything ever groups
+			// or dedupes dispute reasons by string equality, group on the `membership-not-admitted:<variant>`
+			// prefix, not the whole string.
+			return {
+				admit: false,
+				reason: `${MEMBERSHIP_NOT_ADMITTED}:low-confidence-downsize (declared=${declared.length}, floor=${floor}, assumedClusterSize=${this.assumedClusterSize})`
+			};
 		}
 
 		const expected = Object.keys(derived!.peers ?? {});
 		const kEst = expected.length;
 
 		// Predicate 2: floor derived from the member's OWN confident estimate.
-		const floor = Math.max(this.minAbsoluteClusterSize, Math.ceil(this.membershipAdmissionFraction * kEst));
+		const floor = this.admissionFloor(kEst);
 		if (declared.length < floor) {
 			log('cluster-member:admission-reject', {
 				messageHash: record.messageHash,
@@ -923,7 +954,10 @@ export class ClusterMember implements ICluster {
 				floor,
 				kEst
 			});
-			return { admit: false, reason: `${MEMBERSHIP_NOT_ADMITTED}:below-floor` };
+			return {
+				admit: false,
+				reason: `${MEMBERSHIP_NOT_ADMITTED}:below-floor (declared=${declared.length}, floor=${floor}, kEst=${kEst})`
+			};
 		}
 
 		// Predicate 3: consistency with the derived view within tolerance.
@@ -944,6 +978,27 @@ export class ClusterMember implements ICluster {
 		return { admit: true };
 	}
 
+	/**
+	 * The smallest declared peer set admissible against a cohort-size reference `k`, whether `k` is
+	 * measured (the confident path's `kEst`) or asserted (`assumedClusterSize`). One function so the
+	 * fallback can never be stricter than the measured path — which it was, demanding the full configured
+	 * size with no fraction and no slack. Clamped at `minAbsoluteClusterSize`, so a degenerate `k` of 0, 1
+	 * or negative yields the absolute floor rather than a floor that admits everything. A non-finite scaled
+	 * size (a `NaN` or `Infinity` config value) is likewise treated as no usable reference rather than
+	 * propagating: an unguarded `NaN` floor fails EVERY comparison, which would silently make the node
+	 * reject every unconfident write.
+	 *
+	 * NOTE: partition safety needs `2 · membershipAdmissionFraction · superMajorityThreshold > 1` — each
+	 * side of a split must recruit `fraction · threshold · K` distinct honest members, and two sides cannot
+	 * both find them in one K-peer cluster. At the shipped defaults (0.75 · 0.67) that product is 1.005 —
+	 * true, but with almost no margin. If either default is ever lowered, re-check Theorem 2 in
+	 * `docs/correctness.md` before shipping it.
+	 */
+	private admissionFloor(k: number): number {
+		const scaled = Math.ceil(this.membershipAdmissionFraction * k);
+		return Math.max(this.minAbsoluteClusterSize, Number.isFinite(scaled) ? scaled : 0);
+	}
+
 	/**
 	 * Derive this member's own view of the record's block cluster via the injected capability, or
 	 * `undefined` when it cannot (no capability, no coordinating block id, or a derivation error — all of
@@ -997,6 +1052,20 @@ export class ClusterMember implements ICluster {
 					const blockResults = await this.storageRepo.get({ blockIds });
 					for (const blockId of blockIds) {
 						const blockResult = blockResults[blockId];
+						if (blockResult?.unavailable !== undefined) {
+							// This member cannot establish the block's revision, so it cannot judge
+							// staleness. Vote reject rather than approve on an answer it knows is a
+							// guess — approving would let a stale pend reach consensus on the strength
+							// of a member that could not check it. (Before StorageRepo caught
+							// materialization faults per block, this read threw out of the promise
+							// handler; rejecting keeps the fail-closed posture with a signed reason.)
+							log('cluster-member:validation-block-unavailable', {
+								messageHash: record.messageHash,
+								blockId,
+								reason: blockResult.unavailable
+							});
+							return { valid: false, reason: `block ${blockId} unavailable (${blockResult.unavailable}): cannot verify revision` };
+						}
 						const latestRev = blockResult?.state?.latest?.rev;
 						if (latestRev !== undefined && latestRev >= pendRequest.rev) {
 							log('cluster-member:validation-stale-revision', {
@@ -1124,9 +1193,13 @@ export class ClusterMember implements ICluster {
 	 *   - **behind**: we missed the prior `pend` cluster-transaction (cohort drift
 	 *     between the independent pend and commit phases, or transient unreachability),
 	 *     so we lack the pending action — `StorageRepo.commit` *throws* "Pending
-	 *     action … not found".
+	 *     action … not found";
+	 *   - **behind (no base)**: we DID see the pend, but we never saw the revision that
+	 *     created the block, so the transform has nothing to apply to —
+	 *     `StorageRepo.commit` returns `success:false` with a `missing-base-revision`
+	 *     reason rather than recording a revision it could not materialize.
 	 *
-	 * For the **behind** case we hold no revision of the committed blocks at all, so we
+	 * For both **behind** cases we hold no usable revision of the committed blocks, so we
 	 * actively reconcile: pull the committed revision from a cohort peer that holds it
 	 * (`reconcileBlock`) and restore it locally. Lazy read-repair on a later read cannot
 	 * recover it on its own when cohort drift has left the block under-replicated (no
@@ -1215,7 +1288,8 @@ export class ClusterMember implements ICluster {
 			}
 			if (!result.success) {
 				// success:false is a StaleFailure. `missing` ⇒ ahead/stale divergence
-				// (we already hold ≥ this rev): tolerate, do NOT reconcile downward. A bare
+				// (we already hold ≥ this rev): tolerate, do NOT reconcile downward. A
+				// missing-base reason ⇒ behind divergence, reconcile (below). Any other bare
 				// `reason` with no `missing` ⇒ a genuine internalCommit fault: propagate so
 				// handleConsensus rolls back the executed marker and rethrows.
 				if (result.missing?.length) {
@@ -1228,6 +1302,22 @@ export class ClusterMember implements ICluster {
 					});
 					return;
 				}
+				// This member holds no materializable base for one of the blocks, so
+				// `StorageRepo.commit` REFUSED rather than record a revision it could never serve.
+				// Same "behind" divergence as a missing pend — and the same cure: pull the committed
+				// revision from a cohort peer. Reconciling here (after commit released its per-block
+				// latches) is what makes refusing safe; fetching inside the commit path would deadlock
+				// against the latch `saveReplicatedBlock` needs to persist what it fetched.
+				if (isMissingBaseRevisionFailure(result)) {
+					log('cluster-member:consensus-commit-diverged', {
+						messageHash,
+						actionId: commit.actionId,
+						divergence: 'behind',
+						reason: result.reason
+					});
+					await this.reconcileDivergentCommit(record, commit);
+					return;
+				}
 				throw new Error(`Consensus commit for action ${commit.actionId} failed: ${result.reason ?? 'unknown reason'}`);
 			}
 			return;
@@ -1371,7 +1461,12 @@ export class ClusterMember implements ICluster {
 	private async reconcileOneBlock(messageHash: string, blockId: BlockId, committed: ActionRev, cohortPeerIds: string[]): Promise<void> {
 		try {
 			await this.withReconcileTimeout(this.reconcileBlock!(blockId, committed, cohortPeerIds), blockId);
-			log('cluster-member:consensus-commit-reconciled', { messageHash, blockId, rev: committed.rev });
+			// "attempted", not "reconciled": the callback returns void, and a quorum decline is a
+			// normal, non-throwing outcome — so reaching here means the pass ran to completion, NOT
+			// that anything was restored. `reconcile:restored` (reconcile-block.ts) is the line that
+			// says the bytes actually landed; `reconcile:no-rev-quorum` / `reconcile:no-content-quorum`
+			// say they did not.
+			log('cluster-member:consensus-commit-reconcile-attempted', { messageHash, blockId, rev: committed.rev });
 		} catch (err) {
 			log('cluster-member:consensus-commit-reconcile-failed', {
 				messageHash,

package/src/cluster/reconcile-block.ts

@@ -0,0 +1,191 @@
+import type { ActionRev, BlockId, IBlock } from "@optimystic/db-core";
+import type { BlockArchive } from "../storage/struct.js";
+import type { ReconcileBlockCallback } from "./cluster-repo.js";
+import type { IPeerReputation } from "../reputation/types.js";
+import { PenaltyReason } from "../reputation/types.js";
+import {
+	selectQuorumRev, selectQuorumBlock, canonicalBlockHash, corroboratorCapacity,
+	type RevClaim, type BlockHashCandidate, type QuorumRev
+} from "./quorum-restore.js";
+import { createLogger } from '../logger.js';
+
+const log = createLogger('reconcile-block');
+
+/**
+ * Wall-clock bound on one whole reconcile pass (all cohort peers, both quorums, the persist).
+ * Shared by both callers so a slow or unreachable cohort peer stalls neither the commit path
+ * (`ClusterMember.withReconcileTimeout` — a stall there holds up consensus execution) nor the read
+ * path (`CoordinatorRepo.restoreCorroborated` — a stall there holds up a caller's `get`).
+ */
+export const RECONCILE_TIMEOUT_MS = 5000;
+
+/** One cohort peer's answer for a block: its highest revision, and the block bytes if it carried them. */
+interface ReconcileCandidate {
+	peerId: string;
+	rev: number;
+	actionId: string;
+	/** Present only when the serving archive carried a materialized block for `rev`. */
+	block?: IBlock;
+}
+
+/** Collaborators {@link createReconcileBlock} needs, injected so the logic stays transport-agnostic. */
+export interface ReconcileBlockDeps {
+	/** This node's own peer id; excluded from the cohort targets. */
+	selfPeerId: string;
+	/** Fetch one cohort peer's archive for `blockId` — `undefined` when it is unreachable or holds nothing. */
+	fetchArchive: (peerId: string, blockId: BlockId) => Promise<BlockArchive | undefined>;
+	/** Persist the agreed content through the churn-replication funnel. */
+	saveReplicatedBlock: (blockId: BlockId, block: IBlock, source: ActionRev) => Promise<void>;
+	/** Proportional corroboration threshold; the cohort's `simpleMajorityThreshold`. */
+	simpleMajorityThreshold: number;
+	/**
+	 * Yardstick the corroboration floor is measured against — the floor for
+	 * {@link corroboratorCapacity}. Required, not optional: unlike the membership admission gate there
+	 * is no "unknown" handling here, so a caller that cannot state an asserted cohort size should pass
+	 * its configured `clusterSize` (the strict direction) rather than a small placeholder. The failure
+	 * mode of overstating it is a block that stays unrepaired — degraded, not dead; of understating it,
+	 * a shrunken cohort view that can relax the floor to a single voter. `resolveClusterPolicy`
+	 * (`cluster/cluster-policy.ts`) resolves it for a real node and defaults it to `clusterSize`.
+	 */
+	repairCorroborationClusterSize: number;
+	/** Best-effort misbehavior reporting; a throwing implementation is swallowed. */
+	reputation?: Pick<IPeerReputation, 'reportPeer'>;
+}
+
+/**
+ * Highest revision an archive covers. Keys arrive as strings off the wire from an untrusted peer,
+ * so a non-numeric one is skipped rather than poisoning the maximum with `NaN`; folding instead of
+ * `Math.max(...keys)` also keeps a wide archive off the argument-count limit.
+ */
+function maxRevision(revisions: BlockArchive['revisions']): number | undefined {
+	let max: number | undefined;
+	for (const key of Object.keys(revisions)) {
+		const rev = Number(key);
+		if (Number.isFinite(rev) && (max === undefined || rev > max)) max = rev;
+	}
+	return max;
+}
+
+/**
+ * The claim a peer's archive makes: its highest revision, provided that revision is at least the
+ * one we committed. `undefined` when the peer served nothing usable (unreachable, empty archive,
+ * or only revisions older than the commit we are healing).
+ */
+function toCandidate(peerId: string, archive: BlockArchive | undefined, committedRev: number): ReconcileCandidate | undefined {
+	if (!archive) return undefined;
+	const maxRev = maxRevision(archive.revisions);
+	if (maxRev === undefined || maxRev < committedRev) return undefined;
+	const data = archive.revisions[maxRev];
+	if (!data?.action) return undefined;
+	return { peerId, rev: maxRev, actionId: data.action.actionId, block: data.block };
+}
+
+/**
+ * One peer's answer, isolated. `fetchArchive` is contracted to answer `undefined` for an
+ * unreachable peer, but a raw `Promise.all` over the cohort would let a single rejecting fetch
+ * discard the answers every other peer already gave — turning a heal the cohort could complete
+ * into a decline. One peer's failure costs only that peer's vote.
+ */
+async function fetchCandidate(
+	deps: ReconcileBlockDeps,
+	peerId: string,
+	blockId: BlockId,
+	committedRev: number
+): Promise<ReconcileCandidate | undefined> {
+	try {
+		return toCandidate(peerId, await deps.fetchArchive(peerId, blockId), committedRev);
+	} catch (err) {
+		log('reconcile:fetch-error', { blockId, peerId, error: (err as Error).message });
+		return undefined;
+	}
+}
+
+/**
+ * Hash the block bytes of every candidate that both corroborates `selected` and actually carried content.
+ *
+ * NOTE: this canonical-JSON-serializes and sha256s every carrier's whole block on every reconcile.
+ * Negligible at today's cohort widths and block sizes; if blocks grow large or cohorts wide enough
+ * for this to show up on a commit-path profile, hash incrementally at receive time instead.
+ */
+async function hashCarriers(candidates: ReconcileCandidate[], selected: QuorumRev): Promise<BlockHashCandidate[]> {
+	const carriers = candidates.filter(c => c.rev === selected.rev && c.actionId === selected.actionId && c.block);
+	return await Promise.all(
+		carriers.map(async c => ({ peerId: c.peerId, hash: await canonicalBlockHash(c.block!), block: c.block! }))
+	);
+}
+
+/** Report cohort members that served content contradicting the agreed hash. Best-effort; never throws. */
+function penalizeContradictingContent(
+	reputation: Pick<IPeerReputation, 'reportPeer'> | undefined,
+	candidates: BlockHashCandidate[],
+	agreedHash: string,
+	blockId: BlockId
+): void {
+	if (!reputation) return;
+	try {
+		for (const c of candidates) {
+			if (c.hash !== agreedHash) {
+				reputation.reportPeer(c.peerId, PenaltyReason.InvalidRestoration, `reconcile:${blockId}`);
+			}
+		}
+	} catch (err) {
+		log('reconcile:penalize-error', { blockId, error: (err as Error).message });
+	}
+}
+
+/**
+ * Active reconciliation for a block this member committed without a materializable base
+ * (cohort drift between the independent pend and commit cluster-transactions, or a refused
+ * `missing-base-revision` commit). Queries the commit cohort — self already excluded by
+ * `ClusterMember.reconcileDivergentCommit` — for the block, picks the target revision by quorum
+ * corroboration rather than raw `Math.max` (a lone peer inflating its rev cannot steer
+ * reconciliation), verifies the cohort agrees on the *content* at that revision, and persists it.
+ *
+ * Both quorums are capped by {@link corroboratorCapacity}: demanding two corroborators from a
+ * cohort that contains exactly one other peer is a permanent deadlock, not a safety property —
+ * the node can never heal and stays unreadable forever.
+ *
+ * **Exposure at capacity 1 (documented, not accidental).** Block ids are random 256-bit strings
+ * (`db-core` `structs.ts`), NOT content-addressed, so nothing on the receive path can re-derive
+ * the id from the bytes: `canonicalBlockHash` is a cross-peer *agreement* hash, never a check
+ * against `blockId`. A sole cohort peer's content is therefore believed on its word. That adds no
+ * trust the cohort had not already extended — the same peer's `(rev, actionId)` claim is likewise
+ * uncorroborable at that size (see `selectQuorumRev`'s capacity note), and a two-member cohort has
+ * no honest majority to appeal to in the first place. Closing it needs commit-cert verification,
+ * tracked by backlog `debt-read-repair-commit-cert-verification`.
+ *
+ * Declines are cheap and retryable: nothing is persisted, nothing is marked, and the next commit
+ * or churn/rebalance pass tries again.
+ */
+export function createReconcileBlock(deps: ReconcileBlockDeps): ReconcileBlockCallback {
+	return async (blockId, committed, cohortPeerIds) => {
+		const targets = cohortPeerIds.filter(id => id !== deps.selfPeerId);
+		if (targets.length === 0) return;
+
+		const fetched = await Promise.all(
+			targets.map(peerId => fetchCandidate(deps, peerId, blockId, committed.rev))
+		);
+		const candidates = fetched.filter((c): c is ReconcileCandidate => c !== undefined);
+		const capacity = corroboratorCapacity(targets.length, deps.repairCorroborationClusterSize);
+
+		const revClaims: RevClaim[] = candidates.map(({ peerId, rev, actionId }) => ({ peerId, rev, actionId }));
+		const selected = selectQuorumRev(revClaims, deps.simpleMajorityThreshold, capacity);
+		if (!selected) {
+			// Leave the block behind; churn/rebalance and the next commit retry.
+			log('reconcile:no-rev-quorum', { blockId, rev: committed.rev, responders: revClaims.length, capacity });
+			return;
+		}
+
+		const hashCandidates = await hashCarriers(candidates, selected);
+		const agreed = selectQuorumBlock(hashCandidates, deps.simpleMajorityThreshold, capacity);
+		if (!agreed) {
+			log('reconcile:no-content-quorum', { blockId, rev: selected.rev, carriers: hashCandidates.length, capacity });
+			return;
+		}
+
+		penalizeContradictingContent(deps.reputation, hashCandidates, agreed.hash, blockId);
+
+		await deps.saveReplicatedBlock(blockId, agreed.block, { actionId: selected.actionId, rev: selected.rev });
+		log('reconcile:restored', { blockId, rev: selected.rev, actionId: selected.actionId });
+	};
+}