@optimystic/db-p2p 0.14.0 → 0.16.2

package/src/cluster/peer-key-binding.ts

@@ -0,0 +1,37 @@
+/**
+ * Key ↔ peer-id binding predicate for the cluster/dispute vote paths.
+ *
+ * For a libp2p Ed25519 identity the peer-id **is** the multihash of the public key, so a public key
+ * that a record self-asserts for a peer id can be checked against the id itself — no network lookup.
+ * Both the cluster two-phase-commit path (`cluster-repo.ts`) and the dispute path (`dispute-service.ts`)
+ * read a signing key straight from the record they are handed; without this check a coordinator could
+ * attach a key it controls under an honest peer's id and have a forged vote verify. This predicate is
+ * the gate that ties a self-asserted key back to the identity it claims.
+ *
+ * Mirrors the binding logic already used by the cohort-topic substrate in `cohort-topic/peer-sig.ts`
+ * (`peerIdFromString(str).publicKey.raw`); intentionally NOT importing that module, which is
+ * specialized for its own wire encoding.
+ *
+ * NOTE: this proves only that a vote attributed to peer id `X` was signed by the key `X` names. It does
+ * NOT decide whether `X` is legitimately a cohort member — a coordinator that mints fresh keypairs and
+ * uses each key's own derived id passes this check for every one of them. Sybil/membership is a separate
+ * layer (cohort-topic membership certificates); do not assume it is solved here.
+ */
+
+import { peerIdFromString } from "@libp2p/peer-id";
+
+/**
+ * True iff `rawKey` is the Ed25519 public key that `peerIdStr` names. Total: returns `false` on a
+ * non-Ed25519 id, a missing/mismatched key, or any malformed input — never throws.
+ */
+export function peerIdBindsPublicKey(peerIdStr: string, rawKey: Uint8Array): boolean {
+	try {
+		const peerId = peerIdFromString(peerIdStr);
+		if (peerId.type !== 'Ed25519' || peerId.publicKey === undefined) return false;
+		const expected = peerId.publicKey.raw; // 32 bytes for Ed25519
+		if (expected.length !== rawKey.length) return false;
+		return expected.every((b, i) => b === rawKey[i]);
+	} catch {
+		return false;
+	}
+}

package/src/cluster/rebalance-monitor.ts

@@ -15,6 +15,13 @@ export interface RebalanceEvent {
 	lost: string[]
 	/** Peers that are now closer for the lost blocks: blockId → peerId[] */
 	newOwners: Map<string, string[]>
+	/**
+	 * Replication floor `N` for this event — the cohort size FRET assembled at check time
+	 * ({@link RebalanceMonitor.getCohortSize}). The reaction gates release of a `lost` block on
+	 * confirming it replicated to this many new owners, so a lost block is never released below the
+	 * floor. See `docs/arachnode-ring-handoff.md` § Part 2.
+	 */
+	floor: number
 	/** Timestamp of the topology change that triggered this */
 	triggeredAt: number
 }
@@ -33,13 +40,27 @@ export interface RebalanceMonitorDeps {
 	fret: FretService
 	partitionDetector: PartitionDetector
 	fretAdapter: ArachnodeFretAdapter
+	/**
+	 * The owned-block tracked set. When provided (e.g. the shared `ownedBlocks` set wired in
+	 * `libp2p-node-base`), the monitor references this exact `Set` instead of constructing its own,
+	 * so it stays in lock-step with the `SpreadOnChurnMonitor` that shares it. Omit for standalone
+	 * construction (unit tests) — a fresh private `Set` preserves all existing behavior. Note: only
+	 * `trackedBlocks` is shared; `responsibilitySnapshot` stays per-monitor (it is rebalance's own
+	 * was-responsible memory, not owned-block tracking).
+	 */
+	trackedBlocks?: Set<string>
 }
 
 type RebalanceHandler = (event: RebalanceEvent) => void
 
 export class RebalanceMonitor implements Startable {
 	private running = false
-	private readonly trackedBlocks = new Set<string>()
+	private readonly trackedBlocks: Set<string>
+	// Per-monitor was-responsible memory (NOT shared, unlike trackedBlocks). When the shared
+	// trackedBlocks set is mutated externally — spread's no-local-data self-prune, or the node's
+	// responsibility-loss eviction going through untrackBlock — a snapshot entry for a since-removed
+	// block may linger here. That is acceptable: performRebalanceCheck only iterates trackedBlocks, so
+	// a lingering entry is inert; if the block is later re-fed, its responsibility is simply re-derived.
 	private readonly responsibilitySnapshot = new Map<string, boolean>()
 	private readonly handlers: RebalanceHandler[] = []
 	private debounceTimer: ReturnType<typeof setTimeout> | null = null
@@ -58,6 +79,10 @@ export class RebalanceMonitor implements Startable {
 		private readonly deps: RebalanceMonitorDeps,
 		config: RebalanceMonitorConfig = {}
 	) {
+		// Share the injected owned-block set when present (so spread + rebalance never drift);
+		// otherwise own a private set (standalone construction / unit tests). Only trackedBlocks is
+		// shared — responsibilitySnapshot stays per-monitor.
+		this.trackedBlocks = deps.trackedBlocks ?? new Set<string>()
 		this.debounceMs = config.debounceMs ?? 5000
 		this.minRebalanceIntervalMs = config.minRebalanceIntervalMs ?? 60000
 		this.suppressDuringPartition = config.suppressDuringPartition ?? true
@@ -192,10 +217,16 @@ export class RebalanceMonitor implements Startable {
 
 		log('rebalance check: gained=%d lost=%d', gained.length, lost.length)
 
-		return { gained, lost, newOwners, triggeredAt }
+		return { gained, lost, newOwners, floor: this.getCohortSize(), triggeredAt }
 	}
 
-	private getCohortSize(): number {
+	/**
+	 * The replication floor `N` — the cohort size FRET assembles for a block. Public so the ring-shift
+	 * handoff and the rebalance reaction can gate release on confirming replication to this many
+	 * holders (`docs/arachnode-ring-handoff.md` § Replication floor). Derives from FRET's network-size
+	 * estimate: `clamp(ceil(sqrt(n_est)), 1, 3)`, defaulting to 3 when no confident estimate exists.
+	 */
+	getCohortSize(): number {
 		const diag: any = (this.deps.fret as any).getDiagnostics?.()
 		const estimate = diag?.estimate ?? diag?.n
 		if (typeof estimate === 'number' && Number.isFinite(estimate) && estimate > 0) {

package/src/cluster/service.ts

@@ -5,6 +5,7 @@ import type { Startable, Logger, Stream, Connection, StreamHandler, PeerId } fro
 import type { ICluster, ClusterRecord } from '@optimystic/db-core';
 import { encodePeers, type RedirectPayload } from '../repo/redirect.js';
 import { toClusterErrorEnvelope } from './cluster-error.js';
+import { MAX_CONTROL_MESSAGE_BYTES } from '../protocol-limits.js';
 import type { Uint8ArrayList } from 'uint8arraylist';
 
 interface BaseComponents {
@@ -204,6 +205,11 @@ export class ClusterService implements Startable {
 
 				// Encode and yield the response
 				yield new TextEncoder().encode(JSON.stringify(response));
+				// One request per stream: every real ClusterClient sends exactly one
+				// request per dial (see ProtocolClient.processMessage), so complete the
+				// generator after the first response. A second frame a peer queued is
+				// then never read or parsed. Mirrors sync/block-transfer.
+				return;
 			}
 		};
 
@@ -211,7 +217,7 @@ export class ClusterService implements Startable {
 			try {
 				const responses = pipe(
 					stream,
-					(source) => lpDecode(source),
+					(source) => lpDecode(source, { maxDataLength: MAX_CONTROL_MESSAGE_BYTES }),
 					processStream.bind(this),
 					(source) => lpEncode(source)
 				);

package/src/cluster/spread-on-churn.ts

@@ -48,6 +48,13 @@ export interface SpreadOnChurnDeps {
 	peerNetwork: IPeerNetwork
 	clusterSize: number
 	protocolPrefix?: string
+	/**
+	 * The owned-block tracked set. When provided (e.g. the shared `ownedBlocks` set wired in
+	 * `libp2p-node-base`), the monitor references this exact `Set` instead of constructing its own,
+	 * so it stays in lock-step with the `RebalanceMonitor` that shares it. Omit for standalone
+	 * construction (unit tests) — a fresh private `Set` preserves all existing behavior.
+	 */
+	trackedBlocks?: Set<string>
 }
 
 export interface SpreadEvent {
@@ -83,7 +90,7 @@ const DEFAULT_CONFIG: SpreadOnChurnConfig = {
 
 export class SpreadOnChurnMonitor implements Startable {
 	private running = false
-	private readonly trackedBlocks = new Set<string>()
+	private readonly trackedBlocks: Set<string>
 	private readonly handlers: SpreadHandler[] = []
 	private debounceTimer: ReturnType<typeof setTimeout> | null = null
 	private departureTimestamps: number[] = []
@@ -96,6 +103,9 @@ export class SpreadOnChurnMonitor implements Startable {
 		private readonly deps: SpreadOnChurnDeps,
 		config: Partial<SpreadOnChurnConfig> = {}
 	) {
+		// Share the injected owned-block set when present (so spread + rebalance never drift);
+		// otherwise own a private set (standalone construction / unit tests).
+		this.trackedBlocks = deps.trackedBlocks ?? new Set<string>()
 		this.config = { ...DEFAULT_CONFIG, ...config }
 		this.onConnectionClose = () => this.handleDeparture()
 	}

package/src/cluster/supermajority-coupling.ts

@@ -0,0 +1,37 @@
+import { DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
+
+/** A component that exposes the super-majority threshold it actually runs on. */
+export interface HasEffectiveSuperMajorityThreshold {
+	readonly effectiveSuperMajorityThreshold: number;
+}
+
+/**
+ * Fail-fast coupling check for a live node's consensus wiring.
+ *
+ * The cluster **member** (what accepts a super-majority of promises as sufficient for a commit) and the
+ * **coordinator** (what declares a transaction committed once it has that super-majority) MUST run the
+ * SAME `superMajorityThreshold`. If they disagree, the coordinator can declare a transaction final on a
+ * super-majority the member never accepts as final — a silent phase-disagreement that only surfaces
+ * mid-consensus, far too late.
+ *
+ * On a live node both are fed from a single resolved `consensusConfig`, so this check normally passes. It
+ * exists to catch *future* drift — divergent fallback defaults, a mis-threaded `clusterPolicy`, a
+ * per-deployment config that reaches only one side — by throwing at construction with both values and
+ * their provenance, rather than letting the node come up mismatched.
+ *
+ * @throws Error naming both resolved thresholds and where they come from, if they are not equal.
+ */
+export function assertSuperMajorityCoupling(
+	member: HasEffectiveSuperMajorityThreshold,
+	coordinator: HasEffectiveSuperMajorityThreshold
+): void {
+	const memberThreshold = member.effectiveSuperMajorityThreshold;
+	const coordinatorThreshold = coordinator.effectiveSuperMajorityThreshold;
+	if (memberThreshold !== coordinatorThreshold) {
+		throw new Error(
+			`Super-majority threshold mismatch at node startup: cluster member resolved ${memberThreshold} but coordinator resolved ${coordinatorThreshold}. ` +
+			`Both derive from consensusConfig.superMajorityThreshold (options.clusterPolicy?.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD=${DEFAULT_SUPER_MAJORITY_THRESHOLD}); ` +
+			`they must be equal or the coordinator would declare transactions committed on a super-majority the member rejects as final.`
+		);
+	}
+}

package/src/cohort-topic/cohort-gossip-driver.ts

@@ -16,6 +16,7 @@ import {
 	toGossipRecord,
 	willingnessBitsHex,
 	selfWillingnessBits,
+	type ChildLinkRefV1,
 	type CohortGossipV1,
 	type CohortTopicSummary,
 	type GossipRecordRefV1,
@@ -35,6 +36,20 @@ import {
  */
 export const DEFAULT_GOSSIP_INTERVAL_MS = 5_000;
 
+/**
+ * Default `T_willingness_heartbeat` (ms): how often a genuinely-**idle** but **willing** engine re-broadcasts
+ * a willingness-only heartbeat so a cold cohort can bootstrap (siblings hear it, instantiate, and reciprocate
+ * their own willingness) without waiting on a first registration that can never be admitted while the view is
+ * empty. See `docs/cohort-topic.md` §Cold-start instantiation / §Configuration.
+ *
+ * On the order of the ping interval (~30 s, `ttl/3` Core) — a few gossip rounds at the 5 s cadence. A
+ * record-carrying (non-idle) round already ships willingness every round and resets this clock, so the
+ * throttle governs only engines with nothing else to say. The very first idle round after an engine is
+ * created emits immediately (no wait), so bootstrap converges in ~2 rounds; the throttle only paces the
+ * steady-state re-broadcast of an idle willing cohort.
+ */
+export const DEFAULT_WILLINGNESS_HEARTBEAT_MS = 30_000;
+
 /**
  * Per-`CoordEngine` queue of registration-record deltas accumulated between gossip rounds. The renewal
  * cohort side calls {@link PendingDeltas.touch} on every served ping/re-attach and
@@ -46,18 +61,46 @@ export interface PendingDeltas {
 	touch(rec: RegistrationRecord): void;
 	/** Queue an eviction ref and drop any pending record for the same key (a stale record can't also re-advertise). */
 	evicted(rec: RegistrationRecord): void;
+	/**
+	 * Queue a child-cohort **link** for replication (keyed by `(topicId, childCohortCoord)`; last write wins on
+	 * `effectiveAt`). Enqueued by the parent engine only when the local child registry actually changed, so a
+	 * stale/no-op record is not re-gossiped. A link and a later unlink for the same child in one round collapse
+	 * to whichever carries the newer `effectiveAt`.
+	 */
+	childLink(topicId: Uint8Array, childCohortCoord: Uint8Array, effectiveAt: number): void;
+	/** Queue a child-cohort **unlink** (a released/demoted child) for replication; same key + last-writer-wins as {@link childLink}. */
+	childUnlink(topicId: Uint8Array, childCohortCoord: Uint8Array, effectiveAt: number): void;
 	/** True iff nothing is queued. */
 	isEmpty(): boolean;
 	/** Drain the queue into wire-shaped deltas, clearing it. */
-	drain(): { records: GossipRecordV1[]; evicted: GossipRecordRefV1[] };
+	drain(): { records: GossipRecordV1[]; evicted: GossipRecordRefV1[]; childLinks: ChildLinkRefV1[]; childUnlinks: ChildLinkRefV1[] };
+}
+
+/** A queued child link/unlink: the wire ref plus whether it is a link (`true`) or an unlink (`false`). */
+interface PendingChildDelta {
+	ref: ChildLinkRefV1;
+	linked: boolean;
 }
 
 /** Build an empty {@link PendingDeltas} queue. */
 export function createPendingDeltas(): PendingDeltas {
 	const records = new Map<string, RegistrationRecord>();
 	const evicted = new Map<string, GossipRecordRefV1>();
+	const childDeltas = new Map<string, PendingChildDelta>();
 	const keyOf = (topicId: Uint8Array, participantId: Uint8Array): string =>
 		`${bytesToB64url(topicId)}|${bytesToB64url(participantId)}`;
+	// A child delta and its later opposite (link→unlink) share one key so the round drains only the newest.
+	const childKeyOf = (topicId: Uint8Array, childCohortCoord: Uint8Array): string =>
+		`${bytesToB64url(topicId)}|${bytesToB64url(childCohortCoord)}`;
+	const queueChild = (topicId: Uint8Array, childCohortCoord: Uint8Array, effectiveAt: number, linked: boolean): void => {
+		const key = childKeyOf(topicId, childCohortCoord);
+		const held = childDeltas.get(key);
+		// Last-writer-wins on effectiveAt: never let an older link/unlink shadow a newer one queued the same round.
+		if (held !== undefined && effectiveAt < held.ref.effectiveAt) {
+			return;
+		}
+		childDeltas.set(key, { ref: { topicId: bytesToB64url(topicId), childCohortCoord: bytesToB64url(childCohortCoord), effectiveAt }, linked });
+	};
 	return {
 		touch(rec: RegistrationRecord): void {
 			const key = keyOf(rec.topicId, rec.participantId);
@@ -70,19 +113,35 @@ export function createPendingDeltas(): PendingDeltas {
 		},
 		evicted(rec: RegistrationRecord): void {
 			const key = keyOf(rec.topicId, rec.participantId);
-			evicted.set(key, { topicId: bytesToB64url(rec.topicId), participantId: bytesToB64url(rec.participantId) });
+			// Stamp the evicted record's lastPing so the receiver can gate the delete on freshness (a stale
+			// eviction must not delete a record the participant has since re-registered — see bus.mergeRecords).
+			evicted.set(key, { topicId: bytesToB64url(rec.topicId), participantId: bytesToB64url(rec.participantId), lastPing: rec.lastPing });
 			records.delete(key);
 		},
+		childLink(topicId: Uint8Array, childCohortCoord: Uint8Array, effectiveAt: number): void {
+			queueChild(topicId, childCohortCoord, effectiveAt, true);
+		},
+		childUnlink(topicId: Uint8Array, childCohortCoord: Uint8Array, effectiveAt: number): void {
+			queueChild(topicId, childCohortCoord, effectiveAt, false);
+		},
 		isEmpty(): boolean {
-			return records.size === 0 && evicted.size === 0;
+			return records.size === 0 && evicted.size === 0 && childDeltas.size === 0;
 		},
-		drain(): { records: GossipRecordV1[]; evicted: GossipRecordRefV1[] } {
+		drain(): { records: GossipRecordV1[]; evicted: GossipRecordRefV1[]; childLinks: ChildLinkRefV1[]; childUnlinks: ChildLinkRefV1[] } {
+			const childLinks: ChildLinkRefV1[] = [];
+			const childUnlinks: ChildLinkRefV1[] = [];
+			for (const delta of childDeltas.values()) {
+				(delta.linked ? childLinks : childUnlinks).push(delta.ref);
+			}
 			const out = {
 				records: [...records.values()].map(toGossipRecord),
 				evicted: [...evicted.values()],
+				childLinks,
+				childUnlinks,
 			};
 			records.clear();
 			evicted.clear();
+			childDeltas.clear();
 			return out;
 		},
 	};
@@ -96,6 +155,16 @@ export interface GossipFrameInputs {
 	readonly coord: string;
 	/** This cohort's epoch, base64url. */
 	readonly cohortEpoch: string;
+	/** This cohort's tree tier `d` — carried on the frame so a cold sibling instantiates at the right tier. */
+	readonly treeTier: number;
+	/**
+	 * True to emit a **willingness-only heartbeat** even when this engine is idle (no resident topics, no
+	 * pending deltas) — provided the node is actually willing for some tier. Lets an idle-but-willing engine
+	 * still tell siblings it will serve, so a cold cohort can bootstrap (§Cold-start instantiation). The caller
+	 * (the gossip round) sets this from the per-engine heartbeat clock; a non-idle round ignores it (it already
+	 * ships willingness).
+	 */
+	readonly heartbeat: boolean;
 	/** This node's tier profile (the `∧` half of the willingness vector). */
 	readonly profile: NodeProfile;
 	/** This member's load barometer (load buckets + the load-shed half of willingness). */
@@ -108,18 +177,29 @@ export interface GossipFrameInputs {
 	readonly records: GossipRecordV1[];
 	/** Drained eviction refs for this round. */
 	readonly evicted: GossipRecordRefV1[];
+	/** Drained child-cohort link refs for this round (cross-member child-set convergence). */
+	readonly childLinks: ChildLinkRefV1[];
+	/** Drained child-cohort unlink refs for this round (a released/demoted child). */
+	readonly childUnlinks: ChildLinkRefV1[];
 	/** Round timestamp, unix ms. */
 	readonly timestamp: number;
 }
 
 /**
- * Assemble one round's {@link CohortGossipV1}, or `undefined` when this engine is **idle** — no resident
- * topics and no pending deltas, so it has nothing for siblings and the host skips the broadcast (idle
- * empty engines cost no gossip). `willingnessBits` is `profile ∧ load` ({@link selfWillingnessBits}); the
+ * Assemble one round's {@link CohortGossipV1}, or `undefined` when this engine has nothing to say. An engine
+ * is **idle** when it holds no resident topics and no pending deltas. An idle engine normally builds no frame
+ * (idle empty engines cost no gossip) — **except** on a willingness heartbeat ({@link GossipFrameInputs.heartbeat}),
+ * where an idle engine that is willing for at least one tier (`selfWillingnessBits !== 0`) still emits a
+ * willingness/load-only frame (empty `topicSummaries`, no `records`/`evicted`) so siblings can hear it and a
+ * cold cohort can bootstrap (§Cold-start instantiation). An idle-and-unwilling engine stays silent even on a
+ * heartbeat (nothing to bootstrap). `willingnessBits` is `profile ∧ load` ({@link selfWillingnessBits}); the
  * `signature` slot is left empty for the host's peer-key signer to fill before broadcast.
  */
 export function buildCohortGossip(i: GossipFrameInputs): CohortGossipV1 | undefined {
-	if (i.topicSummaries.length === 0 && i.records.length === 0 && i.evicted.length === 0) {
+	const willingness = selfWillingnessBits(i.profile, i.barometer);
+	const idle = i.topicSummaries.length === 0 && i.records.length === 0 && i.evicted.length === 0
+		&& i.childLinks.length === 0 && i.childUnlinks.length === 0;
+	if (idle && !(i.heartbeat && willingness !== 0)) {
 		return undefined;
 	}
 	const g: CohortGossipV1 = {
@@ -127,7 +207,8 @@ export function buildCohortGossip(i: GossipFrameInputs): CohortGossipV1 | undefi
 		fromMember: i.fromMember,
 		coord: i.coord,
 		cohortEpoch: i.cohortEpoch,
-		willingnessBits: willingnessBitsHex(selfWillingnessBits(i.profile, i.barometer)),
+		treeTier: i.treeTier,
+		willingnessBits: willingnessBitsHex(willingness),
 		loadBuckets: i.barometer.loadBuckets(),
 		windowSeconds: i.windowSeconds,
 		topicSummaries: i.topicSummaries,
@@ -140,5 +221,11 @@ export function buildCohortGossip(i: GossipFrameInputs): CohortGossipV1 | undefi
 	if (i.evicted.length > 0) {
 		g.evicted = i.evicted;
 	}
+	if (i.childLinks.length > 0) {
+		g.childLinks = i.childLinks;
+	}
+	if (i.childUnlinks.length > 0) {
+		g.childUnlinks = i.childUnlinks;
+	}
 	return g;
 }

package/src/cohort-topic/cohort-gossip-transport.ts

@@ -64,6 +64,16 @@ export class FretCohortGossipTransport implements ICohortGossipTransport {
 		return () => this.handlers.delete(handler);
 	}
 
+	/**
+	 * Number of live inbound subscriptions (one per subscribed gossip bus). Test/diagnostic introspection:
+	 * each {@link CoordEngine}'s bus subscribes via {@link onMessage} and drops its subscription on `close()`,
+	 * so this tracks the number of live coord engines (+ the node-level participant bus) — used to observe that
+	 * the coord-engine registry's LRU eviction actually tears evicted engines down (no leaked subscriptions).
+	 */
+	get subscriberCount(): number {
+		return this.handlers.size;
+	}
+
 	/** Feed an inbound gossip frame (called by the host's `/cohort-gossip` protocol handler). */
 	deliver(fromPeerId: string, msg: Uint8Array): void {
 		const from: PeerRef = { id: peerIdToBytes(fromPeerId) };