@feltdb/core 0.6.13 → 0.6.14

package/dist/create/server-source/crates/feltdb/src/distributed_transactions.rs

@@ -8,6 +8,7 @@
 ///
 /// Invariant: All replicas converge to identical state_hash for same operations
 
+use crate::causal_dependency_barrier::CausalDependencyBarrier;
 use crate::convergence::VectorClock;
 use crate::durable_operation_log::OperationLog;
 use crate::state_hash::StateHash;
@@ -168,6 +169,124 @@ pub struct DistributedTransactionExecutor {
     pub envelopes_seen: std::collections::HashSet<EnvelopeId>,
     pub transaction_executor: TransactionExecutor,
     pub operation_log: Option<OperationLog>,
+    /// Causal delivery gate for replicated envelopes.
+    ///
+    /// Holds identifiers and dependency clocks only. A blocked envelope's
+    /// payload stays in the durable operation log and is replayed from there
+    /// when its dependencies land.
+    pub causal_barrier: CausalDependencyBarrier,
+    /// The bound on the causal backlog.
+    pub causal_capacity: CausalCapacity,
+    /// The next identity this node will issue for its own operations.
+    ///
+    /// Recovered from the durable log rather than reset, because
+    /// `(origin, sequence)` is an identity and receivers deduplicate on it. A
+    /// counter that restarts re-issues identities that already exist, and the
+    /// receiver discards the new operation as a duplicate.
+    next_origin_sequence: u64,
+    /// Envelopes that are durable in the log but not buffered, because the
+    /// backlog was at its bound when they arrived.
+    ///
+    /// Derived state, like `envelopes_seen`: it is rebuilt from the log on
+    /// recovery and exists so that a redelivery of a deferred envelope is
+    /// recognised without a log scan.
+    pub deferred: std::collections::HashSet<EnvelopeId>,
+}
+
+/// What the receiver did with a replicated envelope.
+///
+/// This is a typed outcome rather than a string, because a caller has to be
+/// able to tell "applied" from "held back" from "already known" without
+/// matching on prose. The previous signature returned
+/// `Result<StateHash, String>` and signalled a duplicate as
+/// `Err("Duplicate envelope")`.
+#[derive(Clone, Debug, PartialEq)]
+pub enum ReceiveOutcome {
+    /// Dependencies were satisfied; the envelope was applied.
+    Applied { state_hash: StateHash },
+    /// Durable, but not yet applied: some dependency has not arrived.
+    /// The payload is in the operation log; the barrier holds its identity.
+    PendingDependencies { missing: Vec<String> },
+    /// This envelope has been seen before. Applying it again is a no-op.
+    AlreadyKnown,
+    /// Durable, but not buffered: the backlog is at its bound.
+    ///
+    /// This is *not* a refusal and nothing is lost. The envelope was fsynced to
+    /// the operation log before this was decided, exactly as a buffered one is;
+    /// what is withheld is the in-memory metadata entry. The caller's obligation
+    /// is transport backpressure -- stop reading from that peer so TCP blocks
+    /// the sender -- because returning a refusal to a peer means either silent
+    /// loss or a retransmission protocol this repository does not have.
+    ///
+    /// The envelope is admitted from the log once the backlog drains.
+    CapacityDeferred {
+        missing: Vec<String>,
+        pending_entries: usize,
+        pending_bytes: usize,
+    },
+}
+
+/// What the executor did with a locally submitted transaction.
+///
+/// Capacity refusal is a distinct outcome rather than an `Err(String)`, because
+/// a caller has to tell "this replica is saturated, retry" from "this
+/// transaction is invalid". The two need opposite responses and the previous
+/// signature made them the same value.
+#[derive(Clone, Debug)]
+pub enum SubmitOutcome {
+    Admitted(TransactionEnvelope),
+    /// The causal backlog is at its bound, so no new local work is admitted.
+    /// Nothing was executed, persisted, or applied: there is nothing to undo
+    /// and nothing to recover.
+    CapacityExceeded {
+        pending_entries: usize,
+        pending_bytes: usize,
+        limit_entries: usize,
+        limit_bytes: usize,
+    },
+}
+
+/// The bound on the causal backlog, as decided in
+/// `docs/architecture/causal-backlog-bound.md`.
+///
+/// Two limits, because they are driven by different variables. Bytes bound the
+/// outstanding payload volume; entries bound the metadata, whose per-entry cost
+/// scales with cluster size rather than with operation size, so a byte bound
+/// alone would score ten thousand pending one-byte operations as nearly free.
+///
+/// Both are inclusive: the bound is `<=`, so filling exactly to it is accepted.
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub struct CausalCapacity {
+    pub max_pending_entries: usize,
+    pub max_pending_bytes: usize,
+}
+
+impl CausalCapacity {
+    /// The shipping default.
+    ///
+    /// These are policy, not measurements, and are deliberately not derived
+    /// from a benchmark constant -- the figure that used to justify backlog
+    /// sizing here was `backlog_depth * 1500`, which was the benchmark's own
+    /// input rather than an observation. What the acceptance test establishes
+    /// is that the bound is *enforced through the production path*, and it
+    /// reports the occupancy it actually observed rather than asserting a
+    /// predicted footprint.
+    pub const DEFAULT_MAX_PENDING_ENTRIES: usize = 10_000;
+    pub const DEFAULT_MAX_PENDING_BYTES: usize = 64 * 1024 * 1024;
+
+    /// A bound small enough to be reached by a test through the real path.
+    pub fn new(max_pending_entries: usize, max_pending_bytes: usize) -> Self {
+        Self { max_pending_entries, max_pending_bytes }
+    }
+}
+
+impl Default for CausalCapacity {
+    fn default() -> Self {
+        Self {
+            max_pending_entries: Self::DEFAULT_MAX_PENDING_ENTRIES,
+            max_pending_bytes: Self::DEFAULT_MAX_PENDING_BYTES,
+        }
+    }
 }
 
 impl DistributedTransactionExecutor {
@@ -185,6 +304,10 @@ impl DistributedTransactionExecutor {
             envelopes_seen: std::collections::HashSet::new(),
             transaction_executor: TransactionExecutor::new(node_id),
             operation_log: None,
+            causal_barrier: CausalDependencyBarrier::new(),
+            causal_capacity: CausalCapacity::default(),
+            next_origin_sequence: 1,
+            deferred: std::collections::HashSet::new(),
         }
     }
 
@@ -209,12 +332,25 @@ impl DistributedTransactionExecutor {
             None
         };
 
+        // Identity is recovered here as well as in `load_from_disk`, because a
+        // caller that opens an existing log and issues an operation without
+        // replaying it must still not re-issue an identity that log already
+        // contains.
+        let next_origin_sequence = match &operation_log {
+            Some(log) => Self::next_sequence_after(log, &node_id)?,
+            None => 1,
+        };
+
         Ok(Self {
             local_node_id: node_id.clone(),
             replicas,
             envelopes_seen: std::collections::HashSet::new(),
             transaction_executor: TransactionExecutor::new(node_id),
             operation_log,
+            causal_barrier: CausalDependencyBarrier::new(),
+            causal_capacity: CausalCapacity::default(),
+            next_origin_sequence,
+            deferred: std::collections::HashSet::new(),
         })
     }
 
@@ -228,6 +364,147 @@ impl DistributedTransactionExecutor {
 
     /// Execute transaction locally and prepare for replication
     /// Ordering: execute → persist → update state → record dedup
+    // ---------------------------------------------------------------------
+    // Durable operation identity
+    //
+    // `(origin, sequence)` is an identity, not a counter. Receivers
+    // deduplicate on it, so two operations must never be assigned the same
+    // one -- and "must never" has to hold across a process restart, which is
+    // where it previously did not: `feltdb_node` kept the counter in memory
+    // and started it at 1, so a restarted origin re-issued identities its own
+    // earlier operations already held and its peers discarded the new writes
+    // as duplicates. See `dedup_bound_investigation.rs`.
+    //
+    // The authority is the durable log, and specifically the log rather than
+    // the causal clock. Those are different identity domains: the clock
+    // component advances only on a successful apply, while the sequence is
+    // burned by an attempt, so the two disagree by exactly the number of
+    // refused transactions. `replication-convergence.md` fixes that the
+    // sequence is sparse and that making it dense is a separate deferred
+    // decision, so this derives identity from the sequence domain and leaves
+    // that decision alone.
+    // ---------------------------------------------------------------------
+
+    /// The next identity this node would issue.
+    pub fn next_origin_sequence(&self) -> u64 {
+        self.next_origin_sequence
+    }
+
+    /// Claim the next identity for an operation this node originates.
+    ///
+    /// Advancing on the attempt rather than on success is deliberate: it
+    /// preserves the sparse-sequence semantics the convergence contract froze,
+    /// where a refused transaction burns a number. Making the sequence dense is
+    /// a separate decision and is not made here as a side effect.
+    ///
+    /// A number burned and then lost to a crash is safe to re-issue, because
+    /// nothing carrying it was ever persisted: recovery derives from what the
+    /// log holds, so an identity is only reserved forever once an envelope
+    /// bearing it is durable. That is the invariant -- no two *persisted*
+    /// envelopes from one origin share a sequence -- and it is the one
+    /// receivers depend on.
+    pub fn allocate_origin_sequence(&mut self) -> u64 {
+        let claimed = self.next_origin_sequence;
+        self.next_origin_sequence = claimed.saturating_add(1);
+        claimed
+    }
+
+    /// Derive the next identity from a durable log: one past the highest this
+    /// node has already committed to it.
+    ///
+    /// Filtered to locally-originated envelopes, because the log also holds
+    /// replicated ones and another node's sequence says nothing about this
+    /// node's identity space.
+    fn next_sequence_after(log: &OperationLog, node_id: &str) -> Result<u64, std::io::Error> {
+        let highest = log
+            .load_all()?
+            .iter()
+            .filter(|envelope| envelope.envelope_id.originating_node == node_id)
+            .map(|envelope| envelope.envelope_id.sequence)
+            .max()
+            .unwrap_or(0);
+        Ok(highest.saturating_add(1))
+    }
+
+    /// Submit local work, allocating its identity from durable state.
+    ///
+    /// This is the entry point a node should use: it owns the allocation, so
+    /// the identity cannot be assigned by one caller and persisted by another,
+    /// and there is no counter outside the executor to fall out of step with
+    /// the log.
+    pub fn submit_local_transaction(
+        &mut self,
+        transaction_id: String,
+        parent_version: StateVersion,
+        operations: Vec<Operation>,
+        consistency_contract: ConsistencyContract,
+    ) -> Result<SubmitOutcome, String> {
+        // Allocation happens after the capacity check inside
+        // `try_execute_local_transaction`, so a refusal does not burn an
+        // identity: nothing was admitted, and there is nothing to account for.
+        if self.would_breach_bound(Self::operations_bytes(&operations)) {
+            return Ok(SubmitOutcome::CapacityExceeded {
+                pending_entries: self.causal_barrier.pending_count(),
+                pending_bytes: self.causal_barrier.pending_payload_bytes(),
+                limit_entries: self.causal_capacity.max_pending_entries,
+                limit_bytes: self.causal_capacity.max_pending_bytes,
+            });
+        }
+        let sequence = self.allocate_origin_sequence();
+        self.execute_local_transaction(
+            sequence,
+            transaction_id,
+            parent_version,
+            operations,
+            consistency_contract,
+        )
+        .map(SubmitOutcome::Admitted)
+    }
+
+    /// Submit local work, subject to the causal backlog bound.
+    ///
+    /// The bound is checked *before* anything is executed, persisted or
+    /// applied, so a refusal leaves nothing to undo and nothing to recover:
+    /// the operation was never admitted. That is what makes refusal the right
+    /// answer on this path and the wrong one for a replicated arrival, which
+    /// was already admitted and made durable somewhere else.
+    pub fn try_execute_local_transaction(
+        &mut self,
+        sequence: u64,
+        transaction_id: String,
+        parent_version: StateVersion,
+        operations: Vec<Operation>,
+        consistency_contract: ConsistencyContract,
+    ) -> Result<SubmitOutcome, String> {
+        let projected_bytes = Self::operations_bytes(&operations);
+        if self.would_breach_bound(projected_bytes) {
+            return Ok(SubmitOutcome::CapacityExceeded {
+                pending_entries: self.causal_barrier.pending_count(),
+                pending_bytes: self.causal_barrier.pending_payload_bytes(),
+                limit_entries: self.causal_capacity.max_pending_entries,
+                limit_bytes: self.causal_capacity.max_pending_bytes,
+            });
+        }
+        self.execute_local_transaction(
+            sequence,
+            transaction_id,
+            parent_version,
+            operations,
+            consistency_contract,
+        )
+        .map(SubmitOutcome::Admitted)
+    }
+
+    /// The size local work would contribute, measured before it is executed.
+    fn operations_bytes(operations: &[Operation]) -> usize {
+        serde_json::to_string(operations).map(|json| json.len()).unwrap_or(0)
+    }
+
+    /// Execute local work unconditionally.
+    ///
+    /// This does not consult the backlog bound; `try_execute_local_transaction`
+    /// is the admitting entry point and the one the node binary uses. This
+    /// remains for callers that have already made the admission decision.
     pub fn execute_local_transaction(
         &mut self,
         sequence: u64,
@@ -249,7 +526,7 @@ impl DistributedTransactionExecutor {
         }
 
         // Step 2: Create envelope for replication
-        let envelope = TransactionEnvelope::new(
+        let mut envelope = TransactionEnvelope::new(
             self.local_node_id.clone(),
             sequence,
             transaction_id,
@@ -257,6 +534,19 @@ impl DistributedTransactionExecutor {
             consistency_contract,
         );
 
+        // Step 2a: Stamp the envelope with this node's causal knowledge.
+        //
+        // `TransactionEnvelope::new` starts from an empty clock and increments
+        // only the origin, so every envelope it built carried {origin: 1}
+        // whatever the node had already seen. That is a position, not a
+        // history: a receiver given those clocks has no dependencies to
+        // enforce, which is why causal ordering could not have worked on this
+        // path even once the barrier was wired to it. The clock must carry the
+        // applied frontier this operation was produced on top of.
+        let mut stamped = self.causal_barrier.get_frontier().clone();
+        stamped.increment(&self.local_node_id);
+        envelope.vector_clock = stamped;
+
         // Step 3: Durably persist operation to log (with fsync)
         // This must succeed before we update in-memory state
         self.persist_envelope(&envelope)?;
@@ -270,6 +560,12 @@ impl DistributedTransactionExecutor {
         // Step 5: Record envelope to prevent duplicates (in-memory, derived from log)
         self.envelopes_seen.insert(envelope.envelope_id.clone());
 
+        // Step 6: A locally executed operation is applied, so it advances the
+        // frontier exactly as a replicated one does. Own writes are always
+        // causally eligible: they are built on the frontier they advance.
+        self.causal_barrier
+            .record_applied(&Self::barrier_key(&envelope.envelope_id), &envelope.vector_clock);
+
         Ok(envelope)
     }
 
@@ -280,37 +576,419 @@ impl DistributedTransactionExecutor {
         message: ReplicationMessage,
         parent_version: StateVersion,
     ) -> Result<StateHash, String> {
-        // Step 1: Check for duplicate (prevent re-application)
-        if self.envelopes_seen.contains(&message.envelope.envelope_id) {
-            return Err("Duplicate envelope".to_string());
+        match self.receive_replicated(message, parent_version)? {
+            ReceiveOutcome::Applied { state_hash } => Ok(state_hash),
+            ReceiveOutcome::PendingDependencies { missing } => Err(format!(
+                "Envelope is durable but waiting on dependencies: {}",
+                missing.join(", ")
+            )),
+            ReceiveOutcome::AlreadyKnown => Err("Duplicate envelope".to_string()),
+            ReceiveOutcome::CapacityDeferred { missing, .. } => Err(format!(
+                "CAPACITY_DEFERRED: envelope is durable but the causal backlog is at its bound, waiting on: {}",
+                missing.join(", ")
+            )),
+        }
+    }
+
+    /// The causal receive path.
+    ///
+    /// Ordering is deliberate and is the substance of this change:
+    ///
+    ///   dedup -> persist -> causal eligibility -> apply -> release
+    ///
+    /// Persisting *before* deciding eligibility is what makes "held back" a
+    /// durable state rather than an in-memory one. An envelope that arrives
+    /// ahead of its dependencies is fsynced to the operation log and then
+    /// retained in the barrier by identity alone; if the process dies at that
+    /// moment, recovery replays the log and rediscovers it as pending. This is
+    /// also what catch-up will require, because a peer sending A3, A1, A2 in
+    /// transport order needs the receiver to be able to say "I have A3
+    /// durably, and I cannot apply it yet".
+    pub fn receive_replicated(
+        &mut self,
+        message: ReplicationMessage,
+        parent_version: StateVersion,
+    ) -> Result<ReceiveOutcome, String> {
+        let envelope_id = message.envelope.envelope_id.clone();
+        let key = Self::barrier_key(&envelope_id);
+
+        // Step 1: Duplicate delivery is a no-op, whether the original was
+        // applied or is still waiting. Both states are recorded before this
+        // point is reached again, so one logical operation stays one.
+        if self.envelopes_seen.contains(&envelope_id)
+            || self.causal_barrier.is_pending(&key)
+            || self.deferred.contains(&envelope_id)
+        {
+            return Ok(ReceiveOutcome::AlreadyKnown);
+        }
+
+        // Step 2: Durability first. The log is the source of truth, so an
+        // envelope becomes durable on receipt and not on application.
+        self.persist_envelope(&message.envelope)?;
+
+        // Step 3: Eligibility. The dependencies are everything the origin had
+        // seen when it produced this operation, which is its clock with its own
+        // entry stepped back by one; the operation itself is not its own
+        // dependency.
+        let dependencies = Self::dependencies_of(&message.envelope);
+        if !self.causal_barrier.is_eligible(&dependencies) {
+            let missing = self.missing_dependencies(&dependencies);
+
+            // Step 3a: The bound. Buffering costs an entry and holds a payload
+            // outstanding in the log; if either would breach its limit, the
+            // metadata entry is withheld and the envelope stays durable and
+            // deferred. It is never refused and never evicted -- refusing a
+            // peer means silent loss or a retransmission protocol that does not
+            // exist here, and eviction is silent loss under another name.
+            let payload_bytes = Self::envelope_bytes(&message.envelope);
+            if self.would_breach_bound(payload_bytes) {
+                self.deferred.insert(envelope_id);
+                return Ok(ReceiveOutcome::CapacityDeferred {
+                    missing,
+                    pending_entries: self.causal_barrier.pending_count(),
+                    pending_bytes: self.causal_barrier.pending_payload_bytes(),
+                });
+            }
+
+            self.causal_barrier
+                .retain_pending_sized(key, dependencies, payload_bytes);
+            return Ok(ReceiveOutcome::PendingDependencies { missing });
         }
 
-        // Step 2: Execute transaction through 14C
+        // Step 4: Apply, which advances the frontier.
+        let state_hash = self.apply_envelope(&message.envelope, parent_version.clone())?;
+
+        // Step 5: This application may have unblocked others, and may have
+        // freed the capacity a deferred envelope was waiting for. Drain both.
+        self.drain(parent_version)?;
+
+        Ok(ReceiveOutcome::Applied { state_hash })
+    }
+
+    /// Drive the backlog forward from outside the receive path.
+    ///
+    /// A deferred envelope is durable but unbuffered, and the event that frees
+    /// capacity for it may arrive on a different connection than the one that
+    /// was deferred. A caller applying transport backpressure needs a way to
+    /// make progress without reading from the backpressured peer, and this is
+    /// it.
+    pub fn drain_backlog(&mut self, parent_version: StateVersion) -> Result<usize, String> {
+        self.drain(parent_version)
+    }
+
+    /// Whether admitting one more entry of this size would breach either bound.
+    ///
+    /// Inclusive, as the contract specifies: filling exactly to the bound is
+    /// accepted, and the breach is the step past it.
+    fn would_breach_bound(&self, payload_bytes: usize) -> bool {
+        let entries = self.causal_barrier.pending_count();
+        let bytes = self.causal_barrier.pending_payload_bytes();
+        entries + 1 > self.causal_capacity.max_pending_entries
+            || bytes + payload_bytes > self.causal_capacity.max_pending_bytes
+    }
+
+    /// The envelope's size as the durable log stores it.
+    ///
+    /// Measured with the log's own serialisation rather than a nominal
+    /// per-operation constant, so a large envelope consumes its actual byte
+    /// count.
+    fn envelope_bytes(envelope: &TransactionEnvelope) -> usize {
+        serde_json::to_string(envelope).map(|json| json.len()).unwrap_or(0)
+    }
+
+    /// Apply one envelope and advance the applied frontier.
+    fn apply_envelope(
+        &mut self,
+        envelope: &TransactionEnvelope,
+        parent_version: StateVersion,
+    ) -> Result<StateHash, String> {
         let transition = self.transaction_executor.execute_transaction(
-            message.envelope.transaction_id.clone(),
+            envelope.transaction_id.clone(),
             parent_version,
-            message.envelope.operations.clone(),
-            message.envelope.consistency_contract.clone(),
+            envelope.operations.clone(),
+            envelope.consistency_contract.clone(),
         );
 
         if !transition.is_successful() {
             return Err("Transaction execution failed".to_string());
         }
 
-        // Step 3: Durably persist operation to log (with fsync)
-        self.persist_envelope(&message.envelope)?;
-
-        // Step 4: Record envelope to prevent duplicates (in-memory)
-        self.envelopes_seen.insert(message.envelope.envelope_id.clone());
+        self.envelopes_seen.insert(envelope.envelope_id.clone());
 
-        // Step 5: Update this replica's state (receiver updates itself)
+        let state_hash = StateHash::from_hex(transition.to_version.state_hash.clone());
         if let Some(replica) = self.replicas.get_mut(&self.local_node_id) {
-            let state_hash = StateHash::from_hex(transition.to_version.state_hash.clone());
-            replica.apply_transaction(&message.envelope, state_hash);
+            replica.apply_transaction(envelope, state_hash.clone());
+        }
+
+        // The frontier advances here and only here, so it describes what has
+        // been successfully applied rather than what was claimed or received.
+        self.causal_barrier
+            .record_applied(&Self::barrier_key(&envelope.envelope_id), &envelope.vector_clock);
+
+        Ok(state_hash)
+    }
+
+    /// Drive the backlog forward until it stops moving.
+    ///
+    /// Two things can make progress and each can enable the other: applying a
+    /// pending envelope advances the frontier, which may make other pending
+    /// envelopes eligible; and applying anything frees capacity, which may let
+    /// a deferred envelope be admitted from the log. So this alternates until
+    /// neither step changes anything, which terminates because every iteration
+    /// that continues has strictly reduced the number of unapplied envelopes.
+    fn drain(&mut self, parent_version: StateVersion) -> Result<usize, String> {
+        let mut progressed_total = 0;
+        loop {
+            let released = self.release_eligible(parent_version.clone())?;
+            let admitted = self.admit_deferred(parent_version.clone())?;
+            progressed_total += released + admitted;
+            if released == 0 && admitted == 0 {
+                return Ok(progressed_total);
+            }
         }
+    }
+
+    /// Apply every pending envelope the frontier now satisfies, repeatedly,
+    /// because releasing one can make the next eligible.
+    ///
+    /// The log is read once per pass rather than once per key. It used to be
+    /// once per key: `replay_from_log` called `load_all()` for every eligible
+    /// entry, so a release cascade over a backlog of n entries against a log of
+    /// m records did n full scans. That is the exact shape of load this change
+    /// makes reachable -- a bounded backlog is one that is allowed to get
+    /// large -- so the quadratic scan had to go with it.
+    fn release_eligible(&mut self, parent_version: StateVersion) -> Result<usize, String> {
+        let mut released = 0;
+        loop {
+            let ready = self.causal_barrier.eligible_pending();
+            if ready.is_empty() {
+                return Ok(released);
+            }
+            let by_key = self.log_by_key()?;
+            let mut progressed = false;
+            for key in ready {
+                let Some(envelope) = by_key.get(&key) else {
+                    // The barrier holds an identity whose payload is not in the
+                    // log. Dropping the entry would be silent loss, so leave it
+                    // pending and surface it rather than pretend it applied.
+                    continue;
+                };
+                self.apply_envelope(&envelope.clone(), parent_version.clone())?;
+                released += 1;
+                progressed = true;
+            }
+            if !progressed {
+                return Ok(released);
+            }
+        }
+    }
+
+    /// Move envelopes that are durable but unbuffered into the backlog, as far
+    /// as the bound now allows.
+    ///
+    /// This is what makes `CapacityDeferred` safe: the envelope was fsynced
+    /// before it was deferred, so admitting it later needs no cooperation from
+    /// the sender and survives a restart in between. Entries are admitted in
+    /// key order so that two replicas draining the same set drain it
+    /// identically -- behaviour at capacity must not depend on iteration order
+    /// or timing.
+    fn admit_deferred(&mut self, parent_version: StateVersion) -> Result<usize, String> {
+        if self.deferred.is_empty() {
+            return Ok(0);
+        }
+        let by_key = self.log_by_key()?;
+        let mut keys: Vec<String> = self
+            .deferred
+            .iter()
+            .map(Self::barrier_key)
+            .collect();
+        keys.sort();
+
+        let mut admitted = 0;
+        for key in keys {
+            let Some(envelope) = by_key.get(&key).cloned() else { continue };
+            let dependencies = Self::dependencies_of(&envelope);
+            let payload_bytes = Self::envelope_bytes(&envelope);
+
+            if self.causal_barrier.is_eligible(&dependencies) {
+                self.deferred.remove(&envelope.envelope_id);
+                self.apply_envelope(&envelope, parent_version.clone())?;
+                admitted += 1;
+                continue;
+            }
+            if self.would_breach_bound(payload_bytes) {
+                // Still no room. Leave it deferred; it is durable and will be
+                // reconsidered on the next drain or after a restart.
+                continue;
+            }
+            self.deferred.remove(&envelope.envelope_id);
+            self.causal_barrier
+                .retain_pending_sized(key, dependencies, payload_bytes);
+            admitted += 1;
+        }
+        Ok(admitted)
+    }
 
-        // Return the state hash
-        Ok(StateHash::from_hex(transition.to_version.state_hash))
+    /// The durable log, indexed by barrier key.
+    ///
+    /// This is why the barrier never needs the payload: it holds the identity,
+    /// and the bytes stay where they were already fsynced.
+    fn log_by_key(&self) -> Result<HashMap<String, TransactionEnvelope>, String> {
+        let Some(ref log) = self.operation_log else {
+            return Ok(HashMap::new());
+        };
+        let envelopes = log
+            .load_all()
+            .map_err(|e| format!("Failed to replay operation log: {}", e))?;
+        Ok(envelopes
+            .into_iter()
+            .map(|envelope| (Self::barrier_key(&envelope.envelope_id), envelope))
+            .collect())
+    }
+
+    /// The stable string identity the barrier is keyed by.
+    fn barrier_key(envelope_id: &EnvelopeId) -> String {
+        format!("{}:{}", envelope_id.originating_node, envelope_id.sequence)
+    }
+
+    /// The causal dependencies of an envelope: what its origin had applied
+    /// before producing it.
+    fn dependencies_of(envelope: &TransactionEnvelope) -> VectorClock {
+        let mut dependencies = envelope.vector_clock.clone();
+        let own = dependencies.get(&envelope.originating_node);
+        dependencies
+            .clocks
+            .insert(envelope.originating_node.clone(), own.saturating_sub(1));
+        dependencies
+    }
+
+    /// Which dimensions of a dependency clock the frontier has not reached.
+    fn missing_dependencies(&self, dependencies: &VectorClock) -> Vec<String> {
+        let frontier = self.causal_barrier.get_frontier();
+        let mut missing: Vec<String> = dependencies
+            .clocks
+            .iter()
+            .filter(|(node, required)| frontier.get(node) < **required)
+            .map(|(node, required)| {
+                format!("{}@{} (have {})", node, required, frontier.get(node))
+            })
+            .collect();
+        missing.sort();
+        missing
+    }
+
+    /// How many envelopes are durable but waiting on dependencies.
+    ///
+    /// Instrumentation only; no limit is enforced. See
+    /// docs/architecture/causal-backlog-bound.md.
+    pub fn pending_causal_count(&self) -> usize {
+        self.causal_barrier.pending_count()
+    }
+
+    /// Approximate heap cost of the pending causal metadata, in bytes.
+    pub fn pending_causal_metadata_bytes(&self) -> usize {
+        self.causal_barrier.pending_metadata_bytes()
+    }
+
+    /// Durable payload bytes the backlog is waiting on. The byte bound is
+    /// expressed in these.
+    pub fn pending_causal_payload_bytes(&self) -> usize {
+        self.causal_barrier.pending_payload_bytes()
+    }
+
+    /// Envelopes that are durable but not buffered, because the bound was
+    /// reached when they arrived. They are not lost and not evicted.
+    pub fn deferred_causal_count(&self) -> usize {
+        self.deferred.len()
+    }
+
+    /// Keys of envelopes that are durable but not buffered, sorted.
+    pub fn deferred_causal_keys(&self) -> Vec<String> {
+        let mut keys: Vec<String> = self.deferred.iter().map(Self::barrier_key).collect();
+        keys.sort();
+        keys
+    }
+
+    /// Maximum occupancy ever reached, for entries and payload bytes.
+    ///
+    /// Reported alongside the current values because a backlog that filled and
+    /// drained between two reads is invisible to a current-value gauge, and the
+    /// question an operator has is whether the bound was ever approached.
+    pub fn max_causal_occupancy(&self) -> (usize, usize) {
+        (
+            self.causal_barrier.max_pending_entries_seen(),
+            self.causal_barrier.max_pending_payload_bytes_seen(),
+        )
+    }
+
+    /// The bound in force on this executor.
+    pub fn causal_capacity(&self) -> CausalCapacity {
+        self.causal_capacity
+    }
+
+    /// Set the bound. Used by the node binary so an acceptance test can reach
+    /// the bound through the production path rather than by constructing a
+    /// barrier directly.
+    pub fn set_causal_capacity(&mut self, capacity: CausalCapacity) {
+        self.causal_capacity = capacity;
+    }
+
+    /// The applied causal frontier: what this replica has successfully applied.
+    pub fn causal_frontier(&self) -> &VectorClock {
+        self.causal_barrier.get_frontier()
+    }
+
+    /// Identities of envelopes durable but not yet applied.
+    pub fn pending_causal_keys(&self) -> Vec<String> {
+        self.causal_barrier.pending_keys()
+    }
+
+    /// Materialize the applied records, order-independently.
+    ///
+    /// This exists because the chained `H(parent, command)` state hash is a
+    /// missing-operation detector for an ordered history, not a convergence
+    /// identity: two replicas that applied the same set of concurrent
+    /// operations in different orders hold the same data and report different
+    /// chained hashes. `CanonicalState` is a BTreeMap keyed by
+    /// (collection, record_id), so its hash depends on the resulting records
+    /// and not on the path taken to them, which is what "the replicas agree"
+    /// should mean.
+    ///
+    /// Built from the durable log, filtered to what the barrier records as
+    /// applied and taken in application order, so a pending envelope's
+    /// operations are excluded until it is actually released.
+    pub fn canonical_state(&self) -> Result<crate::state_hash::CanonicalState, String> {
+        let mut state = crate::state_hash::CanonicalState::new();
+        let Some(ref log) = self.operation_log else {
+            return Ok(state);
+        };
+        let envelopes = log
+            .load_all()
+            .map_err(|e| format!("Failed to read operation log: {}", e))?;
+        let by_key: HashMap<String, &TransactionEnvelope> = envelopes
+            .iter()
+            .map(|envelope| (Self::barrier_key(&envelope.envelope_id), envelope))
+            .collect();
+
+        for key in self.causal_barrier.applied_keys() {
+            let Some(envelope) = by_key.get(key) else {
+                continue;
+            };
+            for operation in &envelope.operations {
+                let command = &operation.command;
+                let fields: serde_json::Map<String, serde_json::Value> = command
+                    .fields
+                    .iter()
+                    .map(|(name, value)| (name.clone(), value.clone()))
+                    .collect();
+                state.set_record(
+                    command.collection.clone(),
+                    command.record_id.clone(),
+                    serde_json::Value::Object(fields),
+                );
+            }
+        }
+        Ok(state)
     }
 
     /// Get current state of a replica
@@ -366,14 +1044,77 @@ impl DistributedTransactionExecutor {
         let log = OperationLog::new(&log_path)?;
         let envelopes = log.load_all()?;
 
-        // Reconstruct dedup set from loaded envelopes (deterministic, atomic with log)
-        for envelope in &envelopes {
-            self.envelopes_seen.insert(envelope.envelope_id.clone());
+        self.operation_log = Some(log);
+
+        // Rebuild the causal state, not just the dedup set.
+        //
+        // The log records what was *received*, in arrival order, and arrival
+        // order is not causal order -- that is the whole reason the barrier
+        // exists. So recovery cannot simply mark everything seen: it has to
+        // replay eligibility, exactly as the live path does, and come out with
+        // the same split between applied and still-pending. This is what makes
+        // "deliver B, restart, deliver A" work: B is durable across the
+        // restart, is rediscovered as pending rather than as applied, and is
+        // released when A finally arrives.
+        let mut undecided: Vec<TransactionEnvelope> = envelopes.clone();
+        loop {
+            let mut progressed = false;
+            let mut still_undecided = Vec::new();
+            for envelope in undecided.into_iter() {
+                let dependencies = Self::dependencies_of(&envelope);
+                if self.causal_barrier.is_eligible(&dependencies) {
+                    self.envelopes_seen.insert(envelope.envelope_id.clone());
+                    self.causal_barrier.record_applied(
+                        &Self::barrier_key(&envelope.envelope_id),
+                        &envelope.vector_clock,
+                    );
+                    progressed = true;
+                } else {
+                    still_undecided.push(envelope);
+                }
+            }
+            undecided = still_undecided;
+            if !progressed || undecided.is_empty() {
+                break;
+            }
         }
 
-        // Note: Replica state is not rebuilt from log in this initial version
-        // It will be recovered through normal replication when this node rejoins the cluster
-        self.operation_log = Some(log);
+        // Whatever could not be made eligible is durable and waiting, and is
+        // restored to the barrier in that state -- as far as the bound allows.
+        //
+        // Recovery has to respect the same bound the live path does, or a node
+        // could restart into an over-full backlog that it would never have
+        // accepted while running, and the bound would hold only until the first
+        // crash. Entries are restored in key order so that recovery is
+        // deterministic: the same log and the same bound produce the same split
+        // between pending and deferred every time.
+        undecided.sort_by_key(|envelope| Self::barrier_key(&envelope.envelope_id));
+        for envelope in undecided {
+            let dependencies = Self::dependencies_of(&envelope);
+            let payload_bytes = Self::envelope_bytes(&envelope);
+            if self.would_breach_bound(payload_bytes) {
+                self.deferred.insert(envelope.envelope_id.clone());
+                continue;
+            }
+            self.causal_barrier.retain_pending_sized(
+                Self::barrier_key(&envelope.envelope_id),
+                dependencies,
+                payload_bytes,
+            );
+        }
+
+        // Identity is recovered from the same log, and this is the half that
+        // was missing: the causal frontier was already rebuilt here, so a
+        // restarted origin came back knowing its causal position and not its
+        // identity position, and issued colliding identities on top of a
+        // correct clock.
+        self.next_origin_sequence = envelopes
+            .iter()
+            .filter(|envelope| envelope.envelope_id.originating_node == self.local_node_id)
+            .map(|envelope| envelope.envelope_id.sequence)
+            .max()
+            .unwrap_or(0)
+            .saturating_add(1);
 
         Ok(envelopes.len())
     }