@feltdb/core 0.6.13 → 0.7.0

package/dist/create/server-source/crates/feltdb/src/bin/feltdb_node.rs

@@ -37,20 +37,29 @@
 //! resend-last         put the identical envelope back on the wire, to test dedup
 //! query-state         the local replica's state hash and applied-operation count
 //! metrics             per-link byte and message counters, straight from the transport
+//! causal-state        applied frontier, and what is durable but held back
+//! hold-inbound PEER   park messages from PEER instead of delivering them
+//! release-inbound PEER [reverse]
+//!                     deliver what was parked, optionally newest-first
 //! peers               per-link connection state
 //! shutdown            exit cleanly
 //! ```
 
 use feltdb::convergence::VectorClock;
-use feltdb::distributed_transactions::{DistributedTransactionExecutor, ReplicationMessage};
+use feltdb::distributed_transactions::{
+    CausalCapacity, DistributedTransactionExecutor, ReceiveOutcome, ReplicationMessage,
+    SubmitOutcome,
+};
 use feltdb::replication_protocol::ProtocolTransport;
+use feltdb::replica_acknowledgements::{AcknowledgementStore, ReclamationEvidence, ReplicaAcknowledgement};
+use feltdb::replica_membership::MembershipStore;
 use feltdb::state_hash::StateHash;
 use feltdb::tcp_transport::TcpTransport;
 use feltdb::transactions::{
     ConsistencyContract, Operation, OperationCommand, OperationId, StateVersion,
 };
 use serde_json::json;
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::io::{self, BufRead, Write};
 use std::path::PathBuf;
 use std::sync::atomic::{AtomicBool, Ordering};
@@ -86,6 +95,53 @@ struct RecvLink {
     io: TcpTransport,
 }
 
+/// Hand one replicated message to the executor and record what it decided.
+///
+/// The harness makes no ordering decision of its own here. It calls the same
+/// `receive_replicated` the shipping receive path calls, and reports the typed
+/// outcome back so a test can assert on "applied" versus "held back" rather
+/// than inferring it from state.
+fn deliver(
+    executor: &mut DistributedTransactionExecutor,
+    version: &mut StateVersion,
+    message: ReplicationMessage,
+) -> serde_json::Value {
+    let origin = message.envelope.originating_node.clone();
+    let key = format!(
+        "{}:{}",
+        message.envelope.envelope_id.originating_node, message.envelope.envelope_id.sequence
+    );
+    let parent = version.clone();
+    match executor.receive_replicated(message, parent) {
+        Ok(ReceiveOutcome::Applied { state_hash }) => {
+            // Only a real application advances the chained version, so a held
+            // or duplicated message cannot make the node look like it moved.
+            version.state_hash = state_hash.to_string();
+            version.vector_clock.increment(&origin);
+            json!({ "envelope": key, "outcome": "applied" })
+        }
+        Ok(ReceiveOutcome::PendingDependencies { missing }) => {
+            json!({ "envelope": key, "outcome": "pending", "missing": missing })
+        }
+        Ok(ReceiveOutcome::AlreadyKnown) => {
+            json!({ "envelope": key, "outcome": "already_known" })
+        }
+        Ok(ReceiveOutcome::CapacityDeferred { missing, pending_entries, pending_bytes }) => {
+            // Durable, but not buffered. The caller's obligation is to stop
+            // reading from this peer; the envelope is admitted from the log
+            // once the backlog drains.
+            json!({
+                "envelope": key,
+                "outcome": "capacity_deferred",
+                "missing": missing,
+                "pending_entries": pending_entries,
+                "pending_bytes": pending_bytes,
+            })
+        }
+        Err(error) => json!({ "envelope": key, "outcome": "error", "error": error }),
+    }
+}
+
 fn parse_link(value: &str) -> Result<(String, String), String> {
     let (peer, addr) = value
         .split_once('=')
@@ -118,6 +174,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let mut data_dir: Option<PathBuf> = None;
     let mut send_specs: Vec<(String, String)> = Vec::new();
     let mut recv_specs: Vec<(String, String)> = Vec::new();
+    let mut max_pending_entries: Option<usize> = None;
+    let mut max_pending_bytes: Option<usize> = None;
 
     let mut index = 1;
     while index < args.len() {
@@ -143,6 +201,30 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                 index += 1;
                 recv_specs.push(parse_link(args.get(index).ok_or("--recv requires PEER=ADDR")?)?);
             }
+            // The bound is settable so an acceptance test can reach it through
+            // the production path. Filling the shipping default would mean
+            // sending ten thousand blocked envelopes over TCP; a test that
+            // instead constructed a barrier and filled it directly would be
+            // measuring a component nothing runs, which is the defect this
+            // whole line of work exists to stop repeating.
+            "--max-pending-entries" => {
+                index += 1;
+                max_pending_entries = Some(
+                    args.get(index)
+                        .ok_or("--max-pending-entries requires a value")?
+                        .parse::<usize>()
+                        .map_err(|e| format!("--max-pending-entries: {e}"))?,
+                );
+            }
+            "--max-pending-bytes" => {
+                index += 1;
+                max_pending_bytes = Some(
+                    args.get(index)
+                        .ok_or("--max-pending-bytes requires a value")?
+                        .parse::<usize>()
+                        .map_err(|e| format!("--max-pending-bytes: {e}"))?,
+                );
+            }
             other => return Err(format!("unknown argument {other}").into()),
         }
         index += 1;
@@ -159,10 +241,45 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         dir.join("operations.jsonl")
     });
 
+    // Membership lives beside the log and is loaded from disk, never rebuilt
+    // from --send/--recv. A node's connection flags say who it talks to; they
+    // say nothing about who is owed history, and conflating the two is what
+    // this store exists to stop.
+    let mut membership = MembershipStore::open(
+        data_dir
+            .as_ref()
+            .map(|dir| dir.join("membership.json"))
+            .unwrap_or_else(|| PathBuf::from("membership.json")),
+    )
+    .map_err(|e| e.to_string())?;
+
+    // Acknowledgements are durable and shared with the receive tasks, because
+    // they arrive on the socket rather than through a command.
+    let acknowledgements = Arc::new(Mutex::new(
+        AcknowledgementStore::open(
+            data_dir
+                .as_ref()
+                .map(|dir| dir.join("acknowledgements.json"))
+                .unwrap_or_else(|| PathBuf::from("acknowledgements.json")),
+        )
+        .map_err(|e| e.to_string())?,
+    ));
+
     let initial_state = StateHash::from_hex("0".repeat(64));
     let mut executor =
         DistributedTransactionExecutor::with_log(node_id.clone(), initial_state.clone(), log_path.clone())?;
 
+    // The bound is set before recovery, not after, because recovery restores
+    // the backlog and has to respect the same limit the live path does.
+    // Otherwise a node could restart into a backlog it would never have
+    // accepted while running, and the bound would hold only until the first
+    // crash.
+    let default_capacity = CausalCapacity::default();
+    executor.set_causal_capacity(CausalCapacity::new(
+        max_pending_entries.unwrap_or(default_capacity.max_pending_entries),
+        max_pending_bytes.unwrap_or(default_capacity.max_pending_bytes),
+    ));
+
     // Replaying the log restores the deduplication set, so a restarted node
     // does not re-apply operations it already durably recorded.
     let mut replayed = 0usize;
@@ -194,7 +311,25 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     // interleave and chain onto a parent that is already stale.
     let core = Arc::new(Mutex::new((executor, StateVersion::new(VectorClock::new(), "0".repeat(64)))));
     let should_exit = Arc::new(AtomicBool::new(false));
-    let sequence = Arc::new(Mutex::new(1u64));
+    // Per-peer delivery hold.
+    //
+    // `Some(queue)` means messages arriving from that peer are received off the
+    // socket as normal and parked instead of being handed to the executor.
+    // This is how a test produces out-of-causal-order arrival without faking
+    // the network: the bytes really do cross TCP, and only the order in which
+    // the receiver is offered them is controlled. A real network reorders
+    // delivery this way; the harness just does it on demand.
+    let held: Arc<Mutex<HashMap<String, Vec<ReplicationMessage>>>> =
+        Arc::new(Mutex::new(HashMap::new()));
+    // Peers this node has stopped reading from because the causal backlog is
+    // at its bound.
+    //
+    // This is the backpressure the bounding decision commits to, and not
+    // reading is the whole mechanism: the socket's receive buffer fills, the
+    // TCP window closes, and the sender blocks. Nothing is dropped and nothing
+    // is refused, so no retransmission protocol is needed -- which is good,
+    // because there isn't one.
+    let backpressured: Arc<Mutex<HashSet<String>>> = Arc::new(Mutex::new(HashSet::new()));
     // Kept so `resend-last` can put the identical envelope back on the wire.
     let last_envelope: Arc<Mutex<Option<feltdb::distributed_transactions::TransactionEnvelope>>> =
         Arc::new(Mutex::new(None));
@@ -230,6 +365,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         let core = core.clone();
         let exit = should_exit.clone();
         let peer = link.peer.clone();
+        let held = held.clone();
+        let backpressured = backpressured.clone();
+        let acknowledgements = acknowledgements.clone();
         tokio::spawn(async move {
             while !exit.load(Ordering::Relaxed) {
                 {
@@ -247,22 +385,57 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                     if exit.load(Ordering::Relaxed) {
                         return;
                     }
+
+                    // Do not read from a backpressured peer. While paused, keep
+                    // trying to drain: the work that frees capacity may arrive
+                    // on another connection, and a deferred envelope is already
+                    // durable, so draining needs nothing from this socket.
+                    while backpressured.lock().await.contains(&peer) {
+                        if exit.load(Ordering::Relaxed) {
+                            return;
+                        }
+                        {
+                            let mut guard = core.lock().await;
+                            let (executor, version) = &mut *guard;
+                            let parent = version.clone();
+                            let _ = executor.drain_backlog(parent);
+                            if executor.pending_causal_count()
+                                < executor.causal_capacity().max_pending_entries
+                            {
+                                backpressured.lock().await.remove(&peer);
+                            }
+                        }
+                        sleep(Duration::from_millis(25)).await;
+                    }
+
                     match transport.receive().await {
                         Ok(messages) => {
+                            // Acknowledgements arrive interleaved with
+                            // envelopes on the same connection. They are
+                            // durable before they count as evidence.
+                            for ack in transport.take_acknowledgements().await {
+                                let mut store = acknowledgements.lock().await;
+                                if let Err(error) = store.record(ack) {
+                                    eprintln!("acknowledgement not recorded: {error}");
+                                }
+                            }
                             for message in messages {
-                                let mut guard = core.lock().await;
-                                let (executor, version) = &mut *guard;
-                                let origin = message.envelope.originating_node.clone();
-                                let parent = version.clone();
-                                // A duplicate is refused by the executor, which
-                                // is the behaviour under test, not an error
-                                // here — and a refused message must not advance
-                                // the version.
-                                if let Ok(applied) =
-                                    executor.receive_replicated_transaction(message, parent)
                                 {
-                                    version.state_hash = applied.to_string();
-                                    version.vector_clock.increment(&origin);
+                                    let mut hold = held.lock().await;
+                                    if let Some(queue) = hold.get_mut(&peer) {
+                                        queue.push(message);
+                                        continue;
+                                    }
+                                }
+                                let outcome = {
+                                    let mut guard = core.lock().await;
+                                    let (executor, version) = &mut *guard;
+                                    deliver(executor, version, message)
+                                };
+                                if outcome.get("outcome").and_then(|value| value.as_str())
+                                    == Some("capacity_deferred")
+                                {
+                                    backpressured.lock().await.insert(peer.clone());
                                 }
                             }
                         }
@@ -309,17 +482,18 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         match verb {
             "execute-tx" => {
                 let record = if argument.is_empty() { "item".to_string() } else { argument };
-                let seq = {
-                    let mut guard = sequence.lock().await;
-                    let current = *guard;
-                    *guard += 1;
-                    current
-                };
 
                 let mut fields = HashMap::new();
                 fields.insert("record".to_string(), json!(record.clone()));
 
-                let parent = { core.lock().await.1.clone() };
+                // The identity comes from the executor, which derives it from the
+                // durable log. This binary used to keep its own counter,
+                // starting at 1 on every launch, so a restarted node re-issued
+                // identities its own earlier operations already held.
+                let (parent, seq) = {
+                    let guard = core.lock().await;
+                    (guard.1.clone(), guard.0.next_origin_sequence())
+                };
                 let operation = Operation::new(
                     OperationId::new(node_id.clone(), seq),
                     parent.clone(),
@@ -336,14 +510,17 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                 let envelope = {
                     let mut guard = core.lock().await;
                     let (executor, version) = &mut *guard;
-                    let result = executor.execute_local_transaction(
-                        seq,
+                    // The admitting entry point: local work is refused when the
+                    // causal backlog is at its bound, before anything is
+                    // executed or persisted, so a refusal leaves nothing to
+                    // undo.
+                    let result = executor.submit_local_transaction(
                         format!("tx_{record}"),
                         version.clone(),
                         vec![operation],
                         ConsistencyContract::local(),
                     );
-                    if result.is_ok() {
+                    if matches!(result, Ok(SubmitOutcome::Admitted(_))) {
                         if let Some(replica) = executor.get_replica_state(&node_id) {
                             version.state_hash = replica.state_hash.to_string();
                             version.vector_clock.increment(&node_id);
@@ -353,7 +530,27 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                 };
 
                 match envelope {
-                    Ok(envelope) => {
+                    Ok(SubmitOutcome::CapacityExceeded {
+                        pending_entries,
+                        pending_bytes,
+                        limit_entries,
+                        limit_bytes,
+                    }) => {
+                        // A distinct outcome, not an error: the caller should
+                        // retry, and nothing was admitted or lost.
+                        println!(
+                            "TX_REFUSED {}",
+                            json!({
+                                "record": record,
+                                "reason": "CAPACITY_EXCEEDED",
+                                "pending_entries": pending_entries,
+                                "pending_bytes": pending_bytes,
+                                "limit_entries": limit_entries,
+                                "limit_bytes": limit_bytes,
+                            })
+                        );
+                    }
+                    Ok(SubmitOutcome::Admitted(envelope)) => {
                         *last_envelope.lock().await = Some(envelope.clone());
                         let mut delivered = Vec::new();
                         let mut failed = Vec::new();
@@ -383,7 +580,13 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                         }
                         println!(
                             "TX_EXECUTED {}",
-                            json!({ "record": record, "sequence": seq, "delivered": delivered, "failed": failed })
+                            json!({
+                                "record": record,
+                                // The identity the executor actually issued.
+                                "sequence": envelope.envelope_id.sequence,
+                                "delivered": delivered,
+                                "failed": failed,
+                            })
                         );
                     }
                     Err(error) => println!("ERROR {}", json!({ "reason": error })),
@@ -397,6 +600,127 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             // only interesting if the duplicate actually crosses the wire.
             // Issuing another `execute-tx` would not test it: that produces a
             // new envelope id and is a different transaction.
+            // `resend SEQUENCE` — put one specific envelope from this node's
+            // durable log back on the wire.
+            //
+            // A peer re-sending an operation another node is missing is the
+            // primitive catch-up is built from. This is not catch-up: nothing
+            // decides *what* to send, a test does. It exists so the receiver's
+            // behaviour can be exercised on a gap that a restart left behind.
+            // `forward ORIGIN SEQUENCE` -- put an envelope this node holds, but
+            // did not originate, back on the wire.
+            //
+            // `resend` deliberately only replays this node's own envelopes. A
+            // peer relaying an operation that a *third* node is missing is a
+            // different primitive, and it is the one a backpressured receiver
+            // needs: when a node stops reading from the peer whose envelopes it
+            // deferred, the dependency it is waiting for cannot arrive on that
+            // link. It has to come from somewhere else, which is precisely the
+            // case the read loop's keep-draining-while-paused behaviour exists
+            // to serve.
+            "forward" => {
+                let mut parts = argument.split_whitespace();
+                let origin = parts.next().unwrap_or_default().to_string();
+                let wanted: Option<u64> = parts.next().and_then(|value| value.parse().ok());
+                let envelope = {
+                    let guard = core.lock().await;
+                    let (executor, _) = &*guard;
+                    executor.operation_log.as_ref().and_then(|log| {
+                        log.load_all().ok().and_then(|envelopes| {
+                            envelopes.into_iter().find(|candidate| {
+                                candidate.envelope_id.originating_node == origin
+                                    && Some(candidate.envelope_id.sequence) == wanted
+                            })
+                        })
+                    })
+                };
+                match envelope {
+                    Some(envelope) => {
+                        let mut delivered = Vec::new();
+                        for link in &send_links {
+                            let message = ReplicationMessage::new(
+                                envelope.clone(),
+                                node_id.clone(),
+                                link.peer.clone(),
+                                envelope.envelope_id.sequence,
+                            );
+                            if link.io.send(message.clone()).await.is_ok() {
+                                delivered.push(link.peer.clone());
+                            } else if connect_with_retry(link, 3).await
+                                && link.io.send(message).await.is_ok()
+                            {
+                                delivered.push(link.peer.clone());
+                            }
+                        }
+                        println!(
+                            "FORWARDED {}",
+                            json!({
+                                "origin": origin,
+                                "sequence": wanted,
+                                "delivered": delivered,
+                            })
+                        );
+                    }
+                    None => println!(
+                        "FORWARDED {}",
+                        json!({ "origin": origin, "sequence": wanted, "delivered": [], "found": false })
+                    ),
+                }
+                io::stdout().flush().ok();
+            }
+
+            "resend" => {
+                let wanted: Option<u64> = argument.trim().parse().ok();
+                let envelope = {
+                    let guard = core.lock().await;
+                    let (executor, _) = &*guard;
+                    executor.operation_log.as_ref().and_then(|log| {
+                        log.load_all().ok().and_then(|envelopes| {
+                            envelopes.into_iter().find(|candidate| {
+                                candidate.envelope_id.originating_node == node_id
+                                    && Some(candidate.envelope_id.sequence) == wanted
+                            })
+                        })
+                    })
+                };
+                match envelope {
+                    Some(envelope) => {
+                        let mut delivered = Vec::new();
+                        for link in &send_links {
+                            let message = ReplicationMessage::new(
+                                envelope.clone(),
+                                node_id.clone(),
+                                link.peer.clone(),
+                                envelope.envelope_id.sequence,
+                            );
+                            // The peer this is aimed at may have restarted
+                            // since the link was established, which is the
+                            // whole point of re-sending. Reconnect and retry,
+                            // as `execute-tx` does.
+                            match link.io.send(message.clone()).await {
+                                Ok(()) => delivered.push(link.peer.clone()),
+                                Err(_) => {
+                                    if connect_with_retry(link, 3).await
+                                        && link.io.send(message).await.is_ok()
+                                    {
+                                        delivered.push(link.peer.clone());
+                                    }
+                                }
+                            }
+                        }
+                        println!(
+                            "RESENT {}",
+                            json!({ "envelope": envelope.envelope_id, "delivered": delivered })
+                        );
+                    }
+                    None => println!(
+                        "ERROR {}",
+                        json!({ "reason": format!("no envelope {argument} in this node's log") })
+                    ),
+                }
+                io::stdout().flush().ok();
+            }
+
             "resend-last" => {
                 let envelope = last_envelope.lock().await.clone();
                 match envelope {
@@ -426,12 +750,23 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             "query-state" => {
                 let guard = core.lock().await;
                 let (executor, _) = &*guard;
+                let canonical = executor
+                    .canonical_state()
+                    .map(|state| state.hash().to_string())
+                    .unwrap_or_else(|error| format!("error: {error}"));
                 match executor.get_replica_state(&node_id) {
                     Some(replica) => println!(
                         "STATE {}",
                         json!({
                             "hash": replica.state_hash.to_string(),
                             "operations_applied": replica.operations_applied,
+                            // Order-independent, and therefore the one to
+                            // compare across replicas. The chained hash above
+                            // detects a missing operation in an ordered
+                            // history; it calls two nodes that applied the same
+                            // concurrent operations in different orders
+                            // divergent, which they are not.
+                            "canonical_hash": canonical,
                         })
                     ),
                     None => println!("ERROR {}", json!({ "reason": "no local replica" })),
@@ -439,6 +774,257 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                 io::stdout().flush().ok();
             }
 
+            // What the causal receiver is holding, and why.
+            // Membership commands. Deliberately no compaction frontier and no
+            // acknowledgement tracking: this establishes who exists and who is
+            // required, and reclaims nothing.
+            "cluster-create" => {
+                let result = membership.create_cluster(argument.trim());
+                println!(
+                    "CLUSTER {}",
+                    match result {
+                        Ok(()) => json!({ "cluster_id": argument.trim(), "created": true }),
+                        Err(error) => json!({ "created": false, "error": error.to_string() }),
+                    }
+                );
+                io::stdout().flush().ok();
+            }
+
+            "replica-join" | "replica-approve" | "replica-suspect" | "replica-seen"
+            | "replica-remove" => {
+                let mut parts = argument.split_whitespace();
+                let replica = parts.next().unwrap_or_default().to_string();
+                let result = match verb {
+                    "replica-join" => {
+                        let node = parts.next().unwrap_or(&replica).to_string();
+                        membership.join_replica(&replica, &node)
+                    }
+                    "replica-approve" => membership.approve_replica(&replica),
+                    "replica-suspect" => membership.suspect_replica(&replica),
+                    "replica-seen" => membership.mark_seen(&replica),
+                    _ => membership.remove_replica(&replica),
+                };
+                println!(
+                    "REPLICA {}",
+                    match result {
+                        Ok(()) => json!({ "replica_id": replica, "action": verb, "ok": true }),
+                        Err(error) =>
+                            json!({ "replica_id": replica, "action": verb, "ok": false, "error": error.to_string() }),
+                    }
+                );
+                io::stdout().flush().ok();
+            }
+
+            // Report what this node has durably applied, to every peer.
+            //
+            // The frontier is read from the executor, where it advances only in
+            // record_applied -- after an fsynced envelope is applied. So the
+            // statement "durably applied through this frontier" is true by
+            // construction rather than by assertion.
+            "ack-broadcast" => {
+                let (frontier, marks) = {
+                    let guard = core.lock().await;
+                    let (executor, _) = &*guard;
+                    (executor.causal_frontier().clone(), executor.origin_sequence_high_water())
+                };
+                let view: Vec<String> = membership
+                    .get_members()
+                    .into_iter()
+                    .map(|replica| replica.replica_id)
+                    .collect();
+                let replica_id = membership
+                    .get_members()
+                    .into_iter()
+                    .find(|replica| replica.node_id == node_id)
+                    .map(|replica| replica.replica_id)
+                    .unwrap_or_else(|| node_id.clone());
+
+                let ack = ReplicaAcknowledgement::new(
+                    replica_id.clone(),
+                    node_id.clone(),
+                    frontier,
+                    marks,
+                    view,
+                );
+
+                // A node's own acknowledgement is evidence it holds first-hand:
+                // it knows its own durable frontier without being told. Without
+                // this a node would be permanently blocked on itself, since
+                // nothing else can report on its behalf.
+                {
+                    let mut store = acknowledgements.lock().await;
+                    let _ = store.record(ack.clone());
+                }
+
+                let mut delivered = Vec::new();
+                for link in &send_links {
+                    if link.io.send_acknowledgement(ack.clone()).await.is_ok() {
+                        delivered.push(link.peer.clone());
+                    } else if connect_with_retry(link, 3).await
+                        && link.io.send_acknowledgement(ack.clone()).await.is_ok()
+                    {
+                        delivered.push(link.peer.clone());
+                    }
+                }
+                println!(
+                    "ACK_SENT {}",
+                    json!({
+                        "replica_id": replica_id,
+                        "frontier": ack.applied_frontier.clocks,
+                        "delivered": delivered,
+                    })
+                );
+                io::stdout().flush().ok();
+            }
+
+            "acknowledgements" => {
+                let store = acknowledgements.lock().await;
+                println!("ACKS {}", json!({ "acknowledgements": store.all() }));
+                io::stdout().flush().ok();
+            }
+
+            // What this node's evidence permits. Computes; reclaims nothing.
+            "reclamation-evidence" => {
+                let local: Vec<String> = membership
+                    .get_required_replicas()
+                    .into_iter()
+                    .map(|replica| replica.replica_id)
+                    .collect();
+                let store = acknowledgements.lock().await;
+                let report = match store.evidence(&local) {
+                    ReclamationEvidence::Blocked { waiting_on } => json!({
+                        "state": "BLOCKED",
+                        "waiting_on": waiting_on,
+                        "required_union": store.required_union(&local),
+                    }),
+                    ReclamationEvidence::SafeThrough { frontier, covered } => json!({
+                        "state": "SAFE_THROUGH",
+                        "frontier": frontier.clocks,
+                        "covered": covered,
+                        "required_union": store.required_union(&local),
+                    }),
+                };
+                println!("EVIDENCE {}", report);
+                io::stdout().flush().ok();
+            }
+
+            "members" => {
+                println!(
+                    "MEMBERS {}",
+                    json!({
+                        "cluster_id": membership.cluster().map(|c| c.cluster_id.clone()),
+                        "members": membership.get_members(),
+                        "all": membership.all_replicas(),
+                    })
+                );
+                io::stdout().flush().ok();
+            }
+
+            "required-replicas" => {
+                println!(
+                    "REQUIRED {}",
+                    json!({ "required": membership.get_required_replicas() })
+                );
+                io::stdout().flush().ok();
+            }
+
+            "causal-state" => {
+                let guard = core.lock().await;
+                let (executor, _) = &*guard;
+                let paused = {
+                    let mut peers: Vec<String> =
+                        backpressured.lock().await.iter().cloned().collect();
+                    peers.sort();
+                    peers
+                };
+                println!(
+                    "CAUSAL {}",
+                    json!({
+                        "frontier": executor.causal_frontier().clocks,
+                        "pending": executor.pending_causal_keys(),
+                        "pending_count": executor.pending_causal_count(),
+                        "pending_metadata_bytes": executor.pending_causal_metadata_bytes(),
+                        // Occupancy against the bound that is now enforced.
+                        // Current and maximum, for both dimensions, because a
+                        // backlog that filled and drained between two reads is
+                        // invisible to a current-value gauge.
+                        "pending_payload_bytes": executor.pending_causal_payload_bytes(),
+                        "deferred_count": executor.deferred_causal_count(),
+                        "deferred": executor.deferred_causal_keys(),
+                        "max_pending_entries_seen": executor.max_causal_occupancy().0,
+                        "max_pending_payload_bytes_seen": executor.max_causal_occupancy().1,
+                        "limit_entries": executor.causal_capacity().max_pending_entries,
+                        "limit_bytes": executor.causal_capacity().max_pending_bytes,
+                        // Peers this node has stopped reading from. This is the
+                        // observable for backpressure: the pause lives in the
+                        // socket read loop, so without it a test can only infer
+                        // the behaviour from timing.
+                        //
+                        // Locked after `core`, matching the order the read loop
+                        // takes them in.
+                        "backpressured": paused,
+                    })
+                );
+                io::stdout().flush().ok();
+            }
+
+            // Park messages from a peer instead of delivering them, so a test
+            // can choose the order the receiver is offered them in.
+            "hold-inbound" => {
+                let mut hold = held.lock().await;
+                hold.entry(argument.clone()).or_default();
+                println!("HELD {}", json!({ "peer": argument }));
+                io::stdout().flush().ok();
+            }
+
+            // Deliver what was parked, optionally newest-first, then resume
+            // normal delivery. Each message goes through the same production
+            // receiver as an unheld one.
+            // `release-inbound PEER [reverse] [COUNT]`
+            //
+            // COUNT releases only that many and keeps holding the rest, which
+            // is what lets a test offer an operation while deliberately
+            // withholding the dependency it needs.
+            "release-inbound" => {
+                let mut parts = argument.split_whitespace();
+                let peer = parts.next().unwrap_or_default().to_string();
+                let mut reverse = false;
+                let mut limit: Option<usize> = None;
+                for token in parts {
+                    if token == "reverse" {
+                        reverse = true;
+                    } else if let Ok(value) = token.parse::<usize>() {
+                        limit = Some(value);
+                    }
+                }
+
+                let mut queued = { held.lock().await.remove(&peer).unwrap_or_default() };
+                if reverse {
+                    queued.reverse();
+                }
+                let take = limit.unwrap_or(queued.len()).min(queued.len());
+                let remainder = queued.split_off(take);
+
+                let mut outcomes = Vec::new();
+                for message in queued {
+                    let mut guard = core.lock().await;
+                    let (executor, version) = &mut *guard;
+                    outcomes.push(deliver(executor, version, message));
+                }
+
+                // Anything not released stays held, so the hold is still in
+                // force for messages that arrive afterwards.
+                if !remainder.is_empty() || limit.is_some() {
+                    held.lock().await.insert(peer.clone(), remainder);
+                }
+
+                println!(
+                    "RELEASED {}",
+                    json!({ "peer": peer, "reverse": reverse, "outcomes": outcomes })
+                );
+                io::stdout().flush().ok();
+            }
+
             // The proof that the network carried the work. Counters come from
             // the transport itself, so a test can assert bytes actually moved
             // rather than inferring it from converged state.