@feltdb/core 0.7.3 → 0.7.4

package/dist/create/server-source/crates/feltdb/src/authority_failover.rs

@@ -283,6 +283,10 @@ pub enum AuthorityError {
         provided: FencingToken,
         current: FencingToken,
     },
+    QuorumUnavailable {
+        live: usize,
+        required: usize,
+    },
     Io(String),
 }
 
@@ -296,6 +300,12 @@ impl std::fmt::Display for AuthorityError {
             Self::StaleToken { provided, current } => {
                 write!(f, "stale token {provided}, current is {current}")
             }
+            Self::QuorumUnavailable { live, required } => {
+                write!(
+                    f,
+                    "authority quorum unavailable: {live} live, {required} required"
+                )
+            }
             Self::Io(msg) => write!(f, "authority store: {msg}"),
         }
     }
@@ -308,6 +318,20 @@ impl AuthorityStore {
         cluster_id: &str,
         local_node_id: &str,
         local_priority: u32,
+    ) -> Result<Self, AuthorityError> {
+        Self::open_with_failure_detector(path, cluster_id, local_node_id, local_priority, 1_000, 5)
+    }
+
+    /// Open with an explicitly configured failure detector. Production keeps
+    /// the conservative default above; process-level acceptance tests use a
+    /// shorter interval so failure is event-driven without multi-second sleeps.
+    pub fn open_with_failure_detector<P: AsRef<Path>>(
+        path: P,
+        cluster_id: &str,
+        local_node_id: &str,
+        local_priority: u32,
+        heartbeat_interval_ms: u64,
+        failure_threshold: u32,
     ) -> Result<Self, AuthorityError> {
         let path = path.as_ref().to_path_buf();
         let state = if path.exists() {
@@ -323,7 +347,7 @@ impl AuthorityStore {
         Ok(Self {
             path,
             state,
-            heartbeats: HeartbeatState::new(1000, 5), // 1s heartbeat, 5 missed = failure
+            heartbeats: HeartbeatState::new(heartbeat_interval_ms, failure_threshold),
         })
     }
 
@@ -333,8 +357,8 @@ impl AuthorityStore {
             std::fs::create_dir_all(parent).map_err(|e| AuthorityError::Io(e.to_string()))?;
         }
         let temp = self.path.with_extension("tmp");
-        let bytes =
-            serde_json::to_vec_pretty(&self.state).map_err(|e| AuthorityError::Io(e.to_string()))?;
+        let bytes = serde_json::to_vec_pretty(&self.state)
+            .map_err(|e| AuthorityError::Io(e.to_string()))?;
         {
             use std::io::Write;
             let mut file =
@@ -416,7 +440,9 @@ impl AuthorityStore {
 
         let mut record =
             AuthorityRecord::new(self.state.local_node_id.clone(), new_token, previous);
-        record.acknowledgements.insert(self.state.local_node_id.clone());
+        record
+            .acknowledgements
+            .insert(self.state.local_node_id.clone());
 
         self.state.current_authority = Some(record);
         self.state.local_role = AuthorityRole::Authority;
@@ -512,6 +538,31 @@ impl AuthorityStore {
         Ok(())
     }
 
+    /// Validate an authoritative write against both its fencing term and a
+    /// live quorum. A partitioned former authority may not have observed the
+    /// replacement's newer token yet, so fencing alone is insufficient until
+    /// healing; expiry of its quorum evidence closes that split-brain window.
+    pub fn validate_quorum_write(
+        &self,
+        token: FencingToken,
+        required_quorum: usize,
+    ) -> Result<(), AuthorityError> {
+        self.validate_write(Some(token))?;
+        let live = 1 + self
+            .heartbeats
+            .last_seen
+            .keys()
+            .filter(|node_id| !self.heartbeats.is_failed(node_id))
+            .count();
+        if live < required_quorum {
+            return Err(AuthorityError::QuorumUnavailable {
+                live,
+                required: required_quorum,
+            });
+        }
+        Ok(())
+    }
+
     /// Step down from authority role (voluntary).
     pub fn step_down(&mut self) -> Result<(), AuthorityError> {
         if self.state.local_role == AuthorityRole::Authority {
@@ -746,13 +797,40 @@ mod tests {
         assert!(heartbeats.is_failed("A"));
     }
 
+    #[test]
+    fn partitioned_authority_loses_write_quorum() {
+        let dir = tempfile::tempdir().unwrap();
+        let mut store = AuthorityStore::open_with_failure_detector(
+            dir.path().join("authority.json"),
+            "test-cluster",
+            "A",
+            10,
+            1,
+            1,
+        )
+        .expect("open authority store");
+        let token = match store.claim_authority().expect("claim") {
+            ClaimResult::Granted { token } => token,
+            _ => panic!("authority claim was not granted"),
+        };
+        store.heartbeat("B");
+        assert!(store.validate_quorum_write(token, 2).is_ok());
+        store.heartbeats.last_seen.insert("B".to_string(), 0);
+        assert!(matches!(
+            store.validate_quorum_write(token, 2),
+            Err(AuthorityError::QuorumUnavailable {
+                live: 1,
+                required: 2
+            })
+        ));
+    }
+
     #[test]
     fn recovery_from_fenced() {
         let dir = tempfile::tempdir().unwrap();
         let path = dir.path().join("authority.json");
 
-        let mut store =
-            AuthorityStore::open(&path, "test-cluster", "A", 10).expect("open store");
+        let mut store = AuthorityStore::open(&path, "test-cluster", "A", 10).expect("open store");
         store.claim_authority().expect("claim");
 
         // Get fenced

package/dist/create/server-source/crates/feltdb/src/bin/feltdb_node.rs

@@ -51,9 +51,11 @@ use feltdb::distributed_transactions::{
     CausalCapacity, DistributedTransactionExecutor, ReceiveOutcome, ReplicationMessage,
     SubmitOutcome,
 };
-use feltdb::replication_protocol::ProtocolTransport;
-use feltdb::replica_acknowledgements::{AcknowledgementStore, ReclamationEvidence, ReplicaAcknowledgement};
+use feltdb::replica_acknowledgements::{
+    AcknowledgementStore, ReclamationEvidence, ReplicaAcknowledgement,
+};
 use feltdb::replica_membership::MembershipStore;
+use feltdb::replication_protocol::ProtocolTransport;
 use feltdb::state_hash::StateHash;
 use feltdb::tcp_transport::TcpTransport;
 use feltdb::transactions::{
@@ -127,7 +129,11 @@ fn deliver(
         Ok(ReceiveOutcome::AlreadyKnown) => {
             json!({ "envelope": key, "outcome": "already_known" })
         }
-        Ok(ReceiveOutcome::CapacityDeferred { missing, pending_entries, pending_bytes }) => {
+        Ok(ReceiveOutcome::CapacityDeferred {
+            missing,
+            pending_entries,
+            pending_bytes,
+        }) => {
             // Durable, but not buffered. The caller's obligation is to stop
             // reading from this peer; the envelope is admitted from the log
             // once the backlog drains.
@@ -178,16 +184,15 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let mut max_pending_entries: Option<usize> = None;
     let mut max_pending_bytes: Option<usize> = None;
     let mut authority_priority: u32 = 0;
+    let mut heartbeat_interval_ms: u64 = 1_000;
+    let mut heartbeat_failure_threshold: u32 = 5;
 
     let mut index = 1;
     while index < args.len() {
         match args[index].as_str() {
             "--node-id" => {
                 index += 1;
-                node_id = args
-                    .get(index)
-                    .ok_or("--node-id requires a value")?
-                    .clone();
+                node_id = args.get(index).ok_or("--node-id requires a value")?.clone();
             }
             "--data" => {
                 index += 1;
@@ -197,11 +202,15 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             }
             "--send" => {
                 index += 1;
-                send_specs.push(parse_link(args.get(index).ok_or("--send requires PEER=ADDR")?)?);
+                send_specs.push(parse_link(
+                    args.get(index).ok_or("--send requires PEER=ADDR")?,
+                )?);
             }
             "--recv" => {
                 index += 1;
-                recv_specs.push(parse_link(args.get(index).ok_or("--recv requires PEER=ADDR")?)?);
+                recv_specs.push(parse_link(
+                    args.get(index).ok_or("--recv requires PEER=ADDR")?,
+                )?);
             }
             // The bound is settable so an acceptance test can reach it through
             // the production path. Filling the shipping default would mean
@@ -237,6 +246,22 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                     .parse::<u32>()
                     .map_err(|e| format!("--authority-priority: {e}"))?;
             }
+            "--heartbeat-interval-ms" => {
+                index += 1;
+                heartbeat_interval_ms = args
+                    .get(index)
+                    .ok_or("--heartbeat-interval-ms requires a value")?
+                    .parse::<u64>()
+                    .map_err(|e| format!("--heartbeat-interval-ms: {e}"))?;
+            }
+            "--heartbeat-failure-threshold" => {
+                index += 1;
+                heartbeat_failure_threshold = args
+                    .get(index)
+                    .ok_or("--heartbeat-failure-threshold requires a value")?
+                    .parse::<u32>()
+                    .map_err(|e| format!("--heartbeat-failure-threshold: {e}"))?;
+            }
             other => return Err(format!("unknown argument {other}").into()),
         }
         index += 1;
@@ -282,7 +307,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     // recover its previous role. The cluster ID is set by the first `authority-claim`
     // command, so it starts as the node's own ID.
     let authority = Arc::new(Mutex::new(
-        AuthorityStore::open(
+        AuthorityStore::open_with_failure_detector(
             data_dir
                 .as_ref()
                 .map(|dir| dir.join("authority.json"))
@@ -290,13 +315,18 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             &node_id, // cluster_id defaults to node_id until set
             &node_id,
             authority_priority,
+            heartbeat_interval_ms,
+            heartbeat_failure_threshold,
         )
         .map_err(|e| e.to_string())?,
     ));
 
     let initial_state = StateHash::from_hex("0".repeat(64));
-    let mut executor =
-        DistributedTransactionExecutor::with_log(node_id.clone(), initial_state.clone(), log_path.clone())?;
+    let mut executor = DistributedTransactionExecutor::with_log(
+        node_id.clone(),
+        initial_state.clone(),
+        log_path.clone(),
+    )?;
 
     // The bound is set before recovery, not after, because recovery restores
     // the backlog and has to respect the same limit the live path does.
@@ -338,7 +368,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     // missed an operation observably divergent. It lives beside the executor
     // under one lock so a local commit and an arriving replication cannot
     // interleave and chain onto a parent that is already stale.
-    let core = Arc::new(Mutex::new((executor, StateVersion::new(VectorClock::new(), "0".repeat(64)))));
+    let recovered_version = executor
+        .get_replica_state(&node_id)
+        .map(|replica| StateVersion::new(replica.vector_clock, replica.state_hash.to_string()))
+        .unwrap_or_else(|| StateVersion::new(VectorClock::new(), "0".repeat(64)));
+    let core = Arc::new(Mutex::new((executor, recovered_version)));
     let should_exit = Arc::new(AtomicBool::new(false));
     // Per-peer delivery hold.
     //
@@ -367,7 +401,12 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         .into_iter()
         .map(|(peer, addr)| {
             let transport = TcpTransport::new("127.0.0.1:0".to_string());
-            SendLink { peer, addr, control: Arc::new(Mutex::new(transport.clone())), io: transport }
+            SendLink {
+                peer,
+                addr,
+                control: Arc::new(Mutex::new(transport.clone())),
+                io: transport,
+            }
         })
         .collect();
 
@@ -375,7 +414,12 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         .into_iter()
         .map(|(peer, addr)| {
             let transport = TcpTransport::new(addr.clone());
-            RecvLink { peer, addr, control: Arc::new(Mutex::new(transport.clone())), io: transport }
+            RecvLink {
+                peer,
+                addr,
+                control: Arc::new(Mutex::new(transport.clone())),
+                io: transport,
+            }
         })
         .collect();
 
@@ -509,8 +553,35 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         };
 
         match verb {
-            "execute-tx" => {
-                let record = if argument.is_empty() { "item".to_string() } else { argument };
+            "execute-tx" | "authority-write" => {
+                let record = if verb == "authority-write" {
+                    let mut parts = argument.splitn(2, ' ');
+                    let token = parts.next().and_then(|value| value.parse::<u64>().ok());
+                    let record = parts.next().unwrap_or("item").to_string();
+                    let validation = match token {
+                        Some(token) => authority
+                            .lock()
+                            .await
+                            .validate_quorum_write(FencingToken(token), 2),
+                        None => Err(feltdb::authority_failover::AuthorityError::StaleToken {
+                            provided: FencingToken(0),
+                            current: authority.lock().await.state().current_token(),
+                        }),
+                    };
+                    if let Err(error) = validation {
+                        println!(
+                            "AUTHORITY_WRITE_REJECTED {}",
+                            json!({ "record": record, "reason": error.to_string() })
+                        );
+                        io::stdout().flush().ok();
+                        continue;
+                    }
+                    record
+                } else if argument.is_empty() {
+                    "item".to_string()
+                } else {
+                    argument
+                };
 
                 let mut fields = HashMap::new();
                 fields.insert("record".to_string(), json!(record.clone()));
@@ -854,7 +925,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                 let (frontier, marks) = {
                     let guard = core.lock().await;
                     let (executor, _) = &*guard;
-                    (executor.causal_frontier().clone(), executor.origin_sequence_high_water())
+                    (
+                        executor.causal_frontier().clone(),
+                        executor.origin_sequence_high_water(),
+                    )
                 };
                 let view: Vec<String> = membership
                     .get_members()
@@ -1083,27 +1157,30 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             "list-operations" => {
                 let guard = core.lock().await;
                 let (executor, _) = &*guard;
-                
+
                 // Get all operations from the log
                 let mut all_ops = Vec::new();
                 let mut applied_ops = Vec::new();
                 let pending_ops = executor.pending_causal_keys();
                 let deferred_ops = executor.deferred_causal_keys();
-                
+
                 if let Some(ref log) = executor.operation_log {
                     if let Ok(envelopes) = log.load_all() {
                         for env in &envelopes {
-                            let key = format!("{}:{}", env.envelope_id.originating_node, env.envelope_id.sequence);
+                            let key = format!(
+                                "{}:{}",
+                                env.envelope_id.originating_node, env.envelope_id.sequence
+                            );
                             all_ops.push(key.clone());
                         }
                     }
                 }
-                
+
                 // Get applied operations from the barrier
                 for key in executor.causal_barrier.applied_keys() {
                     applied_ops.push(key.to_string());
                 }
-                
+
                 println!(
                     "OPERATIONS {}",
                     json!({
@@ -1120,23 +1197,26 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             "operation-inventory" => {
                 let guard = core.lock().await;
                 let (executor, _) = &*guard;
-                
+
                 // Get all operations from the log
                 let mut operations = json!({
                     "applied": [],
                     "pending": [],
                     "deferred": [],
                 });
-                
+
                 if let Some(ref log) = executor.operation_log {
                     if let Ok(envelopes) = log.load_all() {
                         let applied_keys = executor.causal_barrier.applied_keys();
                         let pending_keys = executor.pending_causal_keys();
                         let deferred_keys = executor.deferred_causal_keys();
-                        
+
                         for env in &envelopes {
-                            let key = format!("{}:{}", env.envelope_id.originating_node, env.envelope_id.sequence);
-                            
+                            let key = format!(
+                                "{}:{}",
+                                env.envelope_id.originating_node, env.envelope_id.sequence
+                            );
+
                             if applied_keys.contains(&key) {
                                 operations["applied"].as_array_mut().unwrap().push(json!({
                                     "id": key,
@@ -1159,7 +1239,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                         }
                     }
                 }
-                
+
                 println!("INVENTORY {}", operations);
                 io::stdout().flush().ok();
             }
@@ -1198,7 +1278,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                             })
                         );
                     }
-                    Ok(ClaimResult::Rejected { current_authority, current_token }) => {
+                    Ok(ClaimResult::Rejected {
+                        current_authority,
+                        current_token,
+                    }) => {
                         println!(
                             "AUTHORITY_REJECTED {}",
                             json!({
@@ -1230,11 +1313,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             "authority-accept" => {
                 let mut parts = argument.split_whitespace();
                 let authority_node = parts.next().unwrap_or_default().to_string();
-                let token_value: u64 = parts
-                    .next()
-                    .unwrap_or("0")
-                    .parse()
-                    .unwrap_or(0);
+                let token_value: u64 = parts.next().unwrap_or("0").parse().unwrap_or(0);
                 let token = FencingToken(token_value);
 
                 let mut guard = authority.lock().await;
@@ -1265,7 +1344,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                 let operations_applied = {
                     let core_guard = core.lock().await;
                     let (executor, _) = &*core_guard;
-                    executor.get_replica_state(&node_id).map(|r| r.operations_applied).unwrap_or(0)
+                    executor
+                        .get_replica_state(&node_id)
+                        .map(|r| r.operations_applied)
+                        .unwrap_or(0)
                 };
                 println!(
                     "AUTHORITY {}",
@@ -1291,7 +1373,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                 let from_node = parts.next().unwrap_or_default().to_string();
 
                 if from_node.is_empty() {
-                    println!("ERROR {}", json!({ "reason": "heartbeat requires NODE_ID" }));
+                    println!(
+                        "ERROR {}",
+                        json!({ "reason": "heartbeat requires NODE_ID" })
+                    );
                 } else {
                     let mut guard = authority.lock().await;
                     guard.heartbeat(&from_node);
@@ -1377,7 +1462,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             }
 
             _ => {
-                println!("ERROR {}", json!({ "reason": format!("unknown command {verb}") }));
+                println!(
+                    "ERROR {}",
+                    json!({ "reason": format!("unknown command {verb}") })
+                );
                 io::stdout().flush().ok();
             }
         }