@feltdb/core 0.7.2 → 0.7.4

package/dist/create/server-source/crates/feltdb/src/bin/feltdb_node.rs

@@ -45,14 +45,17 @@
 //! shutdown            exit cleanly
 //! ```
 
+use feltdb::authority_failover::{AuthorityStore, ClaimResult, FencingToken, Heartbeat};
 use feltdb::convergence::VectorClock;
 use feltdb::distributed_transactions::{
     CausalCapacity, DistributedTransactionExecutor, ReceiveOutcome, ReplicationMessage,
     SubmitOutcome,
 };
-use feltdb::replication_protocol::ProtocolTransport;
-use feltdb::replica_acknowledgements::{AcknowledgementStore, ReclamationEvidence, ReplicaAcknowledgement};
+use feltdb::replica_acknowledgements::{
+    AcknowledgementStore, ReclamationEvidence, ReplicaAcknowledgement,
+};
 use feltdb::replica_membership::MembershipStore;
+use feltdb::replication_protocol::ProtocolTransport;
 use feltdb::state_hash::StateHash;
 use feltdb::tcp_transport::TcpTransport;
 use feltdb::transactions::{
@@ -126,7 +129,11 @@ fn deliver(
         Ok(ReceiveOutcome::AlreadyKnown) => {
             json!({ "envelope": key, "outcome": "already_known" })
         }
-        Ok(ReceiveOutcome::CapacityDeferred { missing, pending_entries, pending_bytes }) => {
+        Ok(ReceiveOutcome::CapacityDeferred {
+            missing,
+            pending_entries,
+            pending_bytes,
+        }) => {
             // Durable, but not buffered. The caller's obligation is to stop
             // reading from this peer; the envelope is admitted from the log
             // once the backlog drains.
@@ -176,16 +183,16 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let mut recv_specs: Vec<(String, String)> = Vec::new();
     let mut max_pending_entries: Option<usize> = None;
     let mut max_pending_bytes: Option<usize> = None;
+    let mut authority_priority: u32 = 0;
+    let mut heartbeat_interval_ms: u64 = 1_000;
+    let mut heartbeat_failure_threshold: u32 = 5;
 
     let mut index = 1;
     while index < args.len() {
         match args[index].as_str() {
             "--node-id" => {
                 index += 1;
-                node_id = args
-                    .get(index)
-                    .ok_or("--node-id requires a value")?
-                    .clone();
+                node_id = args.get(index).ok_or("--node-id requires a value")?.clone();
             }
             "--data" => {
                 index += 1;
@@ -195,11 +202,15 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             }
             "--send" => {
                 index += 1;
-                send_specs.push(parse_link(args.get(index).ok_or("--send requires PEER=ADDR")?)?);
+                send_specs.push(parse_link(
+                    args.get(index).ok_or("--send requires PEER=ADDR")?,
+                )?);
             }
             "--recv" => {
                 index += 1;
-                recv_specs.push(parse_link(args.get(index).ok_or("--recv requires PEER=ADDR")?)?);
+                recv_specs.push(parse_link(
+                    args.get(index).ok_or("--recv requires PEER=ADDR")?,
+                )?);
             }
             // The bound is settable so an acceptance test can reach it through
             // the production path. Filling the shipping default would mean
@@ -225,6 +236,32 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                         .map_err(|e| format!("--max-pending-bytes: {e}"))?,
                 );
             }
+            // Static priority for authority election. Higher priority nodes are
+            // preferred during failover when multiple candidates have equal state.
+            "--authority-priority" => {
+                index += 1;
+                authority_priority = args
+                    .get(index)
+                    .ok_or("--authority-priority requires a value")?
+                    .parse::<u32>()
+                    .map_err(|e| format!("--authority-priority: {e}"))?;
+            }
+            "--heartbeat-interval-ms" => {
+                index += 1;
+                heartbeat_interval_ms = args
+                    .get(index)
+                    .ok_or("--heartbeat-interval-ms requires a value")?
+                    .parse::<u64>()
+                    .map_err(|e| format!("--heartbeat-interval-ms: {e}"))?;
+            }
+            "--heartbeat-failure-threshold" => {
+                index += 1;
+                heartbeat_failure_threshold = args
+                    .get(index)
+                    .ok_or("--heartbeat-failure-threshold requires a value")?
+                    .parse::<u32>()
+                    .map_err(|e| format!("--heartbeat-failure-threshold: {e}"))?;
+            }
             other => return Err(format!("unknown argument {other}").into()),
         }
         index += 1;
@@ -265,9 +302,31 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         .map_err(|e| e.to_string())?,
     ));
 
+    // Authority state tracks who holds write authority in the cluster. The store
+    // is durable and survives restarts, so a node that crashes can rejoin and
+    // recover its previous role. The cluster ID is set by the first `authority-claim`
+    // command, so it starts as the node's own ID.
+    let authority = Arc::new(Mutex::new(
+        AuthorityStore::open_with_failure_detector(
+            data_dir
+                .as_ref()
+                .map(|dir| dir.join("authority.json"))
+                .unwrap_or_else(|| PathBuf::from("authority.json")),
+            &node_id, // cluster_id defaults to node_id until set
+            &node_id,
+            authority_priority,
+            heartbeat_interval_ms,
+            heartbeat_failure_threshold,
+        )
+        .map_err(|e| e.to_string())?,
+    ));
+
     let initial_state = StateHash::from_hex("0".repeat(64));
-    let mut executor =
-        DistributedTransactionExecutor::with_log(node_id.clone(), initial_state.clone(), log_path.clone())?;
+    let mut executor = DistributedTransactionExecutor::with_log(
+        node_id.clone(),
+        initial_state.clone(),
+        log_path.clone(),
+    )?;
 
     // The bound is set before recovery, not after, because recovery restores
     // the backlog and has to respect the same limit the live path does.
@@ -309,7 +368,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     // missed an operation observably divergent. It lives beside the executor
     // under one lock so a local commit and an arriving replication cannot
     // interleave and chain onto a parent that is already stale.
-    let core = Arc::new(Mutex::new((executor, StateVersion::new(VectorClock::new(), "0".repeat(64)))));
+    let recovered_version = executor
+        .get_replica_state(&node_id)
+        .map(|replica| StateVersion::new(replica.vector_clock, replica.state_hash.to_string()))
+        .unwrap_or_else(|| StateVersion::new(VectorClock::new(), "0".repeat(64)));
+    let core = Arc::new(Mutex::new((executor, recovered_version)));
     let should_exit = Arc::new(AtomicBool::new(false));
     // Per-peer delivery hold.
     //
@@ -338,7 +401,12 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         .into_iter()
         .map(|(peer, addr)| {
             let transport = TcpTransport::new("127.0.0.1:0".to_string());
-            SendLink { peer, addr, control: Arc::new(Mutex::new(transport.clone())), io: transport }
+            SendLink {
+                peer,
+                addr,
+                control: Arc::new(Mutex::new(transport.clone())),
+                io: transport,
+            }
         })
         .collect();
 
@@ -346,7 +414,12 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         .into_iter()
         .map(|(peer, addr)| {
             let transport = TcpTransport::new(addr.clone());
-            RecvLink { peer, addr, control: Arc::new(Mutex::new(transport.clone())), io: transport }
+            RecvLink {
+                peer,
+                addr,
+                control: Arc::new(Mutex::new(transport.clone())),
+                io: transport,
+            }
         })
         .collect();
 
@@ -480,8 +553,35 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         };
 
         match verb {
-            "execute-tx" => {
-                let record = if argument.is_empty() { "item".to_string() } else { argument };
+            "execute-tx" | "authority-write" => {
+                let record = if verb == "authority-write" {
+                    let mut parts = argument.splitn(2, ' ');
+                    let token = parts.next().and_then(|value| value.parse::<u64>().ok());
+                    let record = parts.next().unwrap_or("item").to_string();
+                    let validation = match token {
+                        Some(token) => authority
+                            .lock()
+                            .await
+                            .validate_quorum_write(FencingToken(token), 2),
+                        None => Err(feltdb::authority_failover::AuthorityError::StaleToken {
+                            provided: FencingToken(0),
+                            current: authority.lock().await.state().current_token(),
+                        }),
+                    };
+                    if let Err(error) = validation {
+                        println!(
+                            "AUTHORITY_WRITE_REJECTED {}",
+                            json!({ "record": record, "reason": error.to_string() })
+                        );
+                        io::stdout().flush().ok();
+                        continue;
+                    }
+                    record
+                } else if argument.is_empty() {
+                    "item".to_string()
+                } else {
+                    argument
+                };
 
                 let mut fields = HashMap::new();
                 fields.insert("record".to_string(), json!(record.clone()));
@@ -825,7 +925,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                 let (frontier, marks) = {
                     let guard = core.lock().await;
                     let (executor, _) = &*guard;
-                    (executor.causal_frontier().clone(), executor.origin_sequence_high_water())
+                    (
+                        executor.causal_frontier().clone(),
+                        executor.origin_sequence_high_water(),
+                    )
                 };
                 let view: Vec<String> = membership
                     .get_members()
@@ -1054,27 +1157,30 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             "list-operations" => {
                 let guard = core.lock().await;
                 let (executor, _) = &*guard;
-                
+
                 // Get all operations from the log
                 let mut all_ops = Vec::new();
                 let mut applied_ops = Vec::new();
                 let pending_ops = executor.pending_causal_keys();
                 let deferred_ops = executor.deferred_causal_keys();
-                
+
                 if let Some(ref log) = executor.operation_log {
                     if let Ok(envelopes) = log.load_all() {
                         for env in &envelopes {
-                            let key = format!("{}:{}", env.envelope_id.originating_node, env.envelope_id.sequence);
+                            let key = format!(
+                                "{}:{}",
+                                env.envelope_id.originating_node, env.envelope_id.sequence
+                            );
                             all_ops.push(key.clone());
                         }
                     }
                 }
-                
+
                 // Get applied operations from the barrier
                 for key in executor.causal_barrier.applied_keys() {
                     applied_ops.push(key.to_string());
                 }
-                
+
                 println!(
                     "OPERATIONS {}",
                     json!({
@@ -1091,23 +1197,26 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             "operation-inventory" => {
                 let guard = core.lock().await;
                 let (executor, _) = &*guard;
-                
+
                 // Get all operations from the log
                 let mut operations = json!({
                     "applied": [],
                     "pending": [],
                     "deferred": [],
                 });
-                
+
                 if let Some(ref log) = executor.operation_log {
                     if let Ok(envelopes) = log.load_all() {
                         let applied_keys = executor.causal_barrier.applied_keys();
                         let pending_keys = executor.pending_causal_keys();
                         let deferred_keys = executor.deferred_causal_keys();
-                        
+
                         for env in &envelopes {
-                            let key = format!("{}:{}", env.envelope_id.originating_node, env.envelope_id.sequence);
-                            
+                            let key = format!(
+                                "{}:{}",
+                                env.envelope_id.originating_node, env.envelope_id.sequence
+                            );
+
                             if applied_keys.contains(&key) {
                                 operations["applied"].as_array_mut().unwrap().push(json!({
                                     "id": key,
@@ -1130,7 +1239,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                         }
                     }
                 }
-                
+
                 println!("INVENTORY {}", operations);
                 io::stdout().flush().ok();
             }
@@ -1147,6 +1256,203 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                 io::stdout().flush().ok();
             }
 
+            // Authority failover commands.
+            //
+            // These commands implement the distributed authority protocol that prevents
+            // split-brain and allows automatic failover when the authority node fails.
+
+            // Claim authority for this node. This generates a new fencing token that
+            // supersedes any previous authority. The claim succeeds only if no other
+            // node has a higher token and appears healthy.
+            "authority-claim" => {
+                let mut guard = authority.lock().await;
+                let result = guard.claim_authority();
+                match result {
+                    Ok(ClaimResult::Granted { token }) => {
+                        println!(
+                            "AUTHORITY_CLAIMED {}",
+                            json!({
+                                "node": node_id,
+                                "fencing_token": token.0,
+                                "role": "AUTHORITY",
+                            })
+                        );
+                    }
+                    Ok(ClaimResult::Rejected {
+                        current_authority,
+                        current_token,
+                    }) => {
+                        println!(
+                            "AUTHORITY_REJECTED {}",
+                            json!({
+                                "current_authority": current_authority,
+                                "current_token": current_token.0,
+                                "reason": "another authority is still active",
+                            })
+                        );
+                    }
+                    Ok(ClaimResult::Fenced { token }) => {
+                        println!(
+                            "AUTHORITY_FENCED {}",
+                            json!({
+                                "node": node_id,
+                                "fencing_token": token.0,
+                                "reason": "this node is fenced",
+                            })
+                        );
+                    }
+                    Err(e) => {
+                        println!("ERROR {}", json!({ "reason": e.to_string() }));
+                    }
+                }
+                io::stdout().flush().ok();
+            }
+
+            // Accept a new authority announced by another node.
+            // Format: authority-accept NODE_ID TOKEN
+            "authority-accept" => {
+                let mut parts = argument.split_whitespace();
+                let authority_node = parts.next().unwrap_or_default().to_string();
+                let token_value: u64 = parts.next().unwrap_or("0").parse().unwrap_or(0);
+                let token = FencingToken(token_value);
+
+                let mut guard = authority.lock().await;
+                let result = guard.accept_authority(&authority_node, token);
+                match result {
+                    Ok(()) => {
+                        let state = guard.state();
+                        println!(
+                            "AUTHORITY_ACCEPTED {}",
+                            json!({
+                                "authority_node": authority_node,
+                                "fencing_token": token.0,
+                                "local_role": format!("{:?}", state.local_role),
+                            })
+                        );
+                    }
+                    Err(e) => {
+                        println!("ERROR {}", json!({ "reason": e.to_string() }));
+                    }
+                }
+                io::stdout().flush().ok();
+            }
+
+            // Query the current authority state.
+            "authority-state" => {
+                let guard = authority.lock().await;
+                let state = guard.state();
+                let operations_applied = {
+                    let core_guard = core.lock().await;
+                    let (executor, _) = &*core_guard;
+                    executor
+                        .get_replica_state(&node_id)
+                        .map(|r| r.operations_applied)
+                        .unwrap_or(0)
+                };
+                println!(
+                    "AUTHORITY {}",
+                    json!({
+                        "cluster_id": state.cluster_id,
+                        "local_node": state.local_node_id,
+                        "local_role": format!("{:?}", state.local_role),
+                        "is_authority": state.is_authority(),
+                        "is_fenced": state.is_fenced(),
+                        "authority_node": state.authority_node(),
+                        "current_token": state.current_token().0,
+                        "highest_seen_token": state.highest_seen_token.0,
+                        "operations_applied": operations_applied,
+                    })
+                );
+                io::stdout().flush().ok();
+            }
+
+            // Record a heartbeat from a peer. This is used for failure detection.
+            // Format: heartbeat NODE_ID [TOKEN] [OPS_APPLIED] [IS_AUTHORITY]
+            "heartbeat" => {
+                let mut parts = argument.split_whitespace();
+                let from_node = parts.next().unwrap_or_default().to_string();
+
+                if from_node.is_empty() {
+                    println!(
+                        "ERROR {}",
+                        json!({ "reason": "heartbeat requires NODE_ID" })
+                    );
+                } else {
+                    let mut guard = authority.lock().await;
+                    guard.heartbeat(&from_node);
+                    println!(
+                        "HEARTBEAT_RECORDED {}",
+                        json!({
+                            "from_node": from_node,
+                        })
+                    );
+                }
+                io::stdout().flush().ok();
+            }
+
+            // Check if the current authority has failed based on heartbeat timeout.
+            "authority-failed" => {
+                let guard = authority.lock().await;
+                let failed = guard.authority_failed();
+                let failed_nodes = guard.failed_nodes();
+                let state = guard.state();
+                println!(
+                    "AUTHORITY_CHECK {}",
+                    json!({
+                        "authority_failed": failed,
+                        "current_authority": state.authority_node(),
+                        "failed_nodes": failed_nodes,
+                    })
+                );
+                io::stdout().flush().ok();
+            }
+
+            // Step down from authority role voluntarily.
+            "authority-stepdown" => {
+                let mut guard = authority.lock().await;
+                let was_authority = guard.state().is_authority();
+                let result = guard.step_down();
+                match result {
+                    Ok(()) => {
+                        println!(
+                            "AUTHORITY_STEPDOWN {}",
+                            json!({
+                                "node": node_id,
+                                "was_authority": was_authority,
+                                "now_role": format!("{:?}", guard.state().local_role),
+                            })
+                        );
+                    }
+                    Err(e) => {
+                        println!("ERROR {}", json!({ "reason": e.to_string() }));
+                    }
+                }
+                io::stdout().flush().ok();
+            }
+
+            // Recover from fenced state after catching up.
+            "authority-recover" => {
+                let mut guard = authority.lock().await;
+                let was_fenced = guard.state().is_fenced();
+                let result = guard.recover_from_fenced();
+                match result {
+                    Ok(()) => {
+                        println!(
+                            "AUTHORITY_RECOVERED {}",
+                            json!({
+                                "node": node_id,
+                                "was_fenced": was_fenced,
+                                "now_role": format!("{:?}", guard.state().local_role),
+                            })
+                        );
+                    }
+                    Err(e) => {
+                        println!("ERROR {}", json!({ "reason": e.to_string() }));
+                    }
+                }
+                io::stdout().flush().ok();
+            }
+
             "shutdown" => {
                 should_exit.store(true, Ordering::Relaxed);
                 println!("SHUTDOWN");
@@ -1156,7 +1462,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             }
 
             _ => {
-                println!("ERROR {}", json!({ "reason": format!("unknown command {verb}") }));
+                println!(
+                    "ERROR {}",
+                    json!({ "reason": format!("unknown command {verb}") })
+                );
                 io::stdout().flush().ok();
             }
         }