@feltdb/core 0.7.2 → 0.7.3

package/dist/create/server-source/crates/feltdb/src/authority_failover.rs

@@ -0,0 +1,769 @@
+//! Authority Failover Protocol
+//!
+//! This module implements automatic authority election, failover, and recovery
+//! for distributed FeltDB clusters. The design follows these principles:
+//!
+//! 1. **Single Authority**: At any time, exactly one node holds authority.
+//!    Operations flow through the authority, which coordinates replication.
+//!
+//! 2. **Fencing**: A stale authority cannot accept writes after a new authority
+//!    is elected. Fencing tokens prevent split-brain scenarios.
+//!
+//! 3. **Automatic Failover**: When the authority fails, surviving nodes detect
+//!    the failure and elect a new authority without application intervention.
+//!
+//! 4. **Recovery**: A previously-failed authority can rejoin as a replica and
+//!    catch up to the current state.
+//!
+//! ## Consistency Contract
+//!
+//! - A write is **committed** when the authority has persisted it durably
+//!   and at least one other active replica has acknowledged receipt.
+//! - A committed write survives any single node failure.
+//! - During failover, a brief period of unavailability occurs while
+//!   the new authority is elected.
+//! - The protocol prevents two nodes from believing they are authority
+//!   simultaneously through monotonic fencing tokens.
+//!
+//! ## Failover Sequence
+//!
+//! ```text
+//! Authority A fails
+//!      │
+//!      ▼
+//! Replicas detect failure (heartbeat timeout)
+//!      │
+//!      ▼
+//! Election: highest-priority surviving replica
+//!      │
+//!      ▼
+//! New authority B claims with fencing token
+//!      │
+//!      ▼
+//! B broadcasts authority change
+//!      │
+//!      ▼
+//! Replicas acknowledge new authority
+//!      │
+//!      ▼
+//! B begins accepting writes
+//!      │
+//!      ▼
+//! Old authority A rejoins as replica
+//! ```
+
+use serde::{Deserialize, Serialize};
+use std::collections::{BTreeMap, HashSet};
+use std::path::{Path, PathBuf};
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+fn now_ms() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|elapsed| elapsed.as_millis() as u64)
+        .unwrap_or(0)
+}
+
+/// Authority role of a node in the cluster.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
+pub enum AuthorityRole {
+    /// This node is the authority and accepts writes.
+    Authority,
+    /// This node is a replica and forwards writes to the authority.
+    Replica,
+    /// This node is a candidate seeking to become authority.
+    Candidate,
+    /// This node has been fenced (stale authority).
+    Fenced,
+}
+
+/// A fencing token that prevents stale authorities from operating.
+///
+/// Monotonically increasing: a higher token always supersedes a lower one.
+/// The fencing token is durably stored and survives restarts.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+pub struct FencingToken(pub u64);
+
+impl FencingToken {
+    pub fn initial() -> Self {
+        Self(1)
+    }
+
+    pub fn next(&self) -> Self {
+        Self(self.0.saturating_add(1))
+    }
+
+    pub fn supersedes(&self, other: &FencingToken) -> bool {
+        self.0 > other.0
+    }
+}
+
+impl std::fmt::Display for FencingToken {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "FT:{}", self.0)
+    }
+}
+
+/// Record of an authority election.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct AuthorityRecord {
+    /// The node ID of the authority.
+    pub node_id: String,
+    /// The fencing token for this authority term.
+    pub fencing_token: FencingToken,
+    /// When this authority was elected.
+    pub elected_at_ms: u64,
+    /// The previous authority (if any).
+    pub previous_authority: Option<String>,
+    /// Nodes that acknowledged this authority.
+    pub acknowledgements: HashSet<String>,
+}
+
+impl AuthorityRecord {
+    pub fn new(node_id: String, fencing_token: FencingToken, previous: Option<String>) -> Self {
+        Self {
+            node_id,
+            fencing_token,
+            elected_at_ms: now_ms(),
+            previous_authority: previous,
+            acknowledgements: HashSet::new(),
+        }
+    }
+}
+
+/// Heartbeat tracking for failure detection.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct HeartbeatState {
+    /// Last heartbeat received from each peer.
+    pub last_seen: BTreeMap<String, u64>,
+    /// Heartbeat interval in milliseconds.
+    pub interval_ms: u64,
+    /// How many missed heartbeats before a node is considered failed.
+    pub failure_threshold: u32,
+}
+
+impl HeartbeatState {
+    pub fn new(interval_ms: u64, failure_threshold: u32) -> Self {
+        Self {
+            last_seen: BTreeMap::new(),
+            interval_ms,
+            failure_threshold,
+        }
+    }
+
+    /// Record a heartbeat from a peer.
+    pub fn record_heartbeat(&mut self, node_id: &str) {
+        self.last_seen.insert(node_id.to_string(), now_ms());
+    }
+
+    /// Check if a node appears to have failed.
+    pub fn is_failed(&self, node_id: &str) -> bool {
+        let Some(last) = self.last_seen.get(node_id) else {
+            // Never seen = not yet known, not necessarily failed
+            return false;
+        };
+        let deadline = self.interval_ms * self.failure_threshold as u64;
+        now_ms().saturating_sub(*last) > deadline
+    }
+
+    /// Get all nodes that appear to have failed.
+    pub fn failed_nodes(&self) -> Vec<String> {
+        self.last_seen
+            .keys()
+            .filter(|node_id| self.is_failed(node_id))
+            .cloned()
+            .collect()
+    }
+}
+
+/// Election priority for a node.
+/// Higher priority wins the election when multiple candidates exist.
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
+pub struct ElectionPriority {
+    /// Operations applied (higher is better - more up-to-date).
+    pub operations_applied: u64,
+    /// Static priority assigned to the node (for tie-breaking).
+    pub static_priority: u32,
+    /// Node ID (for deterministic tie-breaking).
+    pub node_id: String,
+}
+
+impl ElectionPriority {
+    pub fn new(operations_applied: u64, static_priority: u32, node_id: String) -> Self {
+        Self {
+            operations_applied,
+            static_priority,
+            node_id,
+        }
+    }
+}
+
+/// The result of an authority claim attempt.
+#[derive(Clone, Debug)]
+pub enum ClaimResult {
+    /// Successfully claimed authority with this fencing token.
+    Granted { token: FencingToken },
+    /// Another node already has a higher fencing token.
+    Rejected {
+        current_authority: String,
+        current_token: FencingToken,
+    },
+    /// This node is fenced and cannot claim authority.
+    Fenced { token: FencingToken },
+}
+
+/// Authority state that is durably persisted.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct AuthorityState {
+    /// The cluster ID this authority state belongs to.
+    pub cluster_id: String,
+    /// Current authority record, if any.
+    pub current_authority: Option<AuthorityRecord>,
+    /// This node's role.
+    pub local_role: AuthorityRole,
+    /// This node's ID.
+    pub local_node_id: String,
+    /// The highest fencing token this node has seen.
+    pub highest_seen_token: FencingToken,
+    /// Election priority for this node.
+    pub local_priority: u32,
+}
+
+impl AuthorityState {
+    pub fn new(cluster_id: String, local_node_id: String, local_priority: u32) -> Self {
+        Self {
+            cluster_id,
+            current_authority: None,
+            local_role: AuthorityRole::Replica,
+            local_node_id,
+            highest_seen_token: FencingToken::initial(),
+            local_priority,
+        }
+    }
+
+    /// Check if this node is currently the authority.
+    pub fn is_authority(&self) -> bool {
+        self.local_role == AuthorityRole::Authority
+    }
+
+    /// Check if this node is fenced.
+    pub fn is_fenced(&self) -> bool {
+        self.local_role == AuthorityRole::Fenced
+    }
+
+    /// Get the current authority node ID.
+    pub fn authority_node(&self) -> Option<&str> {
+        self.current_authority.as_ref().map(|r| r.node_id.as_str())
+    }
+
+    /// Get the current fencing token.
+    pub fn current_token(&self) -> FencingToken {
+        self.current_authority
+            .as_ref()
+            .map(|r| r.fencing_token)
+            .unwrap_or(FencingToken::initial())
+    }
+}
+
+/// Persistent authority store.
+pub struct AuthorityStore {
+    path: PathBuf,
+    state: AuthorityState,
+    heartbeats: HeartbeatState,
+}
+
+#[derive(Debug)]
+pub enum AuthorityError {
+    NoCluster,
+    AlreadyAuthority(String),
+    NotAuthority,
+    Fenced(FencingToken),
+    StaleToken {
+        provided: FencingToken,
+        current: FencingToken,
+    },
+    Io(String),
+}
+
+impl std::fmt::Display for AuthorityError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::NoCluster => write!(f, "no cluster has been created"),
+            Self::AlreadyAuthority(id) => write!(f, "node {id} is already authority"),
+            Self::NotAuthority => write!(f, "this node is not the authority"),
+            Self::Fenced(token) => write!(f, "this node is fenced at {token}"),
+            Self::StaleToken { provided, current } => {
+                write!(f, "stale token {provided}, current is {current}")
+            }
+            Self::Io(msg) => write!(f, "authority store: {msg}"),
+        }
+    }
+}
+
+impl AuthorityStore {
+    /// Open or create an authority store.
+    pub fn open<P: AsRef<Path>>(
+        path: P,
+        cluster_id: &str,
+        local_node_id: &str,
+        local_priority: u32,
+    ) -> Result<Self, AuthorityError> {
+        let path = path.as_ref().to_path_buf();
+        let state = if path.exists() {
+            let bytes = std::fs::read(&path).map_err(|e| AuthorityError::Io(e.to_string()))?;
+            serde_json::from_slice(&bytes).map_err(|e| AuthorityError::Io(e.to_string()))?
+        } else {
+            AuthorityState::new(
+                cluster_id.to_string(),
+                local_node_id.to_string(),
+                local_priority,
+            )
+        };
+        Ok(Self {
+            path,
+            state,
+            heartbeats: HeartbeatState::new(1000, 5), // 1s heartbeat, 5 missed = failure
+        })
+    }
+
+    /// Persist the authority state durably.
+    fn persist(&self) -> Result<(), AuthorityError> {
+        if let Some(parent) = self.path.parent() {
+            std::fs::create_dir_all(parent).map_err(|e| AuthorityError::Io(e.to_string()))?;
+        }
+        let temp = self.path.with_extension("tmp");
+        let bytes =
+            serde_json::to_vec_pretty(&self.state).map_err(|e| AuthorityError::Io(e.to_string()))?;
+        {
+            use std::io::Write;
+            let mut file =
+                std::fs::File::create(&temp).map_err(|e| AuthorityError::Io(e.to_string()))?;
+            file.write_all(&bytes)
+                .map_err(|e| AuthorityError::Io(e.to_string()))?;
+            file.sync_all()
+                .map_err(|e| AuthorityError::Io(e.to_string()))?;
+        }
+        std::fs::rename(&temp, &self.path).map_err(|e| AuthorityError::Io(e.to_string()))?;
+        if let Some(parent) = self.path.parent() {
+            if let Ok(dir) = std::fs::File::open(parent) {
+                let _ = dir.sync_all();
+            }
+        }
+        Ok(())
+    }
+
+    /// Get the current authority state.
+    pub fn state(&self) -> &AuthorityState {
+        &self.state
+    }
+
+    /// Record a heartbeat from a peer.
+    pub fn heartbeat(&mut self, node_id: &str) {
+        self.heartbeats.record_heartbeat(node_id);
+    }
+
+    /// Check if the current authority has failed.
+    pub fn authority_failed(&self) -> bool {
+        if let Some(ref authority) = self.state.current_authority {
+            if authority.node_id == self.state.local_node_id {
+                return false; // We are authority, we haven't failed
+            }
+            return self.heartbeats.is_failed(&authority.node_id);
+        }
+        false
+    }
+
+    /// Get all nodes that appear to have failed.
+    pub fn failed_nodes(&self) -> Vec<String> {
+        self.heartbeats.failed_nodes()
+    }
+
+    /// Claim authority with a new fencing token.
+    ///
+    /// This node attempts to become the authority. If successful:
+    /// - A new fencing token is assigned (one higher than any seen)
+    /// - This node becomes authority
+    /// - The previous authority (if any) is recorded
+    ///
+    /// If another node has a higher token, the claim is rejected.
+    pub fn claim_authority(&mut self) -> Result<ClaimResult, AuthorityError> {
+        if self.state.is_fenced() {
+            return Ok(ClaimResult::Fenced {
+                token: self.state.highest_seen_token,
+            });
+        }
+
+        // If there's already an authority with a higher token, reject
+        if let Some(ref current) = self.state.current_authority {
+            if current.node_id != self.state.local_node_id
+                && !self.heartbeats.is_failed(&current.node_id)
+            {
+                return Ok(ClaimResult::Rejected {
+                    current_authority: current.node_id.clone(),
+                    current_token: current.fencing_token,
+                });
+            }
+        }
+
+        // Claim with a new token
+        let new_token = self.state.highest_seen_token.next();
+        let previous = self
+            .state
+            .current_authority
+            .as_ref()
+            .map(|r| r.node_id.clone());
+
+        let mut record =
+            AuthorityRecord::new(self.state.local_node_id.clone(), new_token, previous);
+        record.acknowledgements.insert(self.state.local_node_id.clone());
+
+        self.state.current_authority = Some(record);
+        self.state.local_role = AuthorityRole::Authority;
+        self.state.highest_seen_token = new_token;
+
+        self.persist()?;
+
+        Ok(ClaimResult::Granted { token: new_token })
+    }
+
+    /// Accept a new authority announcement.
+    ///
+    /// If the fencing token is higher than any we've seen, accept the new
+    /// authority and update our role accordingly.
+    pub fn accept_authority(
+        &mut self,
+        authority_node: &str,
+        token: FencingToken,
+    ) -> Result<(), AuthorityError> {
+        // Check if token is valid
+        if !token.supersedes(&self.state.highest_seen_token)
+            && token != self.state.highest_seen_token
+        {
+            return Err(AuthorityError::StaleToken {
+                provided: token,
+                current: self.state.highest_seen_token,
+            });
+        }
+
+        // Update our state
+        self.state.highest_seen_token = token;
+
+        // If we were authority with a lower token, we are now fenced
+        if self.state.local_role == AuthorityRole::Authority
+            && authority_node != self.state.local_node_id
+        {
+            self.state.local_role = AuthorityRole::Fenced;
+        } else if authority_node != self.state.local_node_id {
+            self.state.local_role = AuthorityRole::Replica;
+        }
+
+        // Record the new authority
+        self.state.current_authority = Some(AuthorityRecord::new(
+            authority_node.to_string(),
+            token,
+            self.state
+                .current_authority
+                .as_ref()
+                .map(|r| r.node_id.clone()),
+        ));
+
+        // Start tracking heartbeats from the new authority.
+        // This allows failure detection to work even if we've never received
+        // a heartbeat yet - the clock starts now.
+        if authority_node != self.state.local_node_id {
+            self.heartbeats.record_heartbeat(authority_node);
+        }
+
+        self.persist()
+    }
+
+    /// Acknowledge the current authority.
+    pub fn acknowledge_authority(&mut self, from_node: &str) -> Result<(), AuthorityError> {
+        if let Some(ref mut authority) = self.state.current_authority {
+            authority.acknowledgements.insert(from_node.to_string());
+            self.persist()?;
+        }
+        Ok(())
+    }
+
+    /// Validate a write operation against the current fencing token.
+    ///
+    /// Returns Ok if the token is valid (current or not provided for local writes).
+    /// Returns Err if the token is stale.
+    pub fn validate_write(&self, token: Option<FencingToken>) -> Result<(), AuthorityError> {
+        if self.state.is_fenced() {
+            return Err(AuthorityError::Fenced(self.state.highest_seen_token));
+        }
+
+        if !self.state.is_authority() {
+            return Err(AuthorityError::NotAuthority);
+        }
+
+        if let Some(provided) = token {
+            if provided != self.state.current_token() {
+                return Err(AuthorityError::StaleToken {
+                    provided,
+                    current: self.state.current_token(),
+                });
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Step down from authority role (voluntary).
+    pub fn step_down(&mut self) -> Result<(), AuthorityError> {
+        if self.state.local_role == AuthorityRole::Authority {
+            self.state.local_role = AuthorityRole::Replica;
+            self.persist()?;
+        }
+        Ok(())
+    }
+
+    /// Recover from fenced state (after catching up).
+    pub fn recover_from_fenced(&mut self) -> Result<(), AuthorityError> {
+        if self.state.local_role == AuthorityRole::Fenced {
+            self.state.local_role = AuthorityRole::Replica;
+            self.persist()?;
+        }
+        Ok(())
+    }
+
+    /// Compute election priority for this node.
+    pub fn election_priority(&self, operations_applied: u64) -> ElectionPriority {
+        ElectionPriority::new(
+            operations_applied,
+            self.state.local_priority,
+            self.state.local_node_id.clone(),
+        )
+    }
+
+    /// Determine the winner of an election given a set of candidates.
+    pub fn elect_authority(candidates: &[ElectionPriority]) -> Option<&ElectionPriority> {
+        candidates.iter().max()
+    }
+}
+
+/// Authority announcement message.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct AuthorityAnnouncement {
+    pub authority_node: String,
+    pub fencing_token: FencingToken,
+    pub elected_at_ms: u64,
+    pub cluster_id: String,
+}
+
+/// Heartbeat message.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Heartbeat {
+    pub from_node: String,
+    pub is_authority: bool,
+    pub fencing_token: FencingToken,
+    pub operations_applied: u64,
+    pub timestamp_ms: u64,
+}
+
+impl Heartbeat {
+    pub fn new(
+        from_node: String,
+        is_authority: bool,
+        fencing_token: FencingToken,
+        operations_applied: u64,
+    ) -> Self {
+        Self {
+            from_node,
+            is_authority,
+            fencing_token,
+            operations_applied,
+            timestamp_ms: now_ms(),
+        }
+    }
+}
+
+/// Election request message.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct ElectionRequest {
+    pub candidate_node: String,
+    pub priority: ElectionPriority,
+    pub proposed_token: FencingToken,
+    pub timestamp_ms: u64,
+}
+
+/// Election response message.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct ElectionResponse {
+    pub from_node: String,
+    pub vote_granted: bool,
+    pub reason: String,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn fencing_token_ordering() {
+        let t1 = FencingToken::initial();
+        let t2 = t1.next();
+        let t3 = t2.next();
+
+        assert!(t2.supersedes(&t1));
+        assert!(t3.supersedes(&t2));
+        assert!(t3.supersedes(&t1));
+        assert!(!t1.supersedes(&t2));
+    }
+
+    #[test]
+    fn authority_claim_and_fencing() {
+        let dir = tempfile::tempdir().unwrap();
+        let path = dir.path().join("authority.json");
+
+        // Node A claims authority
+        let mut store_a =
+            AuthorityStore::open(&path, "test-cluster", "A", 10).expect("open store A");
+        let result = store_a.claim_authority().expect("claim");
+        match result {
+            ClaimResult::Granted { token } => {
+                assert_eq!(token, FencingToken(2)); // initial is 1, next is 2
+            }
+            _ => panic!("expected Granted"),
+        }
+        assert!(store_a.state().is_authority());
+
+        // Node B sees A's authority
+        let path_b = dir.path().join("authority_b.json");
+        let mut store_b =
+            AuthorityStore::open(&path_b, "test-cluster", "B", 5).expect("open store B");
+        store_b
+            .accept_authority("A", FencingToken(2))
+            .expect("accept");
+        assert!(!store_b.state().is_authority());
+        assert_eq!(store_b.state().authority_node(), Some("A"));
+
+        // Node B tries to claim - should be rejected since A is still up
+        let result_b = store_b.claim_authority().expect("claim B");
+        match result_b {
+            ClaimResult::Rejected {
+                current_authority,
+                current_token,
+            } => {
+                assert_eq!(current_authority, "A");
+                assert_eq!(current_token, FencingToken(2));
+            }
+            _ => panic!("expected Rejected"),
+        }
+    }
+
+    #[test]
+    fn failover_when_authority_dies() {
+        let dir = tempfile::tempdir().unwrap();
+
+        // Node A is authority
+        let path_a = dir.path().join("authority_a.json");
+        let mut store_a =
+            AuthorityStore::open(&path_a, "test-cluster", "A", 10).expect("open store A");
+        store_a.claim_authority().expect("claim A");
+
+        // Node B knows about A
+        let path_b = dir.path().join("authority_b.json");
+        let mut store_b =
+            AuthorityStore::open(&path_b, "test-cluster", "B", 5).expect("open store B");
+        store_b
+            .accept_authority("A", FencingToken(2))
+            .expect("accept A");
+
+        // Simulate A dying - mark heartbeat as very old
+        store_b.heartbeats.last_seen.insert("A".to_string(), 0);
+
+        // Now B should be able to claim
+        assert!(store_b.authority_failed());
+        let result_b = store_b.claim_authority().expect("claim B after failover");
+        match result_b {
+            ClaimResult::Granted { token } => {
+                assert_eq!(token, FencingToken(3)); // higher than A's token
+            }
+            _ => panic!("expected Granted after failover"),
+        }
+    }
+
+    #[test]
+    fn stale_authority_is_fenced() {
+        let dir = tempfile::tempdir().unwrap();
+
+        // A is authority with token 2
+        let path_a = dir.path().join("authority_a.json");
+        let mut store_a =
+            AuthorityStore::open(&path_a, "test-cluster", "A", 10).expect("open store A");
+        store_a.claim_authority().expect("claim A");
+        assert!(store_a.state().is_authority());
+
+        // A receives announcement that B is now authority with token 3
+        store_a
+            .accept_authority("B", FencingToken(3))
+            .expect("accept B");
+
+        // A should now be fenced
+        assert!(store_a.state().is_fenced());
+        assert!(!store_a.state().is_authority());
+
+        // A's writes should be rejected
+        let result = store_a.validate_write(None);
+        assert!(matches!(result, Err(AuthorityError::Fenced(_))));
+    }
+
+    #[test]
+    fn election_priority_ordering() {
+        let p1 = ElectionPriority::new(100, 5, "A".to_string());
+        let p2 = ElectionPriority::new(100, 10, "B".to_string());
+        let p3 = ElectionPriority::new(150, 5, "C".to_string());
+
+        // Higher operations wins
+        assert!(p3 > p1);
+        assert!(p3 > p2);
+
+        // Same operations, higher priority wins
+        assert!(p2 > p1);
+
+        // Election selects highest priority
+        let candidates = vec![p1.clone(), p2.clone(), p3.clone()];
+        let winner = AuthorityStore::elect_authority(&candidates);
+        assert_eq!(winner.map(|w| &w.node_id), Some(&"C".to_string()));
+    }
+
+    #[test]
+    fn heartbeat_failure_detection() {
+        let mut heartbeats = HeartbeatState::new(1000, 3); // 1s interval, 3 missed = failure
+
+        // Record a heartbeat
+        heartbeats.record_heartbeat("A");
+        assert!(!heartbeats.is_failed("A"));
+
+        // Manually set an old timestamp
+        heartbeats.last_seen.insert("A".to_string(), 0);
+
+        // Should now be detected as failed (current time is way past 3 intervals)
+        assert!(heartbeats.is_failed("A"));
+    }
+
+    #[test]
+    fn recovery_from_fenced() {
+        let dir = tempfile::tempdir().unwrap();
+        let path = dir.path().join("authority.json");
+
+        let mut store =
+            AuthorityStore::open(&path, "test-cluster", "A", 10).expect("open store");
+        store.claim_authority().expect("claim");
+
+        // Get fenced
+        store
+            .accept_authority("B", FencingToken(3))
+            .expect("accept B");
+        assert!(store.state().is_fenced());
+
+        // Recover
+        store.recover_from_fenced().expect("recover");
+        assert!(!store.state().is_fenced());
+        assert_eq!(store.state().local_role, AuthorityRole::Replica);
+    }
+}