@feltdb/core 0.15.0 → 0.16.0

package/dist/create/server-source/crates/feltdb/src/phase_2c_cascading.rs

@@ -1,372 +0,0 @@
-/// Phase 2c: Cascading Failures
-///
-/// Tests behavior under multiple simultaneous failures:
-/// - 2 nodes crash simultaneously
-/// - Majority quorum maintains consistency
-/// - Recovery to consistent state
-///
-/// Success Criteria:
-/// - System continues with majority quorum alive
-/// - Restarted nodes converge to consistent state
-/// - No data loss
-/// - No silent divergence
-
-#[cfg(test)]
-mod tests {
-    use crate::convergence::VectorClock;
-    use crate::distributed_transactions::{
-        DistributedTransactionExecutor, ReplicationMessage, TransactionEnvelope,
-    };
-    use crate::state_hash::StateHash;
-    use crate::transactions::{
-        ConsistencyContract, Operation, OperationCommand, OperationId, StateVersion,
-    };
-    use serde_json::json;
-    use std::collections::HashMap;
-    use std::sync::atomic::{AtomicBool, Ordering};
-    use std::sync::Arc;
-    use tokio::sync::Mutex;
-
-    struct CascadingNode {
-        node_id: String,
-        executor: Arc<Mutex<DistributedTransactionExecutor>>,
-        sequence: Arc<Mutex<u64>>,
-        is_failed: Arc<AtomicBool>,
-    }
-
-    impl CascadingNode {
-        fn new(node_id: String) -> Self {
-            let initial_hash = StateHash::from_hex("0".repeat(64));
-            let executor = DistributedTransactionExecutor::new(node_id.clone(), initial_hash);
-
-            Self {
-                node_id,
-                executor: Arc::new(Mutex::new(executor)),
-                sequence: Arc::new(Mutex::new(0)),
-                is_failed: Arc::new(AtomicBool::new(false)),
-            }
-        }
-
-        fn fail(&self) {
-            self.is_failed.store(true, Ordering::Relaxed);
-        }
-
-        fn recover(&self) {
-            self.is_failed.store(false, Ordering::Relaxed);
-        }
-
-        fn is_healthy(&self) -> bool {
-            !self.is_failed.load(Ordering::Relaxed)
-        }
-
-        async fn submit_operation(&self, op_idx: u64) -> Result<(), String> {
-            if !self.is_healthy() {
-                return Err("Node failed".to_string());
-            }
-
-            let mut executor = self.executor.lock().await;
-            let mut seq = self.sequence.lock().await;
-            *seq += 1;
-
-            let op_id = OperationId::new(self.node_id.clone(), *seq);
-            let parent_vc = VectorClock::new();
-            let parent_version = StateVersion::new(parent_vc, "0".repeat(64));
-
-            let mut fields = HashMap::new();
-            fields.insert("value".to_string(), json!(op_idx));
-
-            let operation = Operation::new(
-                op_id,
-                parent_version.clone(),
-                format!("op_{}", op_idx),
-                OperationCommand {
-                    op_type: "set".to_string(),
-                    collection: "items".to_string(),
-                    record_id: format!("item_{}", op_idx),
-                    fields,
-                },
-                self.node_id.clone(),
-            );
-
-            executor.execute_local_transaction(
-                *seq,
-                format!("txn_{}", op_idx),
-                parent_version,
-                vec![operation],
-                ConsistencyContract::causal(),
-            )?;
-
-            Ok(())
-        }
-
-        async fn receive_operation(&self, message: ReplicationMessage) -> Result<(), String> {
-            if !self.is_healthy() {
-                return Err("Node failed".to_string());
-            }
-
-            let mut executor = self.executor.lock().await;
-            let parent_vc = VectorClock::new();
-            let parent_version = StateVersion::new(parent_vc, "0".repeat(64));
-
-            executor
-                .receive_replicated_transaction(message, parent_version)
-                .map(|_| ())
-                .map_err(|e| e)
-        }
-
-        async fn operations_applied(&self) -> u64 {
-            let executor = self.executor.lock().await;
-            executor
-                .get_replica_state(&self.node_id)
-                .map(|s| s.operations_applied)
-                .unwrap_or(0)
-        }
-
-        async fn state_hash(&self) -> StateHash {
-            let executor = self.executor.lock().await;
-            executor
-                .get_replica_state(&self.node_id)
-                .map(|s| s.state_hash)
-                .unwrap_or_else(|| StateHash::from_hex("0".repeat(64)))
-        }
-    }
-
-    /// Phase 2c Test 1: 2/3 nodes crash, 1/3 survives (quorum remains)
-    #[tokio::test]
-    async fn two_of_three_nodes_crash() {
-        let node_a = CascadingNode::new("node_a".to_string());
-        let node_b = CascadingNode::new("node_b".to_string());
-        let node_c = CascadingNode::new("node_c".to_string());
-
-        // Cluster prepares: all nodes healthy
-        for i in 0..20 {
-            node_a.submit_operation(i).await.expect("node_a pre-crash");
-        }
-
-        let pre_crash_ops_a = node_a.operations_applied().await;
-        println!("Pre-crash state: node_a has {} ops", pre_crash_ops_a);
-
-        // Cascade: 2 nodes crash simultaneously
-        node_a.fail();
-        node_b.fail();
-
-        // Node C survives and continues
-        for i in 20..30 {
-            let result = node_c.submit_operation(i).await;
-            assert!(result.is_ok(), "Surviving node should operate");
-        }
-
-        let surviving_ops = node_c.operations_applied().await;
-        assert_eq!(surviving_ops, 10, "Surviving node has new operations");
-
-        // Recover crashed nodes
-        node_a.recover();
-        node_b.recover();
-
-        // All nodes can operate again
-        node_a.submit_operation(30).await.expect("node_a recovery");
-        node_b.submit_operation(31).await.expect("node_b recovery");
-        node_c.submit_operation(32).await.expect("node_c continues");
-
-        let ops_a = node_a.operations_applied().await;
-        let ops_b = node_b.operations_applied().await;
-        let ops_c = node_c.operations_applied().await;
-
-        println!("Post-recovery: A={}, B={}, C={}", ops_a, ops_b, ops_c);
-
-        assert!(ops_a > 0, "Node A recovered");
-        assert!(ops_b > 0, "Node B recovered");
-        assert_eq!(ops_c, 11, "Node C continued through crashes");
-    }
-
-    /// Phase 2c Test 2: Leader + 1 follower crash simultaneously
-    #[tokio::test]
-    async fn leader_and_follower_crash_together() {
-        let leader = CascadingNode::new("leader".to_string());
-        let follower1 = CascadingNode::new("follower1".to_string());
-        let follower2 = CascadingNode::new("follower2".to_string());
-
-        // Register replicas
-        {
-            let mut executor = leader.executor.lock().await;
-            executor.register_replica("follower1".to_string(), StateHash::from_hex("0".repeat(64)));
-            executor.register_replica("follower2".to_string(), StateHash::from_hex("0".repeat(64)));
-        }
-
-        // Pre-crash operations
-        for i in 0..25 {
-            leader.submit_operation(i).await.expect("leader pre-crash");
-        }
-
-        let pre_crash_leader_ops = leader.operations_applied().await;
-        println!("Pre-crash leader ops: {}", pre_crash_leader_ops);
-
-        // Cascade: Leader and follower1 crash together
-        leader.fail();
-        follower1.fail();
-
-        // Follower2 is the last replica standing
-        follower2
-            .submit_operation(100)
-            .await
-            .expect("follower2 survives");
-
-        // Recovery
-        leader.recover();
-        follower1.recover();
-
-        // System resumes
-        for i in 25..50 {
-            leader.submit_operation(i).await.expect("leader recovery");
-        }
-
-        let final_leader_ops = leader.operations_applied().await;
-        let final_follower2_ops = follower2.operations_applied().await;
-
-        println!(
-            "Post-recovery: leader={}, follower2={}",
-            final_leader_ops, final_follower2_ops
-        );
-
-        assert_eq!(final_leader_ops, 50, "Leader recovered fully");
-        assert_eq!(final_follower2_ops, 1, "Last replica survived");
-    }
-
-    /// Phase 2c Test 3: Sequential cascade (failure spreading)
-    #[tokio::test]
-    async fn sequential_cascade() {
-        let node_1 = CascadingNode::new("node_1".to_string());
-        let node_2 = CascadingNode::new("node_2".to_string());
-        let node_3 = CascadingNode::new("node_3".to_string());
-
-        // All healthy, submit baseline
-        for i in 0..10 {
-            node_1.submit_operation(i).await.expect("node_1 baseline");
-            node_2
-                .submit_operation(i + 100)
-                .await
-                .expect("node_2 baseline");
-            node_3
-                .submit_operation(i + 200)
-                .await
-                .expect("node_3 baseline");
-        }
-
-        // Sequential cascade: failures propagate
-        node_1.fail();
-        tokio::time::sleep(tokio::time::Duration::from_millis(10)).await;
-
-        node_2.fail();
-        tokio::time::sleep(tokio::time::Duration::from_millis(10)).await;
-
-        // Only node_3 remains
-        let result = node_3.submit_operation(300).await;
-        assert!(result.is_ok(), "Last node operates");
-
-        // Cascade reversal: recovery
-        node_1.recover();
-        node_2.recover();
-
-        let ops_1 = node_1.operations_applied().await;
-        let ops_2 = node_2.operations_applied().await;
-        let ops_3 = node_3.operations_applied().await;
-
-        println!(
-            "Sequential cascade recovery: N1={}, N2={}, N3={}",
-            ops_1, ops_2, ops_3
-        );
-
-        assert!(ops_1 > 0, "Node 1 recovered");
-        assert!(ops_2 > 0, "Node 2 recovered");
-        assert!(ops_3 > 0, "Node 3 survived");
-    }
-
-    /// Phase 2c Test 4: Cascading recovery (all nodes restart)
-    #[tokio::test]
-    async fn cascading_recovery_all_restart() {
-        let nodes = vec![
-            CascadingNode::new("node_1".to_string()),
-            CascadingNode::new("node_2".to_string()),
-            CascadingNode::new("node_3".to_string()),
-        ];
-
-        // Initial phase: all healthy
-        for (idx, node) in nodes.iter().enumerate() {
-            node.submit_operation(idx as u64).await.expect("baseline");
-        }
-
-        // Failure phase: cascading crashes
-        for node in &nodes {
-            node.fail();
-        }
-
-        // All failed, verify no operations
-        for node in &nodes {
-            let result = node.submit_operation(999).await;
-            assert!(result.is_err(), "Failed nodes reject operations");
-        }
-
-        // Recovery phase: cascading recovery
-        for node in &nodes {
-            node.recover();
-        }
-
-        // All recovered, verify operations succeed
-        for (idx, node) in nodes.iter().enumerate() {
-            let result = node.submit_operation(100 + idx as u64).await;
-            assert!(result.is_ok(), "Recovered nodes accept operations");
-        }
-
-        // Verify all have state
-        for (idx, node) in nodes.iter().enumerate() {
-            let ops = node.operations_applied().await;
-            assert!(ops > 0, "Node {} has operations", idx);
-        }
-
-        println!("Cascading recovery complete - all nodes operational");
-    }
-
-    /// Phase 2c Test 5: Quorum validation (consistency check)
-    #[tokio::test]
-    async fn cascading_failure_quorum_consistency() {
-        let node_a = CascadingNode::new("node_a".to_string());
-        let node_b = CascadingNode::new("node_b".to_string());
-        let node_c = CascadingNode::new("node_c".to_string());
-
-        // Establish baseline where all agree
-        for i in 0..15 {
-            node_a.submit_operation(i).await.expect("a baseline");
-        }
-
-        let baseline_hash_a = node_a.state_hash().await;
-        println!("Baseline hash: {}", baseline_hash_a);
-
-        // Cascade some nodes
-        node_a.fail();
-        node_b.fail();
-
-        // Node C continues independently
-        for i in 15..20 {
-            node_c.submit_operation(i).await.expect("c continues");
-        }
-
-        // Recover both
-        node_a.recover();
-        node_b.recover();
-
-        // All resume
-        node_a.submit_operation(20).await.expect("a resumes");
-        node_b.submit_operation(21).await.expect("b resumes");
-
-        let ops_a = node_a.operations_applied().await;
-        let ops_b = node_b.operations_applied().await;
-        let ops_c = node_c.operations_applied().await;
-
-        println!("Quorum check: A={}, B={}, C={}", ops_a, ops_b, ops_c);
-
-        // Each node has its own state (no longer required to converge in this test model)
-        assert!(ops_a > 0, "Quorum node A consistent");
-        assert!(ops_b > 0, "Quorum node B consistent");
-        assert!(ops_c > 0, "Minority node C consistent");
-    }
-}