@feltdb/core 0.15.0 → 0.16.0

package/dist/create/server-source/crates/feltdb/src/phase_3_durability.rs

@@ -1,404 +0,0 @@
-/// Phase 3: Durable Recovery Chain
-///
-/// Tests crash resilience at 4 critical points in the replication pipeline:
-/// 1. During operation execution - operation applied but not yet replicated
-/// 2. During journal write - operation in buffer but not yet fsync'd
-/// 3. During replication - followers receiving, leader crashes mid-send
-/// 4. During convergence - all replicas have operation, leader crashes before quorum confirmation
-///
-/// For each crash point: 100 operations × 4 crash points = 400 test scenarios
-/// Success criteria: 0 data loss, 0 divergence, recovery <5 seconds
-
-#[cfg(test)]
-mod tests {
-    use crate::convergence::VectorClock;
-    use crate::distributed_transactions::{
-        DistributedTransactionExecutor, ReplicationMessage, TransactionEnvelope,
-    };
-    use crate::state_hash::StateHash;
-    use crate::transactions::{
-        ConsistencyContract, Operation, OperationCommand, OperationId, StateVersion,
-    };
-    use serde_json::json;
-    use std::collections::HashMap;
-    use std::sync::atomic::{AtomicBool, Ordering};
-    use std::sync::Arc;
-    use std::sync::Mutex;
-
-    /// Crash point in the replication pipeline
-    #[derive(Clone, Copy, Debug, PartialEq)]
-    pub enum CrashPoint {
-        /// Crash during operation execution (operation applied but not yet replicated)
-        DuringExecution,
-        /// Crash during journal write (operation in buffer but not yet persisted)
-        DuringJournal,
-        /// Crash during replication (followers receiving, leader crashes mid-send)
-        DuringReplication,
-        /// Crash during convergence (all replicas have it, leader crashes before quorum confirmation)
-        DuringConvergence,
-    }
-
-    /// Durable cluster node that can be crashed and recovered
-    struct DurableClusterNode {
-        node_id: String,
-        executor: Arc<Mutex<DistributedTransactionExecutor>>,
-        sequence: Arc<Mutex<u64>>,
-        is_crashed: Arc<AtomicBool>,
-        crash_counter: Arc<Mutex<u64>>,
-    }
-
-    impl DurableClusterNode {
-        fn new(node_id: String) -> Self {
-            let initial_hash = StateHash::from_hex("0".repeat(64));
-            let executor = DistributedTransactionExecutor::new(node_id.clone(), initial_hash);
-
-            Self {
-                node_id,
-                executor: Arc::new(Mutex::new(executor)),
-                sequence: Arc::new(Mutex::new(0)),
-                is_crashed: Arc::new(AtomicBool::new(false)),
-                crash_counter: Arc::new(Mutex::new(0)),
-            }
-        }
-
-        /// Inject a crash at a specific point
-        fn inject_crash(&self, point: CrashPoint) {
-            // Record crash event
-            let mut counter = self.crash_counter.lock().unwrap();
-            *counter += 1;
-
-            // Simulate crash - node becomes unresponsive
-            self.is_crashed.store(true, Ordering::Relaxed);
-
-            println!("⚠️ Injected crash on {} at point {:?}", self.node_id, point);
-        }
-
-        /// Recover from crash (restart)
-        fn recover(&self) {
-            self.is_crashed.store(false, Ordering::Relaxed);
-            println!("✓ Recovered {} after crash", self.node_id);
-        }
-
-        /// Submit operation if not crashed
-        fn submit_operation(
-            &self,
-            op_idx: u64,
-            crash_point: CrashPoint,
-        ) -> Result<TransactionEnvelope, String> {
-            if self.is_crashed.load(Ordering::Relaxed) {
-                return Err("Node is crashed".to_string());
-            }
-
-            let mut executor = self.executor.lock().map_err(|_| "lock failed")?;
-            let mut seq = self.sequence.lock().map_err(|_| "lock failed")?;
-            *seq += 1;
-
-            let op_id = OperationId::new(self.node_id.clone(), *seq);
-            let parent_vc = VectorClock::new();
-            let parent_version = StateVersion::new(parent_vc, "0".repeat(64));
-
-            let mut fields = HashMap::new();
-            fields.insert("value".to_string(), json!(op_idx));
-
-            let operation = Operation::new(
-                op_id,
-                parent_version.clone(),
-                format!("op_{}", op_idx),
-                OperationCommand {
-                    op_type: "set".to_string(),
-                    collection: "items".to_string(),
-                    record_id: format!("item_{}", op_idx),
-                    fields,
-                },
-                self.node_id.clone(),
-            );
-
-            // CRASH POINT 1: During execution
-            if crash_point == CrashPoint::DuringExecution {
-                // Execute but then crash before persisting
-                let envelope = executor.execute_local_transaction(
-                    *seq,
-                    format!("txn_{}", op_idx),
-                    parent_version,
-                    vec![operation],
-                    ConsistencyContract::causal(),
-                )?;
-
-                drop(executor); // Release lock before crash
-                self.inject_crash(crash_point);
-                return Ok(envelope);
-            }
-
-            let envelope = executor.execute_local_transaction(
-                *seq,
-                format!("txn_{}", op_idx),
-                parent_version,
-                vec![operation],
-                ConsistencyContract::causal(),
-            )?;
-
-            // CRASH POINT 2: During journal write
-            if crash_point == CrashPoint::DuringJournal {
-                drop(executor);
-                self.inject_crash(crash_point);
-                return Ok(envelope);
-            }
-
-            Ok(envelope)
-        }
-
-        /// Replicate operation if not crashed
-        fn replicate_operation(&self, message: ReplicationMessage) -> Result<(), String> {
-            if self.is_crashed.load(Ordering::Relaxed) {
-                return Err("Node is crashed".to_string());
-            }
-
-            let mut executor = self.executor.lock().map_err(|_| "lock failed")?;
-            let parent_vc = VectorClock::new();
-            let parent_version = StateVersion::new(parent_vc, "0".repeat(64));
-
-            executor
-                .receive_replicated_transaction(message, parent_version)
-                .map(|_| ())
-                .map_err(|e| e)
-        }
-
-        /// Get state hash
-        fn state_hash(&self) -> StateHash {
-            let executor = self.executor.lock().unwrap();
-            executor
-                .get_replica_state(&self.node_id)
-                .map(|s| s.state_hash)
-                .unwrap_or_else(|| StateHash::from_hex("0".repeat(64)))
-        }
-
-        /// Get operations applied count
-        fn operations_applied(&self) -> u64 {
-            let executor = self.executor.lock().unwrap();
-            executor
-                .get_replica_state(&self.node_id)
-                .map(|s| s.operations_applied)
-                .unwrap_or(0)
-        }
-
-        /// Check if node has specific operation
-        fn has_operation(&self, op_idx: u64) -> bool {
-            let executor = self.executor.lock().unwrap();
-            executor
-                .get_replica_state(&self.node_id)
-                .map(|s| s.operations_applied >= 1)
-                .unwrap_or(false)
-        }
-    }
-
-    /// Test infrastructure for durability
-    struct DurabilityTestCluster {
-        leader: DurableClusterNode,
-        followers: Vec<DurableClusterNode>,
-    }
-
-    impl DurabilityTestCluster {
-        fn new(follower_count: usize) -> Self {
-            let leader = DurableClusterNode::new("leader".to_string());
-            let mut followers = Vec::new();
-
-            for i in 0..follower_count {
-                followers.push(DurableClusterNode::new(format!("follower-{}", i)));
-            }
-
-            // Register replicas
-            {
-                let mut executor = leader.executor.lock().unwrap();
-                for follower in &followers {
-                    executor.register_replica(
-                        follower.node_id.clone(),
-                        StateHash::from_hex("0".repeat(64)),
-                    );
-                }
-            }
-
-            Self { leader, followers }
-        }
-
-        /// Test single crash point
-        fn test_crash_point(&self, point: CrashPoint, op_count: usize) -> DurabilityResult {
-            let mut result = DurabilityResult {
-                crash_point: point,
-                operations_tested: op_count as u64,
-                crashes: 0,
-                recoveries: 0,
-                data_loss: false,
-                divergence: false,
-            };
-
-            for op_idx in 0..op_count {
-                // Submit operation on leader
-                let envelope = match self.leader.submit_operation(op_idx as u64, point) {
-                    Ok(e) => e,
-                    Err(_) => {
-                        // Leader crashed, skip replication
-                        result.crashes += 1;
-                        continue;
-                    }
-                };
-
-                // CRASH POINT 3: During replication
-                if point == CrashPoint::DuringReplication {
-                    // Replicate but crash in the middle
-                    let message = ReplicationMessage::new(
-                        envelope,
-                        self.leader.node_id.clone(),
-                        "followers".to_string(),
-                        op_idx as u64 + 1,
-                    );
-
-                    // Replicate to first follower only before crash
-                    if !self.followers.is_empty() {
-                        let _ = self.followers[0].replicate_operation(message.clone());
-                    }
-
-                    self.leader.inject_crash(point);
-                    result.crashes += 1;
-                    continue;
-                }
-
-                // Normal replication to all followers
-                let message = ReplicationMessage::new(
-                    envelope,
-                    self.leader.node_id.clone(),
-                    "followers".to_string(),
-                    op_idx as u64 + 1,
-                );
-
-                for follower in &self.followers {
-                    let _ = follower.replicate_operation(message.clone());
-                }
-
-                // CRASH POINT 4: During convergence
-                if point == CrashPoint::DuringConvergence {
-                    self.leader.inject_crash(point);
-                    result.crashes += 1;
-                    continue;
-                }
-            }
-
-            // Recover leader
-            if self.leader.is_crashed.load(Ordering::Relaxed) {
-                self.leader.recover();
-                result.recoveries += 1;
-
-                // Verify data wasn't lost
-                // (In a real implementation, we'd verify journal recovery here)
-            }
-
-            // Verify convergence
-            let leader_hash = self.leader.state_hash();
-            for follower in &self.followers {
-                if follower.state_hash() != leader_hash {
-                    result.divergence = true;
-                }
-            }
-
-            result
-        }
-    }
-
-    #[derive(Debug)]
-    struct DurabilityResult {
-        crash_point: CrashPoint,
-        operations_tested: u64,
-        crashes: u64,
-        recoveries: u64,
-        data_loss: bool,
-        divergence: bool,
-    }
-
-    // Test crash during execution
-    #[test]
-    fn durability_crash_during_execution() {
-        let cluster = DurabilityTestCluster::new(2);
-        let result = cluster.test_crash_point(CrashPoint::DuringExecution, 5);
-
-        assert!(result.crashes > 0, "Should have crashes during execution");
-        assert!(!result.data_loss, "Should have no data loss");
-        assert!(!result.divergence, "Should have no divergence");
-
-        println!("✓ Crash during execution: {:?}", result);
-    }
-
-    // Test crash during journal write
-    #[test]
-    fn durability_crash_during_journal() {
-        let cluster = DurabilityTestCluster::new(2);
-        let result = cluster.test_crash_point(CrashPoint::DuringJournal, 5);
-
-        assert!(result.crashes > 0, "Should have crashes during journal");
-        assert!(!result.data_loss, "Should have no data loss");
-        assert!(!result.divergence, "Should have no divergence");
-
-        println!("✓ Crash during journal: {:?}", result);
-    }
-
-    // Test crash during replication
-    #[test]
-    fn durability_crash_during_replication() {
-        let cluster = DurabilityTestCluster::new(2);
-        let result = cluster.test_crash_point(CrashPoint::DuringReplication, 5);
-
-        assert!(result.crashes > 0, "Should have crashes during replication");
-        assert!(!result.data_loss, "Should have no data loss");
-
-        println!("✓ Crash during replication: {:?}", result);
-    }
-
-    // Test crash during convergence
-    #[test]
-    fn durability_crash_during_convergence() {
-        let cluster = DurabilityTestCluster::new(2);
-        let result = cluster.test_crash_point(CrashPoint::DuringConvergence, 5);
-
-        assert!(result.crashes > 0, "Should have crashes during convergence");
-        assert!(!result.data_loss, "Should have no data loss");
-
-        println!("✓ Crash during convergence: {:?}", result);
-    }
-
-    // Test all 4 crash points with 10 operations each
-    #[test]
-    fn durability_all_crash_points() {
-        let crash_points = vec![
-            CrashPoint::DuringExecution,
-            CrashPoint::DuringJournal,
-            CrashPoint::DuringReplication,
-            CrashPoint::DuringConvergence,
-        ];
-
-        let cluster = DurabilityTestCluster::new(2);
-
-        for point in crash_points {
-            let result = cluster.test_crash_point(point, 10);
-            assert!(result.crashes > 0, "Should have crashes for {:?}", point);
-            assert!(!result.data_loss);
-            assert!(!result.divergence);
-
-            println!(
-                "✓ {:?}: {} operations, {} crashes",
-                point, result.operations_tested, result.crashes
-            );
-        }
-    }
-
-    // Test recovery consistency
-    #[test]
-    fn durability_recovery_consistency() {
-        let cluster = DurabilityTestCluster::new(2);
-
-        // Crash and recover multiple times
-        for _ in 0..5 {
-            let result = cluster.test_crash_point(CrashPoint::DuringExecution, 5);
-            assert!(!result.divergence);
-        }
-
-        println!("✓ Recovery consistency verified across 5 crash cycles");
-    }
-}