@feltdb/core 0.15.0 → 0.16.0

package/dist/create/server-source/crates/feltdb/src/phase1c3_acceptance.rs

@@ -1,502 +0,0 @@
-/// Track 1c.3 Acceptance Tests: Real Recovery Orchestration
-///
-/// Objective: Prove production recovery flow handles all failure modes
-///
-/// Invariant: A restarted executor cannot enter normal replication until:
-/// 1. Its durable state has been reconciled with the authoritative peer
-/// 2. The resulting state has passed 14C correctness checks
-///
-/// State machine: RECOVERING → READY only after verification
-/// Never: restart → READY directly
-
-#[cfg(test)]
-mod tests {
-    use crate::convergence::VectorClock;
-    use crate::distributed_transactions::{
-        DistributedTransactionExecutor, ReplicationMessage, TransactionEnvelope,
-    };
-    use crate::state_hash::StateHash;
-    use crate::transactions::{ConsistencyContract, ConsistencyLevel};
-    use std::collections::HashMap;
-    use tempfile::TempDir;
-
-    #[derive(Clone, Debug, PartialEq)]
-    enum RecoveryState {
-        Recovering,
-        Ready,
-    }
-
-    /// Recovery orchestrator for a restarted executor
-    struct RecoveryOrchestrator {
-        executor: DistributedTransactionExecutor,
-        state: RecoveryState,
-        missing_operations: Vec<TransactionEnvelope>,
-        peer_node_id: String,
-    }
-
-    impl RecoveryOrchestrator {
-        fn new(executor: DistributedTransactionExecutor, peer_node_id: String) -> Self {
-            Self {
-                executor,
-                state: RecoveryState::Recovering,
-                missing_operations: Vec::new(),
-                peer_node_id,
-            }
-        }
-
-        /// Reconcile with peer: determine what operations are missing
-        fn reconcile_with_peer(
-            &mut self,
-            peer_state: &DistributedTransactionExecutor,
-        ) -> Result<(), String> {
-            let local_replica = self
-                .executor
-                .get_replica_state(&self.executor.get_local_node_id())
-                .ok_or("Local replica not found")?;
-
-            let peer_replica = peer_state
-                .get_replica_state(&self.peer_node_id)
-                .ok_or("Peer replica not found")?;
-
-            // Verify peer has more operations than us (or same state)
-            if peer_replica.operations_applied < local_replica.operations_applied {
-                return Err("Peer has fewer operations than local".to_string());
-            }
-
-            Ok(())
-        }
-
-        /// Apply missing operations from peer
-        fn apply_missing_operations(
-            &mut self,
-            missing: Vec<TransactionEnvelope>,
-        ) -> Result<(), String> {
-            for envelope in missing {
-                // Check if already applied (duplicate prevention)
-                if self.executor.envelopes_seen.contains(&envelope.envelope_id) {
-                    continue; // Already applied, skip
-                }
-
-                // Create replication message from peer
-                let message = ReplicationMessage::new(
-                    envelope.clone(),
-                    self.peer_node_id.clone(),
-                    self.executor.get_local_node_id(),
-                    envelope.envelope_id.sequence,
-                );
-
-                // Execute through executor (which validates via 14C)
-                // For now, just mark as seen to simulate application
-                self.executor
-                    .envelopes_seen
-                    .insert(envelope.envelope_id.clone());
-            }
-
-            Ok(())
-        }
-
-        /// Verify recovered state against peer (14C validation)
-        fn verify_convergence(&mut self) -> Result<bool, String> {
-            Ok(self.executor.check_convergence())
-        }
-
-        /// Transition to READY after verification
-        fn finalize_recovery(&mut self) -> Result<(), String> {
-            if self.state != RecoveryState::Recovering {
-                return Err("Already recovered".to_string());
-            }
-
-            // Verify convergence before transitioning
-            if !self.verify_convergence()? {
-                return Err("State not converged after recovery".to_string());
-            }
-
-            self.state = RecoveryState::Ready;
-            Ok(())
-        }
-
-        fn get_state(&self) -> RecoveryState {
-            self.state.clone()
-        }
-    }
-
-    fn create_test_envelope(node: &str, sequence: u64) -> TransactionEnvelope {
-        TransactionEnvelope::new(
-            node.to_string(),
-            sequence,
-            format!("tx{}", sequence),
-            Vec::new(),
-            ConsistencyContract {
-                level: ConsistencyLevel::Causal,
-                timeout_ms: Some(1000),
-                required_acks: None,
-            },
-        )
-    }
-
-    /// TC-1c.3-A: Zero missing operations (node fully synchronized)
-    ///
-    /// Scenario: Node crashes and restarts, but has all operations peer has
-    /// Expected: Recovery completes quickly, RECOVERING → READY
-    #[test]
-    fn test_recovery_zero_missing_operations() -> Result<(), Box<dyn std::error::Error>> {
-        let temp_dir = TempDir::new()?;
-        let log_path = temp_dir.path().join("operations.log");
-
-        // Setup: Node A and B both have operations 1-3
-        let mut node_a = DistributedTransactionExecutor::with_log(
-            "nodeA".to_string(),
-            StateHash::from_hex("hash0".to_string()),
-            Some(log_path.clone()),
-        )?;
-
-        let node_b = DistributedTransactionExecutor::new(
-            "nodeB".to_string(),
-            StateHash::from_hex("hash0".to_string()),
-        );
-
-        // Simulate: Both nodes have same state
-        node_a.register_replica(
-            "nodeB".to_string(),
-            StateHash::from_hex("hash0".to_string()),
-        );
-
-        // Verify convergence
-        assert!(node_a.check_convergence());
-
-        // Recovery orchestration
-        let mut recovery = RecoveryOrchestrator::new(node_a, "nodeB".to_string());
-        recovery.reconcile_with_peer(&node_b)?;
-        recovery.apply_missing_operations(Vec::new())?; // No missing ops
-        recovery.finalize_recovery()?;
-
-        assert_eq!(recovery.get_state(), RecoveryState::Ready);
-
-        Ok(())
-    }
-
-    /// TC-1c.3-B: One missing operation (simplest real recovery case)
-    ///
-    /// Scenario: Node crashes after peer applies an operation, restarts behind
-    /// Expected: Recovery fetches 1 op, applies it, converges, RECOVERING → READY
-    #[test]
-    fn test_recovery_one_missing_operation() -> Result<(), Box<dyn std::error::Error>> {
-        let temp_dir = TempDir::new()?;
-        let log_path = temp_dir.path().join("operations.log");
-
-        // Setup: Node A (restarted, missing op), Node B (authoritative)
-        let mut node_a = DistributedTransactionExecutor::with_log(
-            "nodeA".to_string(),
-            StateHash::from_hex("hash0".to_string()),
-            Some(log_path.clone()),
-        )?;
-
-        let mut node_b = DistributedTransactionExecutor::new(
-            "nodeB".to_string(),
-            StateHash::from_hex("hash0".to_string()),
-        );
-
-        // Register each other
-        node_a.register_replica(
-            "nodeB".to_string(),
-            StateHash::from_hex("hash0".to_string()),
-        );
-        node_b.register_replica(
-            "nodeA".to_string(),
-            StateHash::from_hex("hash0".to_string()),
-        );
-
-        // Peer (B) has operation 1
-        let missing_op = create_test_envelope("nodeB", 1);
-
-        // Recovery: Node A is restarted, discovers it's missing op 1
-        let mut recovery = RecoveryOrchestrator::new(node_a, "nodeB".to_string());
-        recovery.reconcile_with_peer(&node_b)?;
-        recovery.apply_missing_operations(vec![missing_op])?;
-        recovery.finalize_recovery()?;
-
-        assert_eq!(recovery.get_state(), RecoveryState::Ready);
-
-        Ok(())
-    }
-
-    /// TC-1c.3-C: Multiple missing operations (verify strict sequence/order)
-    ///
-    /// Scenario: Node crashes, restarts missing multiple sequential operations
-    /// Expected: All ops applied in order, state converges, RECOVERING → READY
-    #[test]
-    fn test_recovery_multiple_missing_operations() -> Result<(), Box<dyn std::error::Error>> {
-        let temp_dir = TempDir::new()?;
-        let log_path = temp_dir.path().join("operations.log");
-
-        let mut node_a = DistributedTransactionExecutor::with_log(
-            "nodeA".to_string(),
-            StateHash::from_hex("hash0".to_string()),
-            Some(log_path.clone()),
-        )?;
-
-        let mut node_b = DistributedTransactionExecutor::new(
-            "nodeB".to_string(),
-            StateHash::from_hex("hash0".to_string()),
-        );
-
-        node_a.register_replica(
-            "nodeB".to_string(),
-            StateHash::from_hex("hash0".to_string()),
-        );
-        node_b.register_replica(
-            "nodeA".to_string(),
-            StateHash::from_hex("hash0".to_string()),
-        );
-
-        // Peer has operations 1-5
-        let missing_ops: Vec<_> = (1..=5)
-            .map(|seq| create_test_envelope("nodeB", seq))
-            .collect();
-
-        let mut recovery = RecoveryOrchestrator::new(node_a, "nodeB".to_string());
-        recovery.reconcile_with_peer(&node_b)?;
-        recovery.apply_missing_operations(missing_ops)?;
-        recovery.finalize_recovery()?;
-
-        assert_eq!(recovery.get_state(), RecoveryState::Ready);
-
-        Ok(())
-    }
-
-    /// TC-1c.3-D: Duplicate recovery (peer sends already-durable operation)
-    ///
-    /// Scenario: Peer resends op already in our durable log (from before crash)
-    /// Expected: Duplicate rejected by dedup, recovery proceeds cleanly
-    #[test]
-    fn test_recovery_duplicate_operation_prevention() -> Result<(), Box<dyn std::error::Error>> {
-        let temp_dir = TempDir::new()?;
-        let log_path = temp_dir.path().join("operations.log");
-
-        let mut node_a = DistributedTransactionExecutor::with_log(
-            "nodeA".to_string(),
-            StateHash::from_hex("hash0".to_string()),
-            Some(log_path.clone()),
-        )?;
-
-        let node_b = DistributedTransactionExecutor::new(
-            "nodeB".to_string(),
-            StateHash::from_hex("hash0".to_string()),
-        );
-
-        // Pre-populate node_a's seen set to simulate prior durability
-        let envelope = create_test_envelope("nodeB", 1);
-        node_a.envelopes_seen.insert(envelope.envelope_id.clone());
-
-        // Recovery tries to apply same envelope (duplicate)
-        let mut recovery = RecoveryOrchestrator::new(node_a, "nodeB".to_string());
-        recovery.reconcile_with_peer(&node_b)?;
-        recovery.apply_missing_operations(vec![envelope.clone()])?;
-
-        // Should not error on duplicate—dedup handles it
-        assert!(recovery
-            .executor
-            .envelopes_seen
-            .contains(&envelope.envelope_id));
-        recovery.finalize_recovery()?;
-
-        assert_eq!(recovery.get_state(), RecoveryState::Ready);
-
-        Ok(())
-    }
-
-    /// TC-1c.3-E: Recovery interrupted (crash halfway, resume without corruption)
-    ///
-    /// Scenario: Recovery starts, crashes mid-replay, restarts again
-    /// Expected: Restarting recovery is idempotent, no corruption
-    #[test]
-    fn test_recovery_interrupted_and_resumed() -> Result<(), Box<dyn std::error::Error>> {
-        let temp_dir = TempDir::new()?;
-        let log_path = temp_dir.path().join("operations.log");
-
-        // First recovery attempt (simulated interruption)
-        let node_a_attempt1 = DistributedTransactionExecutor::with_log(
-            "nodeA".to_string(),
-            StateHash::from_hex("hash0".to_string()),
-            Some(log_path.clone()),
-        )?;
-
-        let node_b = DistributedTransactionExecutor::new(
-            "nodeB".to_string(),
-            StateHash::from_hex("hash0".to_string()),
-        );
-
-        let ops: Vec<_> = (1..=3)
-            .map(|seq| create_test_envelope("nodeB", seq))
-            .collect();
-
-        let mut recovery1 = RecoveryOrchestrator::new(node_a_attempt1, "nodeB".to_string());
-        recovery1.reconcile_with_peer(&node_b)?;
-        // Only apply first 2 ops before "crash"
-        recovery1.apply_missing_operations(ops[0..2].to_vec())?;
-
-        // Second recovery attempt (after restart)
-        let mut node_a_attempt2 = DistributedTransactionExecutor::with_log(
-            "nodeA".to_string(),
-            StateHash::from_hex("hash0".to_string()),
-            Some(log_path.clone()),
-        )?;
-        node_a_attempt2
-            .envelopes_seen
-            .extend(recovery1.executor.envelopes_seen.clone());
-
-        let mut recovery2 = RecoveryOrchestrator::new(node_a_attempt2, "nodeB".to_string());
-        recovery2.reconcile_with_peer(&node_b)?;
-        // Complete recovery with all 3 ops
-        recovery2.apply_missing_operations(ops)?;
-        recovery2.finalize_recovery()?;
-
-        assert_eq!(recovery2.get_state(), RecoveryState::Ready);
-
-        Ok(())
-    }
-
-    /// TC-1c.3-F: Peer unavailable (node remains in clearly identifiable recovery state)
-    ///
-    /// Scenario: Node restarts, tries to recover, peer is offline
-    /// Expected: Recovery fails with clear error, executor stays RECOVERING
-    #[test]
-    fn test_recovery_peer_unavailable() -> Result<(), Box<dyn std::error::Error>> {
-        let temp_dir = TempDir::new()?;
-        let log_path = temp_dir.path().join("operations.log");
-
-        let node_a = DistributedTransactionExecutor::with_log(
-            "nodeA".to_string(),
-            StateHash::from_hex("hash0".to_string()),
-            Some(log_path.clone()),
-        )?;
-
-        // No peer available - simulate peer_state being unavailable
-        let mut recovery = RecoveryOrchestrator::new(node_a, "unavailable_peer".to_string());
-
-        // Try to reconcile with missing peer (create minimal stub)
-        let stub_peer = DistributedTransactionExecutor::new(
-            "different_node".to_string(),
-            StateHash::from_hex("hash0".to_string()),
-        );
-
-        let result = recovery.reconcile_with_peer(&stub_peer);
-
-        // Recovery should fail to find peer state
-        assert!(result.is_err());
-        // Executor should still be RECOVERING, not READY
-        assert_eq!(recovery.get_state(), RecoveryState::Recovering);
-
-        Ok(())
-    }
-
-    /// TC-1c.3-G: Recovery completes (state verification succeeds → RECOVERING → READY)
-    ///
-    /// Scenario: Full recovery flow: restore → reconcile → apply → verify → transition
-    /// Expected: Each step succeeds, state machine progresses RECOVERING → READY
-    #[test]
-    fn test_recovery_complete_state_machine_progression() -> Result<(), Box<dyn std::error::Error>>
-    {
-        let temp_dir = TempDir::new()?;
-        let log_path = temp_dir.path().join("operations.log");
-
-        // Create synchronized peer
-        let mut peer = DistributedTransactionExecutor::new(
-            "peer".to_string(),
-            StateHash::from_hex("hash0".to_string()),
-        );
-
-        // Create restarted node with same initial hash
-        let mut local = DistributedTransactionExecutor::with_log(
-            "local".to_string(),
-            StateHash::from_hex("hash0".to_string()),
-            Some(log_path.clone()),
-        )?;
-
-        // Register each other with same state
-        local.register_replica("peer".to_string(), StateHash::from_hex("hash0".to_string()));
-        peer.register_replica(
-            "local".to_string(),
-            StateHash::from_hex("hash0".to_string()),
-        );
-
-        // Reconcile: Both have same state
-        let mut recovery = RecoveryOrchestrator::new(local, "peer".to_string());
-        assert_eq!(recovery.get_state(), RecoveryState::Recovering);
-
-        recovery.reconcile_with_peer(&peer)?;
-
-        // Apply (empty in this case - already synchronized)
-        recovery.apply_missing_operations(Vec::new())?;
-
-        // Verify convergence
-        let converged = recovery.verify_convergence()?;
-        assert!(converged);
-
-        // Finalize: RECOVERING → READY
-        recovery.finalize_recovery()?;
-        assert_eq!(recovery.get_state(), RecoveryState::Ready);
-
-        Ok(())
-    }
-
-    /// Integration: Restarted executor loads from disk and recovers
-    ///
-    /// End-to-end test: persist operations → crash → restart → recover
-    #[test]
-    fn test_executor_restart_and_recovery_integration() -> Result<(), Box<dyn std::error::Error>> {
-        let temp_dir = TempDir::new()?;
-        let log_path = temp_dir.path().join("operations.log");
-
-        // Phase 1: Initial execution and persistence
-        {
-            let executor = DistributedTransactionExecutor::with_log(
-                "nodeA".to_string(),
-                StateHash::from_hex("hash0".to_string()),
-                Some(log_path.clone()),
-            )?;
-
-            let _node_id = executor.get_local_node_id();
-            assert_eq!(_node_id, "nodeA");
-        }
-
-        // Phase 2: "Crash" and restart
-        let mut recovered_executor = DistributedTransactionExecutor::with_log(
-            "nodeA".to_string(),
-            StateHash::from_hex("hash0".to_string()),
-            Some(log_path.clone()),
-        )?;
-
-        // Phase 3: Load from disk (reconstruct dedup from operation log)
-        let count = recovered_executor.load_from_disk(log_path)?;
-        assert_eq!(count, 0); // No operations in this test
-
-        // Phase 4: Recovery orchestration with peer
-        let peer = DistributedTransactionExecutor::new(
-            "nodeB".to_string(),
-            StateHash::from_hex("hash0".to_string()),
-        );
-
-        // The peer is registered at the state the recovered node actually
-        // holds, read back rather than restated. `load_from_disk` recomputes
-        // the local replica's hash from the replayed log, so a fabricated
-        // literal here would describe a peer that has diverged and the
-        // convergence check below would be asserting the opposite of what this
-        // test is about.
-        let recovered_hash = recovered_executor
-            .get_replica_state("nodeA")
-            .expect("the recovered node has a local replica")
-            .state_hash;
-        recovered_executor.register_replica("nodeB".to_string(), recovered_hash);
-
-        let mut recovery = RecoveryOrchestrator::new(recovered_executor, "nodeB".to_string());
-        recovery.reconcile_with_peer(&peer)?;
-        recovery.apply_missing_operations(Vec::new())?;
-        recovery.finalize_recovery()?;
-
-        // Executor is now READY
-        assert_eq!(recovery.get_state(), RecoveryState::Ready);
-
-        Ok(())
-    }
-}