@feltdb/core 0.4.11 → 0.4.13

package/dist/create/server-source/crates/feltdb/src/multi_node_convergence.rs

@@ -0,0 +1,311 @@
+/// Multi-Node Convergence Testing
+///
+/// Tests whether independent nodes executing the same operations
+/// (in potentially different orders) converge to identical state_hash.
+///
+/// This is the core Gate 1 test: does operation-specific merge
+/// semantics guarantee convergence despite arbitrary delivery order?
+
+use crate::operation_log::MemoryOperationLog;
+use crate::permutation_scheduler::{OperationSchedule, PermutationScheduler};
+use crate::replay_fuzzing::ReplayFuzzer;
+use crate::state_hash::StateHash;
+use crate::state_transition_store::MemoryStateTransitionStore;
+use crate::transactions::Operation;
+use std::collections::HashMap;
+
+/// Result of executing operations on a single simulated node
+#[derive(Clone, Debug)]
+pub struct NodeExecutionResult {
+    pub node_id: String,
+    pub schedule_strategy: String,
+    pub final_state_hash: StateHash,
+    pub operations_applied: u64,
+}
+
+/// Comparison of convergence across multiple nodes
+#[derive(Clone, Debug)]
+pub struct ConvergenceResult {
+    pub all_converged: bool,
+    pub canonical_hash: StateHash,
+    pub node_results: Vec<NodeExecutionResult>,
+    pub divergent_nodes: Vec<(String, StateHash)>,
+}
+
+impl ConvergenceResult {
+    pub fn summary(&self) -> String {
+        if self.all_converged {
+            format!(
+                "✓ Convergence successful: {} nodes → {}",
+                self.node_results.len(),
+                self.canonical_hash
+            )
+        } else {
+            let divergent_str = self
+                .divergent_nodes
+                .iter()
+                .map(|(node, hash)| format!("{}:{}", node, hash))
+                .collect::<Vec<_>>()
+                .join(", ");
+            format!(
+                "✗ Convergence failed: {} canonical, divergent: [{}]",
+                self.canonical_hash, divergent_str
+            )
+        }
+    }
+}
+
+/// Multi-node convergence simulator
+pub struct MultiNodeConvergenceSimulator {
+    node_count: usize,
+}
+
+impl MultiNodeConvergenceSimulator {
+    pub fn new(node_count: usize) -> Self {
+        Self { node_count }
+    }
+
+    /// Execute operations on a single simulated node
+    fn execute_on_node(schedule: &OperationSchedule, node_id: &str) -> NodeExecutionResult {
+        let log = MemoryOperationLog::new();
+        let store = MemoryStateTransitionStore::new();
+
+        // Append operations to log in the schedule order
+        for op in &schedule.operations {
+            let _ = log.append(op.clone());
+        }
+
+        // Execute via replay fuzzer (deterministic replay)
+        let fuzzer = ReplayFuzzer::new(log, store);
+        let result = fuzzer.replay_normal();
+
+        NodeExecutionResult {
+            node_id: node_id.to_string(),
+            schedule_strategy: schedule.id(),
+            final_state_hash: result.final_state_hash,
+            operations_applied: result.operations_replayed,
+        }
+    }
+
+    /// Execute same operations across multiple nodes with different schedules
+    pub fn converge_with_schedules(&self, schedules: Vec<OperationSchedule>) -> ConvergenceResult {
+        let mut results = Vec::new();
+
+        for (idx, schedule) in schedules.iter().enumerate() {
+            let node_id = format!("node{}", idx);
+            let result = Self::execute_on_node(schedule, &node_id);
+            results.push(result);
+        }
+
+        // Extract unique hashes
+        let hashes: Vec<_> = results.iter().map(|r| r.final_state_hash.clone()).collect();
+        let unique_hashes: std::collections::HashSet<_> = hashes.iter().cloned().collect();
+
+        let (all_converged, canonical_hash, divergent_nodes) = if unique_hashes.len() == 1 {
+            let canonical = results[0].final_state_hash.clone();
+            (true, canonical, Vec::new())
+        } else {
+            // Multiple different hashes = divergence
+            let canonical = results[0].final_state_hash.clone();
+            let divergent: Vec<(String, StateHash)> = results
+                .iter()
+                .skip(1)
+                .filter(|r| r.final_state_hash != canonical)
+                .map(|r| (r.node_id.clone(), r.final_state_hash.clone()))
+                .collect();
+            (false, canonical, divergent)
+        };
+
+        ConvergenceResult {
+            all_converged,
+            canonical_hash,
+            node_results: results,
+            divergent_nodes,
+        }
+    }
+
+    /// Test convergence with all deterministic schedules
+    pub fn test_deterministic_convergence(&self, operations: &[Operation]) -> ConvergenceResult {
+        let schedules = PermutationScheduler::deterministic_schedules(operations);
+        self.converge_with_schedules(schedules)
+    }
+
+    /// Test convergence with minimal permutations (fast)
+    pub fn test_minimal_convergence(&self, operations: &[Operation]) -> ConvergenceResult {
+        let schedules = PermutationScheduler::minimal_schedules(operations);
+        self.converge_with_schedules(schedules)
+    }
+
+    /// Test convergence with comprehensive permutations
+    pub fn test_comprehensive_convergence(&self, operations: &[Operation]) -> ConvergenceResult {
+        let schedules = PermutationScheduler::comprehensive_schedules(operations, 50);
+        self.converge_with_schedules(schedules)
+    }
+
+    /// Test convergence with thorough permutations
+    pub fn test_thorough_convergence(&self, operations: &[Operation]) -> ConvergenceResult {
+        let schedules = PermutationScheduler::thorough_schedules(operations);
+        self.converge_with_schedules(schedules)
+    }
+
+    /// Aggregate convergence results across multiple test runs
+    pub fn aggregate_convergence_runs(
+        &self,
+        operations: &[Operation],
+        run_count: usize,
+    ) -> ConvergenceAggregation {
+        let mut run_results = Vec::new();
+        let mut all_passed = true;
+
+        for _ in 0..run_count {
+            let result = self.test_minimal_convergence(operations);
+            if !result.all_converged {
+                all_passed = false;
+            }
+            run_results.push(result);
+        }
+
+        ConvergenceAggregation {
+            all_runs_passed: all_passed,
+            run_results,
+            total_operations_tested: operations.len(),
+        }
+    }
+}
+
+/// Aggregated results from multiple convergence test runs
+#[derive(Clone, Debug)]
+pub struct ConvergenceAggregation {
+    pub all_runs_passed: bool,
+    pub run_results: Vec<ConvergenceResult>,
+    pub total_operations_tested: usize,
+}
+
+impl ConvergenceAggregation {
+    pub fn summary(&self) -> String {
+        let passed = self.run_results.iter().filter(|r| r.all_converged).count();
+        let total = self.run_results.len();
+
+        format!(
+            "Convergence Test Summary: {}/{} runs passed, {} operations tested",
+            passed, total, self.total_operations_tested
+        )
+    }
+
+    pub fn all_passed(&self) -> bool {
+        self.all_runs_passed
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::convergence::VectorClock;
+    use crate::transactions::{OperationId, OperationCommand, StateVersion};
+    use std::collections::HashMap;
+
+    fn create_test_operation(seq: u64, collection: &str, record_id: &str, value: i32) -> Operation {
+        let mut vc = VectorClock::new();
+        vc.increment("test_node");
+        let version = StateVersion::new(vc, "test_hash".to_string());
+
+        Operation::new(
+            OperationId::new("test_node".to_string(), seq),
+            version,
+            format!("Op{}", seq),
+            OperationCommand {
+                op_type: "UpdateField".to_string(),
+                collection: collection.to_string(),
+                record_id: record_id.to_string(),
+                fields: {
+                    let mut m = HashMap::new();
+                    m.insert("value".to_string(), serde_json::json!(value));
+                    m
+                },
+            },
+            "test".to_string(),
+        )
+    }
+
+    #[test]
+    fn test_simulator_creation() {
+        let simulator = MultiNodeConvergenceSimulator::new(3);
+        assert_eq!(simulator.node_count, 3);
+    }
+
+    #[test]
+    fn test_single_operation_convergence() {
+        let ops = vec![create_test_operation(1, "records", "rec1", 10)];
+
+        let simulator = MultiNodeConvergenceSimulator::new(3);
+        let result = simulator.test_deterministic_convergence(&ops);
+
+        assert!(result.all_converged);
+        assert_eq!(result.node_results.len(), 3); // original, reversed, sorted
+    }
+
+    #[test]
+    fn test_multiple_operations_convergence() {
+        let ops = vec![
+            create_test_operation(1, "records", "rec1", 10),
+            create_test_operation(2, "records", "rec2", 20),
+            create_test_operation(3, "records", "rec3", 30),
+        ];
+
+        let simulator = MultiNodeConvergenceSimulator::new(3);
+        let result = simulator.test_minimal_convergence(&ops);
+
+        // All permutations should produce same hash
+        assert!(result.all_converged);
+        assert!(result.node_results.len() > 1);
+    }
+
+    #[test]
+    fn test_convergence_result_summary() {
+        let ops = vec![create_test_operation(1, "records", "rec1", 10)];
+
+        let simulator = MultiNodeConvergenceSimulator::new(3);
+        let result = simulator.test_deterministic_convergence(&ops);
+
+        let summary = result.summary();
+        assert!(summary.contains("Convergence"));
+    }
+
+    #[test]
+    fn test_aggregation_summary() {
+        let ops = vec![
+            create_test_operation(1, "records", "rec1", 10),
+            create_test_operation(2, "records", "rec2", 20),
+        ];
+
+        let simulator = MultiNodeConvergenceSimulator::new(3);
+        let agg = simulator.aggregate_convergence_runs(&ops, 3);
+
+        let summary = agg.summary();
+        assert!(summary.contains("Summary"));
+        assert_eq!(agg.total_operations_tested, 2);
+    }
+
+    #[test]
+    fn test_deterministic_vs_minimal_vs_comprehensive() {
+        let ops = vec![
+            create_test_operation(1, "records", "rec1", 10),
+            create_test_operation(2, "records", "rec2", 20),
+        ];
+
+        let simulator = MultiNodeConvergenceSimulator::new(3);
+
+        let det = simulator.test_deterministic_convergence(&ops);
+        let min = simulator.test_minimal_convergence(&ops);
+        let comp = simulator.test_comprehensive_convergence(&ops);
+
+        // deterministic should have fewer results than minimal/comprehensive
+        assert!(det.node_results.len() <= min.node_results.len());
+        assert!(min.node_results.len() <= comp.node_results.len());
+
+        // All should converge
+        assert!(det.all_converged);
+        assert!(min.all_converged);
+        assert!(comp.all_converged);
+    }
+}

package/dist/create/server-source/crates/feltdb/src/observability.rs

@@ -0,0 +1,366 @@
+/// Observability Layer
+///
+/// Comprehensive monitoring and observability:
+/// - Replication lag tracking
+/// - Convergence status
+/// - Query latency monitoring
+/// - Rejected operations tracking
+/// - Index health metrics
+/// - Recovery state visibility
+
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicU64, Ordering};
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ReplicationLagMetric {
+    pub source_node: String,
+    pub target_node: String,
+    pub lag_ms: u64,
+    pub pending_operations: u64,
+    pub last_update: u64,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ConvergenceStatus {
+    pub node_id: String,
+    pub status: String,
+    pub synchronized_peers: Vec<String>,
+    pub diverged_peers: Vec<String>,
+    pub convergence_percentage: f64,
+    pub timestamp: u64,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct RejectedOperation {
+    pub operation_id: String,
+    pub reason: String,
+    pub timestamp: u64,
+    pub node_id: String,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct IndexHealth {
+    pub index_name: String,
+    pub status: String,
+    pub entries_count: u64,
+    pub memory_bytes: u64,
+    pub last_compaction: u64,
+    pub fragmentation: f64,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct RecoveryState {
+    pub node_id: String,
+    pub status: String,
+    pub operations_replayed: u64,
+    pub operations_remaining: u64,
+    pub recovery_start: u64,
+    pub estimated_completion: u64,
+    pub progress_percentage: f64,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct HealthReport {
+    pub timestamp: u64,
+    pub overall_status: String,
+    pub replication_lag: Vec<ReplicationLagMetric>,
+    pub convergence: Vec<ConvergenceStatus>,
+    pub rejected_operations: Vec<RejectedOperation>,
+    pub index_health: Vec<IndexHealth>,
+    pub recovery_states: Vec<RecoveryState>,
+    pub query_latency_p99: u64,
+    pub operation_throughput: u64,
+}
+
+pub struct ObservabilityManager {
+    node_id: String,
+    replication_lags: HashMap<String, ReplicationLagMetric>,
+    convergence_status: HashMap<String, ConvergenceStatus>,
+    rejected_operations: Vec<RejectedOperation>,
+    index_health: HashMap<String, IndexHealth>,
+    recovery_states: HashMap<String, RecoveryState>,
+
+    query_latency_p99: Arc<AtomicU64>,
+    operation_throughput: Arc<AtomicU64>,
+    total_rejected: Arc<AtomicU64>,
+}
+
+impl ObservabilityManager {
+    pub fn new(node_id: String) -> Self {
+        ObservabilityManager {
+            node_id,
+            replication_lags: HashMap::new(),
+            convergence_status: HashMap::new(),
+            rejected_operations: Vec::new(),
+            index_health: HashMap::new(),
+            recovery_states: HashMap::new(),
+            query_latency_p99: Arc::new(AtomicU64::new(0)),
+            operation_throughput: Arc::new(AtomicU64::new(0)),
+            total_rejected: Arc::new(AtomicU64::new(0)),
+        }
+    }
+
+    pub fn record_replication_lag(&mut self, source: String, target: String, lag_ms: u64, pending: u64) {
+        let now = std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap_or_default()
+            .as_secs();
+
+        self.replication_lags.insert(
+            format!("{}:{}", source.clone(), target.clone()),
+            ReplicationLagMetric {
+                source_node: source,
+                target_node: target,
+                lag_ms,
+                pending_operations: pending,
+                last_update: now,
+            },
+        );
+    }
+
+    pub fn record_convergence_status(
+        &mut self,
+        node_id: String,
+        synchronized_peers: Vec<String>,
+        diverged_peers: Vec<String>,
+    ) {
+        let total = synchronized_peers.len() + diverged_peers.len();
+        let convergence_percentage = if total > 0 {
+            (synchronized_peers.len() as f64 / total as f64) * 100.0
+        } else {
+            100.0
+        };
+
+        let status = if diverged_peers.is_empty() {
+            "converged"
+        } else {
+            "diverged"
+        };
+
+        self.convergence_status.insert(
+            node_id.clone(),
+            ConvergenceStatus {
+                node_id,
+                status: status.to_string(),
+                synchronized_peers,
+                diverged_peers,
+                convergence_percentage,
+                timestamp: std::time::SystemTime::now()
+                    .duration_since(std::time::UNIX_EPOCH)
+                    .unwrap_or_default()
+                    .as_secs(),
+            },
+        );
+    }
+
+    pub fn record_rejected_operation(&mut self, operation_id: String, reason: String) {
+        self.rejected_operations.push(RejectedOperation {
+            operation_id,
+            reason,
+            timestamp: std::time::SystemTime::now()
+                .duration_since(std::time::UNIX_EPOCH)
+                .unwrap_or_default()
+                .as_secs(),
+            node_id: self.node_id.clone(),
+        });
+
+        self.total_rejected.fetch_add(1, Ordering::Relaxed);
+
+        if self.rejected_operations.len() > 10000 {
+            self.rejected_operations.remove(0);
+        }
+    }
+
+    pub fn record_index_health(
+        &mut self,
+        index_name: String,
+        entries: u64,
+        memory_bytes: u64,
+        fragmentation: f64,
+    ) {
+        self.index_health.insert(
+            index_name.clone(),
+            IndexHealth {
+                index_name,
+                status: "healthy".to_string(),
+                entries_count: entries,
+                memory_bytes,
+                last_compaction: std::time::SystemTime::now()
+                    .duration_since(std::time::UNIX_EPOCH)
+                    .unwrap_or_default()
+                    .as_secs(),
+                fragmentation,
+            },
+        );
+    }
+
+    pub fn record_recovery_state(
+        &mut self,
+        node_id: String,
+        replayed: u64,
+        remaining: u64,
+        start_time: u64,
+    ) {
+        let progress_percentage = if replayed + remaining > 0 {
+            (replayed as f64 / (replayed + remaining) as f64) * 100.0
+        } else {
+            100.0
+        };
+
+        let now = std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap_or_default()
+            .as_secs();
+
+        let elapsed = now - start_time;
+        let rate = if elapsed > 0 {
+            replayed / elapsed.max(1)
+        } else {
+            0
+        };
+
+        let estimated_completion = if rate > 0 {
+            now + ((remaining / rate.max(1)) as u64)
+        } else {
+            now + 3600
+        };
+
+        self.recovery_states.insert(
+            node_id.clone(),
+            RecoveryState {
+                node_id,
+                status: if remaining == 0 {
+                    "complete".to_string()
+                } else {
+                    "in_progress".to_string()
+                },
+                operations_replayed: replayed,
+                operations_remaining: remaining,
+                recovery_start: start_time,
+                estimated_completion,
+                progress_percentage,
+            },
+        );
+    }
+
+    pub fn record_query_latency(&self, p99_us: u64) {
+        self.query_latency_p99.store(p99_us, Ordering::Relaxed);
+    }
+
+    pub fn record_operation_throughput(&self, ops_per_sec: u64) {
+        self.operation_throughput.store(ops_per_sec, Ordering::Relaxed);
+    }
+
+    pub fn get_health_report(&self) -> HealthReport {
+        let now = std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap_or_default()
+            .as_secs();
+
+        let overall_status = if self.convergence_status
+            .values()
+            .all(|c| c.diverged_peers.is_empty())
+        {
+            "healthy"
+        } else {
+            "degraded"
+        };
+
+        HealthReport {
+            timestamp: now,
+            overall_status: overall_status.to_string(),
+            replication_lag: self.replication_lags.values().cloned().collect(),
+            convergence: self.convergence_status.values().cloned().collect(),
+            rejected_operations: self.rejected_operations.iter().rev().take(100).cloned().collect(),
+            index_health: self.index_health.values().cloned().collect(),
+            recovery_states: self.recovery_states.values().cloned().collect(),
+            query_latency_p99: self.query_latency_p99.load(Ordering::Relaxed),
+            operation_throughput: self.operation_throughput.load(Ordering::Relaxed),
+        }
+    }
+
+    pub fn export_prometheus_metrics(&self) -> String {
+        let report = self.get_health_report();
+        let mut metrics = String::new();
+
+        metrics.push_str(&format!(
+            "feltdb_query_latency_p99_us {}\n",
+            report.query_latency_p99
+        ));
+        metrics.push_str(&format!(
+            "feltdb_operation_throughput_ops_per_sec {}\n",
+            report.operation_throughput
+        ));
+        metrics.push_str(&format!(
+            "feltdb_rejected_operations_total {}\n",
+            self.total_rejected.load(Ordering::Relaxed)
+        ));
+
+        for lag in &report.replication_lag {
+            metrics.push_str(&format!(
+                "feltdb_replication_lag_ms{{source=\"{}\",target=\"{}\"}} {}\n",
+                lag.source_node, lag.target_node, lag.lag_ms
+            ));
+        }
+
+        metrics
+    }
+}
+
+impl Default for ObservabilityManager {
+    fn default() -> Self {
+        Self::new("node".to_string())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_replication_lag_tracking() {
+        let mut obs = ObservabilityManager::new("node-1".to_string());
+        obs.record_replication_lag("node-1".to_string(), "node-2".to_string(), 150, 50);
+
+        let report = obs.get_health_report();
+        assert_eq!(report.replication_lag.len(), 1);
+        assert_eq!(report.replication_lag[0].lag_ms, 150);
+    }
+
+    #[test]
+    fn test_convergence_tracking() {
+        let mut obs = ObservabilityManager::new("node-1".to_string());
+        obs.record_convergence_status(
+            "node-1".to_string(),
+            vec!["node-2".to_string(), "node-3".to_string()],
+            vec![],
+        );
+
+        let report = obs.get_health_report();
+        assert_eq!(report.overall_status, "healthy");
+        assert_eq!(report.convergence[0].convergence_percentage, 100.0);
+    }
+
+    #[test]
+    fn test_rejected_operations() {
+        let mut obs = ObservabilityManager::new("node-1".to_string());
+        obs.record_rejected_operation("op-1".to_string(), "conflict".to_string());
+        obs.record_rejected_operation("op-2".to_string(), "timeout".to_string());
+
+        let report = obs.get_health_report();
+        assert_eq!(report.rejected_operations.len(), 2);
+    }
+
+    #[test]
+    fn test_prometheus_export() {
+        let mut obs = ObservabilityManager::new("node-1".to_string());
+        obs.record_query_latency(1200);
+        obs.record_operation_throughput(5000);
+
+        let metrics = obs.export_prometheus_metrics();
+        assert!(metrics.contains("feltdb_query_latency_p99_us 1200"));
+        assert!(metrics.contains("feltdb_operation_throughput_ops_per_sec 5000"));
+    }
+}