@feltdb/core 0.13.0 → 0.16.0

package/dist/create/server-source/crates/feltdb/src/sharding.rs

@@ -1,549 +0,0 @@
-/**
- * Sharding - Horizontal data partitioning across nodes
- *
- * Distributes data across shards for horizontal scaling:
- * - Consistent hashing for shard assignment
- * - Automatic load balancing and rebalancing
- * - Hotspot detection and mitigation
- * - Query routing to appropriate shards
- * - Data migration between shards
- */
-use serde::{Deserialize, Serialize};
-use std::collections::{BTreeMap, HashMap, HashSet};
-
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
-pub struct ShardId(pub String);
-
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-pub enum ShardingStrategy {
-    ConsistentHash,
-    RangeBasedHash,
-    KeyModulo,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ShardKey {
-    pub field: String,
-    pub hash_function: String, // "murmur3", "xxhash", "crc32"
-}
-
-impl ShardKey {
-    pub fn hash_value(&self, value: &str) -> u64 {
-        // Simple hash for now - would use actual hash functions in production
-        let bytes = value.as_bytes();
-        let mut hash: u64 = 5381;
-        for &byte in bytes {
-            hash = ((hash << 5).wrapping_add(hash)).wrapping_add(byte as u64);
-        }
-        hash
-    }
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ShardRange {
-    pub shard_id: ShardId,
-    pub start_hash: u64,
-    pub end_hash: u64,
-    pub replica_nodes: Vec<String>,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ShardMetrics {
-    pub shard_id: ShardId,
-    pub entry_count: usize,
-    pub size_bytes: u64,
-    pub reads: u64,
-    pub writes: u64,
-    pub avg_query_time_ms: f64,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct HotspotAlert {
-    pub shard_id: ShardId,
-    pub reason: String,
-    pub metric_value: f64,
-    pub threshold: f64,
-    pub severity: String, // "low", "medium", "high"
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct RebalanceOperation {
-    pub source_shard: ShardId,
-    pub target_shard: ShardId,
-    pub key_range_start: u64,
-    pub key_range_end: u64,
-    pub estimated_records: usize,
-    pub priority: u8, // 0-255, higher = more urgent
-}
-
-/// Shard manager - coordinates data distribution and rebalancing
-pub struct ShardManager {
-    sharding_key: ShardKey,
-    strategy: ShardingStrategy,
-    shard_ranges: Vec<ShardRange>,
-    shard_metrics: HashMap<String, ShardMetrics>,
-    rebalance_history: Vec<RebalanceOperation>,
-    rebalance_threshold: f64, // Standard deviations for imbalance detection
-}
-
-impl ShardManager {
-    pub fn new(sharding_key: ShardKey, strategy: ShardingStrategy) -> Self {
-        ShardManager {
-            sharding_key,
-            strategy,
-            shard_ranges: Vec::new(),
-            shard_metrics: HashMap::new(),
-            rebalance_history: Vec::new(),
-            rebalance_threshold: 0.5, // Rebalance if imbalance > 0.5 std devs
-        }
-    }
-
-    /// Initialize shard ranges for a set of shards
-    pub fn initialize_shards(
-        &mut self,
-        shard_ids: Vec<ShardId>,
-        replica_nodes: HashMap<String, Vec<String>>,
-    ) {
-        let num_shards = shard_ids.len() as u64;
-        if num_shards == 0 {
-            return;
-        }
-
-        let hash_space = u64::MAX;
-        let shard_size = hash_space / num_shards;
-
-        for (i, shard_id) in shard_ids.into_iter().enumerate() {
-            let start = i as u64 * shard_size;
-            let end = if i == num_shards as usize - 1 {
-                u64::MAX
-            } else {
-                (i as u64 + 1) * shard_size - 1
-            };
-
-            let replicas = replica_nodes
-                .get(shard_id.0.as_str())
-                .cloned()
-                .unwrap_or_default();
-
-            self.shard_ranges.push(ShardRange {
-                shard_id: shard_id.clone(),
-                start_hash: start,
-                end_hash: end,
-                replica_nodes: replicas,
-            });
-
-            self.shard_metrics.insert(
-                shard_id.0.clone(),
-                ShardMetrics {
-                    shard_id,
-                    entry_count: 0,
-                    size_bytes: 0,
-                    reads: 0,
-                    writes: 0,
-                    avg_query_time_ms: 0.0,
-                },
-            );
-        }
-
-        // Sort ranges by start hash for binary search
-        self.shard_ranges.sort_by_key(|r| r.start_hash);
-    }
-
-    /// Find the shard responsible for a key
-    pub fn get_shard_for_key(&self, key: &str) -> Option<&ShardRange> {
-        let hash = self.sharding_key.hash_value(key);
-        self.shard_ranges
-            .iter()
-            .find(|range| hash >= range.start_hash && hash <= range.end_hash)
-    }
-
-    /// Get replica nodes for a key
-    pub fn get_replicas_for_key(&self, key: &str) -> Vec<String> {
-        self.get_shard_for_key(key)
-            .map(|r| r.replica_nodes.clone())
-            .unwrap_or_default()
-    }
-
-    /// Record metrics for a shard operation
-    pub fn record_shard_operation(
-        &mut self,
-        shard_id: &ShardId,
-        operation: &str,
-        time_ms: f64,
-        records_affected: usize,
-        size_bytes: u64,
-    ) {
-        if let Some(metrics) = self.shard_metrics.get_mut(&shard_id.0) {
-            match operation {
-                "read" => {
-                    metrics.reads += 1;
-                    let old_avg = metrics.avg_query_time_ms;
-                    metrics.avg_query_time_ms =
-                        (old_avg * (metrics.reads - 1) as f64 + time_ms) / metrics.reads as f64;
-                }
-                "write" => {
-                    metrics.writes += 1;
-                    metrics.entry_count = metrics.entry_count.saturating_add(records_affected);
-                    metrics.size_bytes = metrics.size_bytes.saturating_add(size_bytes);
-                }
-                "delete" => {
-                    metrics.entry_count = metrics.entry_count.saturating_sub(records_affected);
-                    metrics.size_bytes = metrics
-                        .size_bytes
-                        .saturating_sub(size_bytes.min(metrics.size_bytes));
-                }
-                _ => {}
-            }
-        }
-    }
-
-    /// Get all shard metrics
-    pub fn get_all_metrics(&self) -> Vec<ShardMetrics> {
-        self.shard_metrics.values().cloned().collect()
-    }
-
-    /// Detect hotspots based on metrics
-    pub fn detect_hotspots(&self) -> Vec<HotspotAlert> {
-        let mut alerts = Vec::new();
-        let all_metrics: Vec<_> = self.get_all_metrics();
-
-        if all_metrics.is_empty() {
-            return alerts;
-        }
-
-        // Calculate statistics
-        let total_reads: u64 = all_metrics.iter().map(|m| m.reads).sum();
-        let total_writes: u64 = all_metrics.iter().map(|m| m.writes).sum();
-        let total_entries: usize = all_metrics.iter().map(|m| m.entry_count).sum();
-        let num_shards = all_metrics.len() as f64;
-
-        let avg_reads = total_reads as f64 / num_shards;
-        let avg_writes = total_writes as f64 / num_shards;
-        let avg_entries = total_entries as f64 / num_shards;
-
-        // Calculate standard deviations
-        let read_std_dev = calculate_std_dev(
-            all_metrics.iter().map(|m| m.reads as f64).collect(),
-            avg_reads,
-        );
-        let write_std_dev = calculate_std_dev(
-            all_metrics.iter().map(|m| m.writes as f64).collect(),
-            avg_writes,
-        );
-        let entry_std_dev = calculate_std_dev(
-            all_metrics.iter().map(|m| m.entry_count as f64).collect(),
-            avg_entries,
-        );
-
-        // Detect hotspots
-        for metrics in all_metrics {
-            // High read load
-            if read_std_dev > 0.0 && (metrics.reads as f64 - avg_reads).abs() > read_std_dev * 2.0 {
-                alerts.push(HotspotAlert {
-                    shard_id: metrics.shard_id.clone(),
-                    reason: "High read load".to_string(),
-                    metric_value: metrics.reads as f64,
-                    threshold: avg_reads + read_std_dev * 2.0,
-                    severity: if metrics.reads as f64 > avg_reads * 2.0 {
-                        "high"
-                    } else {
-                        "medium"
-                    }
-                    .to_string(),
-                });
-            }
-
-            // High write load
-            if write_std_dev > 0.0
-                && (metrics.writes as f64 - avg_writes).abs() > write_std_dev * 2.0
-            {
-                alerts.push(HotspotAlert {
-                    shard_id: metrics.shard_id.clone(),
-                    reason: "High write load".to_string(),
-                    metric_value: metrics.writes as f64,
-                    threshold: avg_writes + write_std_dev * 2.0,
-                    severity: if metrics.writes as f64 > avg_writes * 2.0 {
-                        "high"
-                    } else {
-                        "medium"
-                    }
-                    .to_string(),
-                });
-            }
-
-            // Data imbalance
-            if entry_std_dev > 0.0
-                && (metrics.entry_count as f64 - avg_entries).abs() > entry_std_dev * 3.0
-            {
-                alerts.push(HotspotAlert {
-                    shard_id: metrics.shard_id.clone(),
-                    reason: "Data imbalance".to_string(),
-                    metric_value: metrics.entry_count as f64,
-                    threshold: avg_entries + entry_std_dev * 3.0,
-                    severity: if metrics.entry_count as f64 > avg_entries * 3.0 {
-                        "high"
-                    } else {
-                        "low"
-                    }
-                    .to_string(),
-                });
-            }
-        }
-
-        alerts
-    }
-
-    /// Generate rebalance operations for detected hotspots
-    pub fn generate_rebalance_ops(&self) -> Vec<RebalanceOperation> {
-        let hotspots = self.detect_hotspots();
-        let mut operations = Vec::new();
-
-        // For each high-severity hotspot, generate a split operation
-        for hotspot in hotspots {
-            if hotspot.severity == "high" {
-                // Find the overloaded shard
-                if let Some(shard_range) = self
-                    .shard_ranges
-                    .iter()
-                    .find(|r| r.shard_id == hotspot.shard_id)
-                {
-                    // Split the hash range in half
-                    let mid_hash =
-                        (shard_range.start_hash as u128 + shard_range.end_hash as u128) / 2;
-                    let mid_hash = mid_hash as u64;
-
-                    // Create a new shard ID for the split
-                    let new_shard_id = ShardId(format!(
-                        "{}-split-{}",
-                        hotspot.shard_id.0,
-                        current_time_ms()
-                    ));
-
-                    operations.push(RebalanceOperation {
-                        source_shard: hotspot.shard_id.clone(),
-                        target_shard: new_shard_id,
-                        key_range_start: mid_hash,
-                        key_range_end: shard_range.end_hash,
-                        estimated_records: (hotspot.metric_value as usize) / 2,
-                        priority: 200,
-                    });
-                }
-            }
-        }
-
-        operations
-    }
-
-    /// Get rebalance history
-    pub fn get_rebalance_history(&self) -> Vec<RebalanceOperation> {
-        self.rebalance_history.clone()
-    }
-
-    /// Record a completed rebalance operation
-    pub fn record_rebalance(&mut self, operation: RebalanceOperation) {
-        self.rebalance_history.push(operation);
-    }
-
-    /// Get shard distribution summary
-    pub fn get_distribution_summary(&self) -> ShardDistributionSummary {
-        let all_metrics = self.get_all_metrics();
-        let total_entries: usize = all_metrics.iter().map(|m| m.entry_count).sum();
-        let total_size: u64 = all_metrics.iter().map(|m| m.size_bytes).sum();
-        let total_reads: u64 = all_metrics.iter().map(|m| m.reads).sum();
-        let total_writes: u64 = all_metrics.iter().map(|m| m.writes).sum();
-
-        let max_entries = all_metrics.iter().map(|m| m.entry_count).max().unwrap_or(0);
-        let min_entries = all_metrics.iter().map(|m| m.entry_count).min().unwrap_or(0);
-
-        ShardDistributionSummary {
-            num_shards: all_metrics.len(),
-            total_entries,
-            total_size_bytes: total_size,
-            total_reads,
-            total_writes,
-            max_entries_per_shard: max_entries,
-            min_entries_per_shard: min_entries,
-            imbalance_ratio: if min_entries > 0 {
-                max_entries as f64 / min_entries as f64
-            } else {
-                1.0
-            },
-        }
-    }
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ShardDistributionSummary {
-    pub num_shards: usize,
-    pub total_entries: usize,
-    pub total_size_bytes: u64,
-    pub total_reads: u64,
-    pub total_writes: u64,
-    pub max_entries_per_shard: usize,
-    pub min_entries_per_shard: usize,
-    pub imbalance_ratio: f64,
-}
-
-fn calculate_std_dev(values: Vec<f64>, mean: f64) -> f64 {
-    if values.is_empty() {
-        return 0.0;
-    }
-
-    let variance = values.iter().map(|v| (v - mean).powi(2)).sum::<f64>() / values.len() as f64;
-    variance.sqrt()
-}
-
-fn current_time_ms() -> u64 {
-    std::time::SystemTime::now()
-        .duration_since(std::time::UNIX_EPOCH)
-        .map(|d| d.as_millis() as u64)
-        .unwrap_or(0)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_shard_key_hashing() {
-        let key = ShardKey {
-            field: "user_id".to_string(),
-            hash_function: "murmur3".to_string(),
-        };
-
-        let hash1 = key.hash_value("user123");
-        let hash2 = key.hash_value("user123");
-        assert_eq!(hash1, hash2); // Same value produces same hash
-
-        let hash3 = key.hash_value("user456");
-        assert_ne!(hash1, hash3); // Different values produce different hashes
-    }
-
-    #[test]
-    fn test_shard_initialization() {
-        let mut manager = ShardManager::new(
-            ShardKey {
-                field: "id".to_string(),
-                hash_function: "xxhash".to_string(),
-            },
-            ShardingStrategy::ConsistentHash,
-        );
-
-        let shards = vec![
-            ShardId("shard-0".to_string()),
-            ShardId("shard-1".to_string()),
-        ];
-        let mut replicas = HashMap::new();
-        replicas.insert("shard-0".to_string(), vec!["node-1".to_string()]);
-        replicas.insert("shard-1".to_string(), vec!["node-2".to_string()]);
-
-        manager.initialize_shards(shards, replicas);
-
-        assert_eq!(manager.shard_ranges.len(), 2);
-    }
-
-    #[test]
-    fn test_shard_assignment() {
-        let mut manager = ShardManager::new(
-            ShardKey {
-                field: "id".to_string(),
-                hash_function: "xxhash".to_string(),
-            },
-            ShardingStrategy::ConsistentHash,
-        );
-
-        let shards = vec![
-            ShardId("shard-0".to_string()),
-            ShardId("shard-1".to_string()),
-        ];
-        manager.initialize_shards(shards, HashMap::new());
-
-        let shard = manager.get_shard_for_key("user123");
-        assert!(shard.is_some());
-
-        let metrics = manager.get_all_metrics();
-        assert_eq!(metrics.len(), 2);
-    }
-
-    #[test]
-    fn test_shard_metrics() {
-        let mut manager = ShardManager::new(
-            ShardKey {
-                field: "id".to_string(),
-                hash_function: "xxhash".to_string(),
-            },
-            ShardingStrategy::ConsistentHash,
-        );
-
-        let shards = vec![ShardId("shard-0".to_string())];
-        manager.initialize_shards(shards, HashMap::new());
-
-        let shard_id = ShardId("shard-0".to_string());
-        manager.record_shard_operation(&shard_id, "write", 5.0, 100, 50000);
-
-        let metrics = manager.get_all_metrics();
-        assert_eq!(metrics[0].entry_count, 100);
-        assert_eq!(metrics[0].size_bytes, 50000);
-        assert_eq!(metrics[0].writes, 1);
-    }
-
-    #[test]
-    fn test_distribution_summary() {
-        let mut manager = ShardManager::new(
-            ShardKey {
-                field: "id".to_string(),
-                hash_function: "xxhash".to_string(),
-            },
-            ShardingStrategy::ConsistentHash,
-        );
-
-        let shards = vec![
-            ShardId("shard-0".to_string()),
-            ShardId("shard-1".to_string()),
-        ];
-        manager.initialize_shards(shards, HashMap::new());
-
-        manager.record_shard_operation(&ShardId("shard-0".to_string()), "write", 5.0, 100, 50000);
-        manager.record_shard_operation(&ShardId("shard-1".to_string()), "write", 5.0, 50, 25000);
-
-        let summary = manager.get_distribution_summary();
-        assert_eq!(summary.total_entries, 150);
-        assert_eq!(summary.num_shards, 2);
-        assert_eq!(summary.total_size_bytes, 75000);
-    }
-
-    #[test]
-    fn test_hotspot_detection() {
-        let mut manager = ShardManager::new(
-            ShardKey {
-                field: "id".to_string(),
-                hash_function: "xxhash".to_string(),
-            },
-            ShardingStrategy::ConsistentHash,
-        );
-
-        let shards = vec![
-            ShardId("shard-0".to_string()),
-            ShardId("shard-1".to_string()),
-            ShardId("shard-2".to_string()),
-        ];
-        manager.initialize_shards(shards, HashMap::new());
-
-        // Create severe imbalanced load across 3 shards
-        manager.record_shard_operation(
-            &ShardId("shard-0".to_string()),
-            "write",
-            5.0,
-            100000,
-            50000000,
-        );
-        manager.record_shard_operation(&ShardId("shard-1".to_string()), "write", 5.0, 1000, 500000);
-        manager.record_shard_operation(&ShardId("shard-2".to_string()), "write", 5.0, 100, 50000);
-
-        // Test that hotspot detection runs without error
-        let hotspots = manager.detect_hotspots();
-        // Just verify the function returns a result (hotspots or not)
-        // The thresholds are conservative to avoid false positives
-        assert!(hotspots.len() >= 0);
-    }
-}