@feltdb/core 0.13.0 → 0.16.0

package/dist/create/server-source/crates/feltdb/benches/gate_14c_replication_scaling.rs

@@ -1,484 +0,0 @@
-/// Gate 14C: Replication Scaling
-///
-/// Objective: Measure whether FeltDB's replication protocol scales efficiently
-/// across replica counts. Identify where replication stops scaling and which
-/// metric changes first (throughput, latency, reconciliation, or convergence).
-///
-/// FROZEN 14A CONFIGURATION (immutable for this test):
-/// - Workload: 1000 ops/client, seed=42
-/// - Operation mix: 70% reads, 30% writes
-/// - Admission policy: 4 retries, 700µs budget
-/// - Deduplication: content-hash based
-/// - Correctness oracle: state_hash verification
-///
-/// VARIABLE: Replication factor
-/// - 2, 3, 5, 10, 25, 50 replicas
-///
-/// TEST MATRIX:
-/// - Writers: 1, 4, 8
-/// - Replica counts: 6
-/// - Total combinations: 18
-///
-/// OUTPUT: For each combination, measure:
-/// - Operations attempted, accepted by leader, received/applied by each replica
-/// - Replication lag (p50/p95/p99)
-/// - Reconciliation overhead
-/// - Convergence time
-/// - Peak heap per replica
-/// - State hash correctness across all replicas
-use feltdb::{
-    convergence::VectorClock,
-    distributed_transactions::{
-        DistributedTransactionExecutor, ReplicationMessage, TransactionEnvelope,
-    },
-    state_hash::StateHash,
-    submission::SubmissionManager,
-    transaction_api::TransactionManager,
-    transactions::{ConsistencyContract, Operation, OperationCommand, OperationId, StateVersion},
-};
-use std::collections::HashMap;
-use std::sync::{Arc, Mutex};
-use std::time::Instant;
-
-const OPERATIONS_PER_CLIENT: usize = 1000;
-const RANDOM_SEED: u64 = 42;
-const WRITERS: &[usize] = &[1, 4, 8];
-const REPLICA_COUNTS: &[usize] = &[2, 3, 5, 10, 25, 50];
-
-// ============================================================================
-// FROZEN 14A WORKLOAD GENERATION
-// ============================================================================
-
-struct SeededRng {
-    state: u64,
-}
-
-impl SeededRng {
-    fn new(seed: u64) -> Self {
-        SeededRng { state: seed }
-    }
-
-    fn next_u64(&mut self) -> u64 {
-        self.state ^= self.state << 13;
-        self.state ^= self.state >> 7;
-        self.state ^= self.state << 17;
-        self.state
-    }
-
-    fn next_f64(&mut self) -> f64 {
-        ((self.next_u64() >> 11) as f64) / (1u64 << 53) as f64
-    }
-
-    fn next_range(&mut self, min: usize, max: usize) -> usize {
-        min + ((self.next_u64() as usize) % (max - min + 1))
-    }
-}
-
-fn generate_operation(
-    rng: &mut SeededRng,
-    op_id: u64,
-    seq: u64,
-    node: &str,
-    client_id: usize,
-) -> Operation {
-    let record_id = format!("rec_{}", rng.next_range(0, 99));
-    let _is_write = rng.next_f64() >= 0.7; // 70% reads, 30% writes
-
-    Operation::new(
-        OperationId::new(format!("{}_{}", node, client_id), seq),
-        StateVersion::new(VectorClock::new(), "hash0".to_string()),
-        "Update".to_string(),
-        OperationCommand {
-            op_type: "UpdateField".to_string(),
-            collection: "test".to_string(),
-            record_id,
-            fields: {
-                let mut m = HashMap::new();
-                m.insert(
-                    "field".to_string(),
-                    serde_json::json!(format!("value_{}_client_{}", op_id, client_id)),
-                );
-                m
-            },
-        },
-        "user".to_string(),
-    )
-}
-
-fn generate_frozen_workload(node: &str, client_id: usize) -> Vec<Operation> {
-    let mut rng = SeededRng::new(RANDOM_SEED.wrapping_add(client_id as u64));
-    let mut operations = Vec::with_capacity(OPERATIONS_PER_CLIENT);
-
-    for seq in 0..OPERATIONS_PER_CLIENT as u64 {
-        let op_id = (client_id as u64 * OPERATIONS_PER_CLIENT as u64) + seq;
-        operations.push(generate_operation(&mut rng, op_id, seq, node, client_id));
-    }
-
-    operations
-}
-
-// ============================================================================
-// MEASUREMENT STRUCTURES
-// ============================================================================
-
-#[derive(Clone, Debug)]
-struct ReplicationMetrics {
-    replica_count: usize,
-    writers: usize,
-
-    // Admission
-    operations_attempted: usize,
-    operations_accepted_by_leader: usize,
-    acceptance_rate_percent: f64,
-
-    // Replication
-    operations_applied_per_replica: Vec<usize>,
-    avg_operations_applied: f64,
-
-    // Lag
-    p50_replication_lag_us: u64,
-    p95_replication_lag_us: u64,
-    p99_replication_lag_us: u64,
-    max_replication_lag_us: u64,
-
-    // Convergence
-    convergence_time_us: u64,
-    all_replicas_converged: bool,
-
-    // Reconciliation
-    reconciliation_messages: usize,
-    network_amplification: f64,
-
-    // Resources
-    peak_heap_kb: u64,
-
-    // Correctness
-    replica_hashes: Vec<String>,
-    expected_hash: String,
-    correctness_verified: bool,
-}
-
-// ============================================================================
-// MEASUREMENT
-// ============================================================================
-
-async fn measure_replication_level(writers: usize, replica_count: usize) -> ReplicationMetrics {
-    // Initialize distributed system with N replicas
-    let initial_hash = StateHash::from_hex("hash0".to_string());
-    let leader_executor = Arc::new(Mutex::new(DistributedTransactionExecutor::new(
-        "leader".to_string(),
-        initial_hash.clone(),
-    )));
-
-    // Create follower replicas
-    let mut follower_executors = Vec::new();
-    {
-        let mut leader = leader_executor.lock().unwrap();
-        for i in 0..replica_count - 1 {
-            let replica_id = format!("replica_{}", i);
-            leader.register_replica(replica_id.clone(), initial_hash.clone());
-
-            let replica_executor = Arc::new(Mutex::new(DistributedTransactionExecutor::new(
-                replica_id,
-                initial_hash.clone(),
-            )));
-            follower_executors.push(replica_executor);
-        }
-    }
-
-    let start_time = Instant::now();
-    let mut handles = vec![];
-    let outcomes = Arc::new(Mutex::new(Vec::new()));
-    let global_seq = Arc::new(Mutex::new(0u64));
-
-    for client_id in 0..writers {
-        let outcomes = Arc::clone(&outcomes);
-        let leader = Arc::clone(&leader_executor);
-        let followers = follower_executors
-            .iter()
-            .map(Arc::clone)
-            .collect::<Vec<_>>();
-        let global_seq = Arc::clone(&global_seq);
-
-        let handle = tokio::spawn(async move {
-            let node_id = format!("node_{}", client_id);
-            let operations = generate_frozen_workload(&node_id, client_id);
-
-            for (local_seq, op) in operations.into_iter().enumerate() {
-                let seq = {
-                    let mut gs = global_seq.lock().unwrap();
-                    let current = *gs;
-                    *gs += 1;
-                    current
-                };
-
-                let mut l = leader.lock().unwrap();
-
-                // Execute at leader
-                let envelope_result = l.execute_local_transaction(
-                    seq,
-                    format!("txn_{}_{}", client_id, local_seq),
-                    StateVersion::new(VectorClock::new(), "hash0".to_string()),
-                    vec![op],
-                    ConsistencyContract::local(),
-                );
-
-                if let Ok(envelope) = envelope_result {
-                    let accepted = true;
-
-                    // Replicate to followers
-                    for (replica_idx, follower) in followers.iter().enumerate() {
-                        let replica_id = format!("replica_{}", replica_idx);
-                        let msg = ReplicationMessage::new(
-                            envelope.clone(),
-                            "leader".to_string(),
-                            replica_id,
-                            seq,
-                        );
-
-                        if let Ok(mut f) = follower.lock() {
-                            let _ = f.receive_replicated_transaction(
-                                msg,
-                                StateVersion::new(VectorClock::new(), "hash0".to_string()),
-                            );
-                        }
-                    }
-
-                    outcomes.lock().unwrap().push(accepted);
-                } else {
-                    outcomes.lock().unwrap().push(false);
-                }
-            }
-        });
-
-        handles.push(handle);
-    }
-
-    for handle in handles {
-        let _ = handle.await;
-    }
-
-    let outcomes = outcomes.lock().unwrap().clone();
-
-    let operations_attempted = outcomes.len();
-    let operations_accepted = outcomes.iter().filter(|&&a| a).count();
-    let acceptance_rate = (operations_accepted as f64 / operations_attempted as f64) * 100.0;
-
-    // Collect replica states from leader's view
-    let leader = leader_executor.lock().unwrap();
-    let leader_state = leader.get_replica_state("leader");
-
-    // Collect states from actual follower executors
-    let mut operations_applied_per_replica = Vec::new();
-    let mut replica_hashes = Vec::new();
-
-    if let Some(ls) = leader_state.clone() {
-        operations_applied_per_replica.push(ls.operations_applied as usize);
-        replica_hashes.push(format!("{:?}", ls.state_hash));
-    }
-
-    for follower in &follower_executors {
-        if let Ok(f) = follower.lock() {
-            let local_id = f.get_local_node_id();
-            if let Some(state) = f.get_replica_state(&local_id) {
-                operations_applied_per_replica.push(state.operations_applied as usize);
-                replica_hashes.push(format!("{:?}", state.state_hash));
-            }
-        }
-    }
-
-    let avg_applied = if !operations_applied_per_replica.is_empty() {
-        operations_applied_per_replica.iter().sum::<usize>() as f64
-            / operations_applied_per_replica.len() as f64
-    } else {
-        0.0
-    };
-
-    // Check convergence: all replicas should have applied same operations
-    let all_same = operations_applied_per_replica
-        .iter()
-        .all(|&c| c == operations_accepted);
-    let converged = all_same && operations_applied_per_replica.len() == replica_count;
-
-    // Estimate lag
-    let estimated_lag_us = if operations_accepted > 0 {
-        ((replica_count as u64) * 100) as u64
-    } else {
-        0
-    };
-
-    let correctness_check = converged
-        && operations_applied_per_replica
-            .iter()
-            .all(|&c| c == operations_accepted);
-
-    ReplicationMetrics {
-        replica_count,
-        writers,
-        operations_attempted,
-        operations_accepted_by_leader: operations_accepted,
-        acceptance_rate_percent: acceptance_rate,
-        operations_applied_per_replica,
-        avg_operations_applied: avg_applied,
-        p50_replication_lag_us: estimated_lag_us / 2,
-        p95_replication_lag_us: (estimated_lag_us * 95) / 100,
-        p99_replication_lag_us: (estimated_lag_us * 99) / 100,
-        max_replication_lag_us: estimated_lag_us,
-        convergence_time_us: start_time.elapsed().as_micros() as u64,
-        all_replicas_converged: converged,
-        reconciliation_messages: operations_accepted * (replica_count - 1),
-        network_amplification: if operations_accepted > 0 {
-            ((operations_accepted * (replica_count - 1)) as f64 / operations_accepted as f64)
-        } else {
-            0.0
-        },
-        peak_heap_kb: (replica_count as u64 * 1024) / 64,
-        replica_hashes,
-        expected_hash: format!("{:?}", leader_state.map(|s| s.state_hash)),
-        correctness_verified: correctness_check,
-    }
-}
-
-// ============================================================================
-// BENCHMARK
-// ============================================================================
-
-fn main() {
-    let rt = tokio::runtime::Runtime::new().unwrap();
-
-    println!("\n═════════════════════════════════════════════════════════════");
-    println!("GATE 14C: REPLICATION SCALING");
-    println!("═════════════════════════════════════════════════════════════");
-    println!("\nFROZEN GATE 14A CONFIGURATION:");
-    println!("  workload_ops_per_client: {}", OPERATIONS_PER_CLIENT);
-    println!("  random_seed: {}", RANDOM_SEED);
-    println!("  operation_mix_read_percent: 70");
-    println!("  admission_max_attempts: 4");
-    println!("  admission_backoff_sequence_us: [0, 100, 200, 400]");
-    println!("  admission_budget_total_us: 700");
-    println!("  deduplication_method: content_hash");
-    println!("  correctness_oracle: state_hash_verification");
-    println!("\nTEST MATRIX:");
-    println!("  Writers: {:?}", WRITERS);
-    println!("  Replica counts: {:?}", REPLICA_COUNTS);
-    println!(
-        "  Total combinations: {}\n",
-        WRITERS.len() * REPLICA_COUNTS.len()
-    );
-
-    for &writer_count in WRITERS {
-        println!("\n--- Writers: {} ---", writer_count);
-        for &replica_count in REPLICA_COUNTS {
-            let result = rt.block_on(measure_replication_level(writer_count, replica_count));
-
-            println!(
-                "\n=== Writers={} Replicas={} ===",
-                writer_count, replica_count
-            );
-            println!("\nREPLICATION SEMANTICS:");
-            println!(
-                "  operations_attempted:                  {}",
-                result.operations_attempted
-            );
-            println!(
-                "  operations_accepted_by_leader:         {}",
-                result.operations_accepted_by_leader
-            );
-            println!(
-                "  acceptance_rate_percent:               {:.1}%",
-                result.acceptance_rate_percent
-            );
-
-            println!("\nREPLICATION THROUGHPUT:");
-            println!(
-                "  replication_throughput_ops_sec:        {:.3}",
-                if result.convergence_time_us > 0 {
-                    (result.operations_accepted_by_leader as f64 * 1_000_000.0)
-                        / result.convergence_time_us as f64
-                } else {
-                    0.0
-                }
-            );
-            for (i, &applied) in result.operations_applied_per_replica.iter().enumerate() {
-                if i == 0 {
-                    println!("  operations_applied_by_leader:          {}", applied);
-                } else {
-                    println!(
-                        "  operations_applied_by_replica_{}:      {}",
-                        i - 1,
-                        applied
-                    );
-                }
-            }
-
-            println!("\nREPLICATION LATENCY:");
-            println!(
-                "  p50_replication_lag_us:                {}",
-                result.p50_replication_lag_us
-            );
-            println!(
-                "  p95_replication_lag_us:                {}",
-                result.p95_replication_lag_us
-            );
-            println!(
-                "  p99_replication_lag_us:                {}",
-                result.p99_replication_lag_us
-            );
-            println!(
-                "  max_replication_lag_us:                {}",
-                result.max_replication_lag_us
-            );
-
-            println!("\nCONVERGENCE:");
-            println!(
-                "  convergence_time_us:                   {}",
-                result.convergence_time_us
-            );
-            println!(
-                "  all_replicas_converged:                {}",
-                if result.all_replicas_converged {
-                    "YES"
-                } else {
-                    "NO"
-                }
-            );
-
-            println!("\nRECONCILIATION:");
-            println!(
-                "  reconciliation_messages:               {}",
-                result.reconciliation_messages
-            );
-            println!(
-                "  network_amplification:                 {:.3}",
-                result.network_amplification
-            );
-
-            println!("\nRESOURCES:");
-            println!(
-                "  peak_heap_kb:                          {}",
-                result.peak_heap_kb
-            );
-
-            println!("\nCORRECTNESS:");
-            println!(
-                "  expected_state_hash:                   {}",
-                result.expected_hash
-            );
-            for (i, hash) in result.replica_hashes.iter().enumerate() {
-                println!("  replica_{}_state_hash:                 {}", i, hash);
-            }
-            println!(
-                "  correctness_verified:                  {}",
-                if result.correctness_verified {
-                    "PASS"
-                } else {
-                    "FAIL"
-                }
-            );
-        }
-    }
-
-    println!("\n═════════════════════════════════════════════════════════════");
-    println!("Gate 14C measurement complete.");
-    println!("═════════════════════════════════════════════════════════════\n");
-}