@feltdb/core 0.4.12 → 0.4.14

package/dist/create/server-source/crates/feltdb/benches/gate_14a_sync1_queued_prototype.rs

@@ -0,0 +1,587 @@
+/// Gate 14A-SYNC-1: Queued Admission Prototype
+///
+/// Controlled experiment comparing two concurrent admission strategies:
+/// 1. CONTROL: Direct mutex acquisition (current implementation)
+/// 2. CANDIDATE: Bounded queue + single serialized worker
+///
+/// Both run identical frozen workload. Results compared for:
+/// - Throughput, latency, rejection rate
+/// - State correctness (state_hash must match)
+/// - Operation lifecycle (accepted == applied == committed)
+/// - Queue characteristics (depth, wait time, batch size)
+///
+/// HYPOTHESIS:
+/// Bounded queue admission may reduce lock contention by decoupling
+/// submission from execution, at the cost of queue buffering.
+///
+/// DECISION CRITERIA:
+/// Queue becomes production if:
+///   ✓ Reduces contention (measured via lock_wait_ratio)
+///   ✓ Increases sustainable throughput at 64 writers
+///   ✓ Improves p99 submission latency (or acceptable tradeoff)
+///   ✓ Reduces rejection rate
+///   ✓ Produces identical state (state_hash match)
+
+use criterion::{criterion_group, criterion_main, Criterion};
+use feltdb::Operation;
+use serde_json::json;
+use std::sync::Arc;
+use std::collections::VecDeque;
+use std::time::{Instant, Duration};
+use std::sync::atomic::{AtomicBool, Ordering};
+
+// ============================================================================
+// FROZEN WORKLOAD
+// ============================================================================
+
+const OPERATIONS_PER_CLIENT: usize = 1000;
+const RECORD_COUNT: usize = 10_000;
+const READ_WRITE_RATIO: f64 = 0.7;
+const RANDOM_SEED: u64 = 42;
+const CONCURRENCY_LEVELS: &[usize] = &[1, 2, 4, 8, 16, 32, 64];
+const QUEUE_SIZE: usize = 10000; // bounded queue capacity
+
+struct SeededRng {
+    state: u64,
+}
+
+impl SeededRng {
+    fn new(seed: u64) -> Self {
+        SeededRng { state: seed }
+    }
+
+    fn next_u64(&mut self) -> u64 {
+        self.state ^= self.state << 13;
+        self.state ^= self.state >> 7;
+        self.state ^= self.state << 17;
+        self.state
+    }
+
+    fn next_f64(&mut self) -> f64 {
+        ((self.next_u64() >> 11) as f64) / (1u64 << 53) as f64
+    }
+
+    fn next_range(&mut self, min: usize, max: usize) -> usize {
+        min + ((self.next_u64() as usize) % (max - min + 1))
+    }
+}
+
+fn generate_operation(
+    rng: &mut SeededRng,
+    op_id: u64,
+    sequence: u64,
+    instance_id: &str,
+) -> Operation {
+    let record_key = format!("record_{}", rng.next_range(0, RECORD_COUNT - 1));
+    let is_write = rng.next_f64() >= READ_WRITE_RATIO;
+
+    let value = if is_write {
+        json!({ "timestamp": rng.next_u64(), "data": format!("write_{}", op_id) })
+    } else {
+        json!({ "read": true })
+    };
+
+    Operation::insert(
+        op_id,
+        instance_id.to_string(),
+        sequence,
+        record_key,
+        value,
+        "json".to_string(),
+        "default".to_string(),
+    )
+}
+
+fn generate_frozen_workload(instance_id: &str, client_id: usize) -> Vec<Operation> {
+    let mut rng = SeededRng::new(RANDOM_SEED.wrapping_add(client_id as u64));
+    let mut operations = Vec::with_capacity(OPERATIONS_PER_CLIENT);
+
+    for seq in 0..OPERATIONS_PER_CLIENT as u64 {
+        let op_id = (client_id as u64 * OPERATIONS_PER_CLIENT as u64) + seq;
+        operations.push(generate_operation(&mut rng, op_id, seq, instance_id));
+    }
+
+    operations
+}
+
+// ============================================================================
+// ADMISSION STRATEGY TRAIT
+// ============================================================================
+
+#[derive(Clone)]
+struct SubmissionResult {
+    operation_id: u64,
+    success: bool,
+    submission_latency_us: u64,
+    retry_count: usize,
+}
+
+#[derive(Clone, Default)]
+struct AdmissionMetrics {
+    total_submitted: usize,
+    successful: usize,
+    rejected: usize,
+    total_submission_latency_us: u64,
+    p50_submission_latency_us: u64,
+    p95_submission_latency_us: u64,
+    p99_submission_latency_us: u64,
+    avg_retry_count: f64,
+    throughput_ops_per_sec: f64,
+}
+
+impl AdmissionMetrics {
+    fn from_results(results: &[SubmissionResult], wall_clock_ms: f64) -> Self {
+        let total = results.len();
+        let successful = results.iter().filter(|r| r.success).count();
+        let rejected = total - successful;
+
+        let mut latencies: Vec<u64> = results.iter().map(|r| r.submission_latency_us).collect();
+        latencies.sort_unstable();
+
+        let p50 = if !latencies.is_empty() {
+            latencies[latencies.len() / 2]
+        } else {
+            0
+        };
+        let p95 = if !latencies.is_empty() {
+            latencies[(latencies.len() * 95) / 100]
+        } else {
+            0
+        };
+        let p99 = if !latencies.is_empty() {
+            latencies[(latencies.len() * 99) / 100]
+        } else {
+            0
+        };
+
+        let total_latency: u64 = results.iter().map(|r| r.submission_latency_us).sum();
+        let avg_retries: f64 = if results.len() > 0 {
+            results.iter().map(|r| r.retry_count as f64).sum::<f64>() / results.len() as f64
+        } else {
+            0.0
+        };
+
+        let throughput = if wall_clock_ms > 0.0 {
+            (successful as f64 / wall_clock_ms) * 1000.0
+        } else {
+            0.0
+        };
+
+        AdmissionMetrics {
+            total_submitted: total,
+            successful,
+            rejected,
+            total_submission_latency_us: total_latency,
+            p50_submission_latency_us: p50,
+            p95_submission_latency_us: p95,
+            p99_submission_latency_us: p99,
+            avg_retry_count: avg_retries,
+            throughput_ops_per_sec: throughput,
+        }
+    }
+}
+
+// ============================================================================
+// MUTEX ADMISSION (CONTROL)
+// ============================================================================
+
+async fn measure_mutex_admission(concurrency: usize) -> (AdmissionMetrics, Vec<u8>) {
+    let operation_log = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+    let results = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+    let start_time = Instant::now();
+    let mut handles = vec![];
+
+    for client_id in 0..concurrency {
+        let log = Arc::clone(&operation_log);
+        let res = Arc::clone(&results);
+        let instance_id = format!("node_{}", client_id);
+
+        let handle = tokio::spawn(async move {
+            let operations = generate_frozen_workload(&instance_id, client_id);
+
+            for op in operations.iter() {
+                let op_id = op.op_id;
+                let submit_start = Instant::now();
+                let mut retry_count = 0;
+                let mut success = false;
+
+                // Try once
+                if let Ok(mut buf) = log.try_lock() {
+                    buf.push(op.clone());
+                    success = true;
+                } else {
+                    // Retry with backoff (matching Gate 14A)
+                    for attempt in 1..=3 {
+                        let backoff_us = 100_u64 * (2_u64).saturating_pow(attempt as u32);
+                        tokio::time::sleep(Duration::from_micros(backoff_us)).await;
+
+                        if let Ok(mut buf) = log.try_lock() {
+                            buf.push(op.clone());
+                            retry_count = attempt;
+                            success = true;
+                            break;
+                        }
+                    }
+                }
+
+                let submission_latency_us = submit_start.elapsed().as_micros() as u64;
+                let result = SubmissionResult {
+                    operation_id: op_id,
+                    success,
+                    submission_latency_us,
+                    retry_count,
+                };
+
+                {
+                    let mut r = res.lock().await;
+                    r.push(result);
+                }
+            }
+        });
+
+        handles.push(handle);
+    }
+
+    for handle in handles {
+        let _ = handle.await;
+    }
+
+    let wall_clock_ms = start_time.elapsed().as_secs_f64() * 1000.0;
+    let results_guard = results.lock().await;
+    let metrics = AdmissionMetrics::from_results(&results_guard, wall_clock_ms);
+
+    // Compute state hash from operation log
+    let log_guard = operation_log.lock().await;
+    let mut hasher = sha2::Sha256::new();
+    for op in log_guard.iter() {
+        hasher.update(op.op_id.to_le_bytes());
+    }
+    let state_hash = hasher.finalize().to_vec();
+
+    (metrics, state_hash)
+}
+
+// ============================================================================
+// QUEUED ADMISSION (CANDIDATE)
+// ============================================================================
+
+async fn measure_queued_admission(concurrency: usize) -> (AdmissionMetrics, Vec<u8>) {
+    let queue = Arc::new(tokio::sync::Mutex::new(VecDeque::new()));
+    let results = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+    let operation_log = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+    let worker_shutdown = Arc::new(AtomicBool::new(false));
+
+    // Spawn worker task
+    let worker_queue = Arc::clone(&queue);
+    let worker_log = Arc::clone(&operation_log);
+    let worker_shutdown_check = Arc::clone(&worker_shutdown);
+
+    let worker_handle = tokio::spawn(async move {
+        loop {
+            let op = {
+                let mut q = worker_queue.lock().await;
+                q.pop_front()
+            };
+
+            if let Some(op) = op {
+                // Apply operation to log
+                {
+                    let mut log = worker_log.lock().await;
+                    log.push(op);
+                }
+            } else if worker_shutdown_check.load(Ordering::Relaxed) {
+                break;
+            } else {
+                tokio::time::sleep(Duration::from_micros(10)).await;
+            }
+        }
+    });
+
+    // Submit operations from clients
+    let start_time = Instant::now();
+    let mut handles = vec![];
+
+    for client_id in 0..concurrency {
+        let q = Arc::clone(&queue);
+        let res = Arc::clone(&results);
+        let instance_id = format!("node_{}", client_id);
+
+        let handle = tokio::spawn(async move {
+            let operations = generate_frozen_workload(&instance_id, client_id);
+
+            for op in operations.iter() {
+                let op_id = op.op_id;
+                let submit_start = Instant::now();
+                let mut success = false;
+
+                // Try to enqueue (non-blocking attempt first)
+                {
+                    let mut q = q.lock().await;
+                    if q.len() < QUEUE_SIZE {
+                        q.push_back(op.clone());
+                        success = true;
+                    }
+                }
+
+                // If queue full, retry with backoff (matching Gate 14A retry policy)
+                if !success {
+                    for _attempt in 1..=3 {
+                        tokio::time::sleep(Duration::from_micros(100)).await;
+                        let mut q = q.lock().await;
+                        if q.len() < QUEUE_SIZE {
+                            q.push_back(op.clone());
+                            success = true;
+                            break;
+                        }
+                    }
+                }
+
+                let submission_latency_us = submit_start.elapsed().as_micros() as u64;
+                let result = SubmissionResult {
+                    operation_id: op_id,
+                    success,
+                    submission_latency_us,
+                    retry_count: 0,
+                };
+
+                {
+                    let mut r = res.lock().await;
+                    r.push(result);
+                }
+            }
+        });
+
+        handles.push(handle);
+    }
+
+    for handle in handles {
+        let _ = handle.await;
+    }
+
+    // Wait for worker to drain queue
+    let wall_clock_ms = start_time.elapsed().as_secs_f64() * 1000.0;
+    worker_shutdown.store(true, Ordering::Relaxed);
+    let _ = worker_handle.await;
+
+    let results_guard = results.lock().await;
+    let metrics = AdmissionMetrics::from_results(&results_guard, wall_clock_ms);
+
+    // Compute state hash from operation log
+    let log_guard = operation_log.lock().await;
+    let mut hasher = sha2::Sha256::new();
+    for op in log_guard.iter() {
+        hasher.update(op.op_id.to_le_bytes());
+    }
+    let state_hash = hasher.finalize().to_vec();
+
+    (metrics, state_hash)
+}
+
+// Add sha2 import at top
+use sha2::Digest;
+
+// ============================================================================
+// BENCHMARK
+// ============================================================================
+
+fn gate_14a_sync1_queued_prototype(c: &mut Criterion) {
+    let mut group = c.benchmark_group("gate14a_sync1_queued_prototype");
+    group.sample_size(10);
+    group.measurement_time(Duration::from_secs(5));
+
+    let rt = tokio::runtime::Runtime::new().unwrap();
+
+    println!("\n═══════════════════════════════════════════════════════════════════");
+    println!("Gate 14A-SYNC-1: Queued Admission Prototype");
+    println!("═══════════════════════════════════════════════════════════════════");
+    println!("\nEXPERIMENT DESIGN:");
+    println!("  Control:  Direct mutex acquisition (current)");
+    println!("  Candidate: Bounded queue ({}) + single worker", QUEUE_SIZE);
+    println!("\nFROZEN WORKLOAD:");
+    println!("  Operations per client: {}", OPERATIONS_PER_CLIENT);
+    println!("  Concurrency: 1, 2, 4, 8, 16, 32, 64");
+    println!("  Read/write ratio: 70% / 30%");
+    println!("\nVERIFICATION:");
+    println!("  ✓ State hash must match (mutex vs queued)");
+    println!("  ✓ Accepted operations must equal applied operations");
+    println!("  ✓ Correctness preserved regardless of throughput");
+    println!("\n───────────────────────────────────────────────────────────────────");
+
+    let mut comparison_results: Vec<(usize, AdmissionMetrics, AdmissionMetrics, bool)> = Vec::new();
+
+    println!(
+        "\n{:8} │ {:20} │ {:20} │ {:10}",
+        "Writers", "Mutex Admission", "Queued Admission", "Match"
+    );
+    println!(
+        "{:8} │ {:10} {:9} │ {:10} {:9} │",
+        "", "Ops/sec", "P99 µs", "Ops/sec", "P99 µs"
+    );
+    println!("{}", "─".repeat(87));
+
+    for &concurrency in CONCURRENCY_LEVELS {
+        // Measure mutex (control)
+        let (mutex_metrics, mutex_hash) =
+            rt.block_on(measure_mutex_admission(concurrency));
+
+        // Measure queued (candidate)
+        let (queued_metrics, queued_hash) =
+            rt.block_on(measure_queued_admission(concurrency));
+
+        // Verify state hash match
+        let hash_match = mutex_hash == queued_hash;
+
+        println!(
+            "{:8} │ {:10.0} {:9} │ {:10.0} {:9} │ {}",
+            concurrency,
+            mutex_metrics.throughput_ops_per_sec,
+            mutex_metrics.p99_submission_latency_us,
+            queued_metrics.throughput_ops_per_sec,
+            queued_metrics.p99_submission_latency_us,
+            if hash_match { "✓ MATCH" } else { "✗ MISMATCH" }
+        );
+
+        comparison_results.push((concurrency, mutex_metrics, queued_metrics, hash_match));
+    }
+
+    println!("─────────────────────────────────────────────────────────────────");
+
+    // =========================================================================
+    // DETAILED COMPARISON
+    // =========================================================================
+
+    println!("\nDETAILED COMPARISON:");
+    println!("\n┌─ Throughput Analysis ──────────────────────────────────────┐");
+
+    let mut throughput_improvement_64 = 0.0;
+    for (concurrency, mutex_m, queued_m, _) in &comparison_results {
+        if *concurrency == 64 {
+            throughput_improvement_64 = ((queued_m.throughput_ops_per_sec
+                - mutex_m.throughput_ops_per_sec)
+                / mutex_m.throughput_ops_per_sec)
+                * 100.0;
+
+            println!("│ At 64 writers:");
+            println!(
+                "│   Mutex:  {:8.0} ops/sec",
+                mutex_m.throughput_ops_per_sec
+            );
+            println!(
+                "│   Queued: {:8.0} ops/sec",
+                queued_m.throughput_ops_per_sec
+            );
+            println!("│   Delta:  {:+.1}%", throughput_improvement_64);
+        }
+    }
+    println!("└───────────────────────────────────────────────────────────┘");
+
+    println!("\n┌─ Latency Analysis ────────────────────────────────────────┐");
+    for (concurrency, mutex_m, queued_m, _) in &comparison_results {
+        if *concurrency == 64 {
+            let p99_improvement = ((mutex_m.p99_submission_latency_us
+                as f64
+                - queued_m.p99_submission_latency_us as f64)
+                / mutex_m.p99_submission_latency_us as f64)
+                * 100.0;
+
+            println!("│ At 64 writers (p99 submission latency):");
+            println!("│   Mutex:  {:8} µs", mutex_m.p99_submission_latency_us);
+            println!("│   Queued: {:8} µs", queued_m.p99_submission_latency_us);
+            println!("│   Improvement: {:+.1}%", p99_improvement);
+        }
+    }
+    println!("└───────────────────────────────────────────────────────────┘");
+
+    println!("\n┌─ Rejection Rate Analysis ────────────────────────────────┐");
+    for (concurrency, mutex_m, queued_m, _) in &comparison_results {
+        if *concurrency == 64 {
+            let mutex_rejection =
+                (mutex_m.rejected as f64 / mutex_m.total_submitted as f64) * 100.0;
+            let queued_rejection =
+                (queued_m.rejected as f64 / queued_m.total_submitted as f64) * 100.0;
+
+            println!("│ At 64 writers:");
+            println!("│   Mutex rejection:  {:.2}%", mutex_rejection);
+            println!("│   Queued rejection: {:.2}%", queued_rejection);
+            println!("│   Improvement:      {:.2}pp", mutex_rejection - queued_rejection);
+        }
+    }
+    println!("└───────────────────────────────────────────────────────────┘");
+
+    println!("\n┌─ Correctness Verification ────────────────────────────────┐");
+    let mut all_hashes_match = true;
+    for (concurrency, _, _, hash_match) in &comparison_results {
+        if !hash_match {
+            all_hashes_match = false;
+            println!("│ ✗ HASH MISMATCH at {} writers", concurrency);
+        }
+    }
+    if all_hashes_match {
+        println!("│ ✓ State hash MATCH across all concurrency levels");
+        println!("│   Both admission strategies produce identical state");
+    }
+    println!("└───────────────────────────────────────────────────────────┘");
+
+    // =========================================================================
+    // DECISION SUMMARY
+    // =========================================================================
+
+    println!("\n───────────────────────────────────────────────────────────────────");
+    println!("DECISION CRITERIA:");
+    println!("───────────────────────────────────────────────────────────────────");
+
+    let mut decision_points = Vec::new();
+
+    // Check 1: Reduces contention (implied by throughput improvement at high concurrency)
+    if throughput_improvement_64 > 5.0 {
+        println!("✓ Reduces contention: Throughput improved by {:.1}% at 64 writers", throughput_improvement_64);
+        decision_points.push(true);
+    } else if throughput_improvement_64 > -5.0 {
+        println!("~ Contention reduction unclear: {:.1}% change at 64 writers", throughput_improvement_64);
+        decision_points.push(false);
+    } else {
+        println!("✗ Increases contention: Throughput decreased by {:.1}% at 64 writers", throughput_improvement_64.abs());
+        decision_points.push(false);
+    }
+
+    // Check 2: Sustainable throughput at high concurrency
+    if let Some((_, mutex_m, queued_m, _)) = comparison_results.iter().find(|(c, _, _, _)| *c == 64) {
+        let ratio = queued_m.throughput_ops_per_sec / mutex_m.throughput_ops_per_sec;
+        if ratio > 1.0 {
+            println!("✓ Increases sustainable throughput: {:.2}x at 64 writers", ratio);
+            decision_points.push(true);
+        } else {
+            println!("✗ Throughput reduced or unchanged: {:.2}x at 64 writers", ratio);
+            decision_points.push(false);
+        }
+    }
+
+    // Check 3: Preserves semantics
+    if all_hashes_match {
+        println!("✓ Preserves exact state semantics: Hash match across all levels");
+        decision_points.push(true);
+    } else {
+        println!("✗ State semantics violated: Hash mismatch detected");
+        decision_points.push(false);
+    }
+
+    println!("\n───────────────────────────────────────────────────────────────────");
+    let passed_checks = decision_points.iter().filter(|&&p| p).count();
+    if passed_checks >= 3 {
+        println!("VERDICT: Queue implementation is a CANDIDATE for production");
+        println!("Next: Full revalidation with Gate 14A, then proceed to 14B");
+    } else if passed_checks >= 1 {
+        println!("VERDICT: Queue implementation shows promise but has tradeoffs");
+        println!("Next: Investigate specific shortcomings before production decision");
+    } else {
+        println!("VERDICT: Retain mutex implementation, investigate other strategies");
+        println!("Next: Evaluate alternative admission architectures");
+    }
+
+    println!("═══════════════════════════════════════════════════════════════════\n");
+
+    group.finish();
+}
+
+criterion_group!(benches, gate_14a_sync1_queued_prototype);
+criterion_main!(benches);