@feltdb/core 0.4.12 → 0.4.14

package/dist/create/server-source/crates/feltdb/benches/gate_14a_rc_root_cause.rs

@@ -0,0 +1,273 @@
+/// Gate 14A-RC: Concurrent Writer Correctness Root-Cause Investigation
+///
+/// Objective: Explain the 8-writer correctness failure from Gate 14A.
+/// Focus: Narrow boundary, identify whether failure is deterministic or probabilistic.
+///
+/// FROZEN WORKLOAD (identical to Gate 14A):
+/// - 1000 operations per client
+/// - 10,000 record dataset
+/// - 70% reads, 30% writes (seed: 42)
+/// - Consensus: Causal
+///
+/// INVESTIGATION STRATEGY:
+/// 1. Boundary characterization: Test 4-10 writers with multiple trials
+/// 2. Identify whether boundary is sharp (7 PASS, 8 FAIL) or probabilistic
+/// 3. Reproduce exact failing sequence for later instrumentation
+/// 4. Establish baseline for state-hash divergence analysis
+
+use criterion::{criterion_group, criterion_main, Criterion};
+use feltdb::Operation;
+use serde_json::json;
+use std::sync::Arc;
+use std::time::Duration;
+use std::collections::HashMap;
+
+// ============================================================================
+// FROZEN WORKLOAD (identical to Gate 14A)
+// ============================================================================
+
+const OPERATIONS_PER_CLIENT: usize = 1000;
+const RECORD_COUNT: usize = 10_000;
+const READ_WRITE_RATIO: f64 = 0.7;
+const RANDOM_SEED: u64 = 42;
+const BOUNDARY_TEST_LEVELS: &[usize] = &[4, 5, 6, 7, 8, 9, 10];
+const TRIALS_PER_LEVEL: usize = 5;
+
+struct SeededRng {
+    state: u64,
+}
+
+impl SeededRng {
+    fn new(seed: u64) -> Self {
+        SeededRng { state: seed }
+    }
+
+    fn next_u64(&mut self) -> u64 {
+        self.state ^= self.state << 13;
+        self.state ^= self.state >> 7;
+        self.state ^= self.state << 17;
+        self.state
+    }
+
+    fn next_f64(&mut self) -> f64 {
+        ((self.next_u64() >> 11) as f64) / (1u64 << 53) as f64
+    }
+
+    fn next_range(&mut self, min: usize, max: usize) -> usize {
+        min + ((self.next_u64() as usize) % (max - min + 1))
+    }
+}
+
+// ============================================================================
+// WORKLOAD GENERATION
+// ============================================================================
+
+fn generate_operation(
+    rng: &mut SeededRng,
+    op_id: u64,
+    sequence: u64,
+    instance_id: &str,
+) -> Operation {
+    let record_key = format!("record_{}", rng.next_range(0, RECORD_COUNT - 1));
+    let is_write = rng.next_f64() >= READ_WRITE_RATIO;
+
+    let value = if is_write {
+        json!({ "timestamp": rng.next_u64(), "data": format!("write_{}", op_id) })
+    } else {
+        json!({ "read": true })
+    };
+
+    Operation::insert(
+        op_id,
+        instance_id.to_string(),
+        sequence,
+        record_key,
+        value,
+        "json".to_string(),
+        "default".to_string(),
+    )
+}
+
+fn generate_frozen_workload(instance_id: &str, client_id: usize) -> Vec<Operation> {
+    let mut rng = SeededRng::new(RANDOM_SEED.wrapping_add(client_id as u64));
+    let mut operations = Vec::with_capacity(OPERATIONS_PER_CLIENT);
+
+    for seq in 0..OPERATIONS_PER_CLIENT as u64 {
+        let op_id = (client_id as u64 * OPERATIONS_PER_CLIENT as u64) + seq;
+        operations.push(generate_operation(&mut rng, op_id, seq, instance_id));
+    }
+
+    operations
+}
+
+// ============================================================================
+// SIMPLE CONCURRENT MEASUREMENT (focused on ingestion accuracy)
+// ============================================================================
+
+async fn measure_concurrency_level(concurrency: usize) -> (usize, usize, bool) {
+    let operation_buffer = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+    let mut handles = vec![];
+
+    for client_id in 0..concurrency {
+        let buffer = Arc::clone(&operation_buffer);
+        let instance_id = format!("node_{}", client_id);
+
+        let handle = tokio::spawn(async move {
+            let operations = generate_frozen_workload(&instance_id, client_id);
+
+            for op in operations.iter() {
+                // Try to ingest operation (matching Gate 14A: try + 3 retries)
+                if let Ok(mut buf) = buffer.try_lock() {
+                    buf.push(op.clone());
+                } else {
+                    // Brief retry with exponential backoff (same as Gate 14A)
+                    let mut succeeded = false;
+                    for attempt in 1..=3 {
+                        let backoff_us = 100_u64 * (2_u64).saturating_pow(attempt as u32);
+                        tokio::time::sleep(Duration::from_micros(backoff_us)).await;
+
+                        if let Ok(mut buf) = buffer.try_lock() {
+                            buf.push(op.clone());
+                            succeeded = true;
+                            break;
+                        }
+                    }
+                    // If all retries exhausted, operation is lost (not pushed)
+                }
+            }
+        });
+
+        handles.push(handle);
+    }
+
+    // Wait for all to complete
+    for handle in handles {
+        let _ = handle.await;
+    }
+
+    // Check final state
+    let buffer = operation_buffer.lock().await;
+    let expected_total = concurrency * OPERATIONS_PER_CLIENT;
+    let actual_total = buffer.len();
+    let correctness_pass = actual_total == expected_total;
+
+    (expected_total, actual_total, correctness_pass)
+}
+
+// ============================================================================
+// BOUNDARY INVESTIGATION
+// ============================================================================
+
+fn gate_14a_rc_root_cause(c: &mut Criterion) {
+    let mut group = c.benchmark_group("gate14a_rc_root_cause");
+    group.sample_size(10);
+    group.measurement_time(Duration::from_secs(5));
+
+    let rt = tokio::runtime::Runtime::new().unwrap();
+
+    println!("\n═══════════════════════════════════════════════════════════");
+    println!("Gate 14A-RC: Concurrent Writer Correctness Root-Cause");
+    println!("═══════════════════════════════════════════════════════════");
+    println!("\nFROZEN WORKLOAD (identical to Gate 14A):");
+    println!("  Operations per client: {}", OPERATIONS_PER_CLIENT);
+    println!("  Record count: {}", RECORD_COUNT);
+    println!("  Read/write ratio: 70% / 30%");
+    println!("  Random seed: {}", RANDOM_SEED);
+    println!("\nBOUNDARY CHARACTERIZATION:");
+    println!("  Testing concurrency levels: {:?}", BOUNDARY_TEST_LEVELS);
+    println!("  Trials per level: {}", TRIALS_PER_LEVEL);
+    println!("  Goal: Identify whether boundary is sharp or probabilistic");
+    println!("\n───────────────────────────────────────────────────────────");
+    println!("Writers | Trials | Pass | Fail | Expected | Actual | MissingOps");
+    println!("───────────────────────────────────────────────────────────");
+
+    let mut boundary_results = Vec::new();
+
+    for &concurrency in BOUNDARY_TEST_LEVELS {
+        let mut pass_count = 0;
+        let mut fail_count = 0;
+        let mut total_missing = 0;
+
+        for _trial in 0..TRIALS_PER_LEVEL {
+            let (expected, actual, correctness_pass) =
+                rt.block_on(measure_concurrency_level(concurrency));
+
+            if correctness_pass {
+                pass_count += 1;
+            } else {
+                fail_count += 1;
+                total_missing += expected - actual;
+            }
+        }
+
+        let avg_missing = if fail_count > 0 {
+            total_missing / fail_count
+        } else {
+            0
+        };
+
+        println!(
+            "{:7} | {:6} | {:4} | {:4} | {:8} | {:6} | {:9}",
+            concurrency, TRIALS_PER_LEVEL, pass_count, fail_count,
+            concurrency * OPERATIONS_PER_CLIENT,
+            concurrency * OPERATIONS_PER_CLIENT - avg_missing,
+            avg_missing
+        );
+
+        boundary_results.push((concurrency, pass_count, fail_count, avg_missing));
+    }
+
+    println!("───────────────────────────────────────────────────────────");
+
+    // Analyze boundary
+    let mut sharp_boundary = None;
+    for i in 0..boundary_results.len() - 1 {
+        if boundary_results[i].1 == TRIALS_PER_LEVEL && boundary_results[i + 1].2 > 0 {
+            sharp_boundary = Some((boundary_results[i].0, boundary_results[i + 1].0));
+        }
+    }
+
+    println!("\nBOUNDARY ANALYSIS:");
+    if let Some((pass_level, fail_level)) = sharp_boundary {
+        println!(
+            "  ✓ SHARP BOUNDARY: {} writers PASS → {} writers FAIL",
+            pass_level, fail_level
+        );
+        println!("\n  This is a deterministic transition.");
+        println!("  Next: Investigate why exactly {} writers exceeds capacity.", fail_level);
+    } else {
+        println!("  ? PROBABILISTIC BOUNDARY (failures at multiple levels)");
+        for (concurrency, pass, fail, missing) in &boundary_results {
+            if *fail > 0 {
+                let rate = (*fail as f64 / TRIALS_PER_LEVEL as f64) * 100.0;
+                println!(
+                    "    {} writers: {}% failure (avg {} operations lost)",
+                    concurrency, rate as i32, missing
+                );
+            }
+        }
+    }
+
+    println!("\nROOT-CAUSE INVESTIGATION ROADMAP:");
+    println!("  1. Verify boundary reproducibility across multiple machine runs");
+    println!("  2. Instrument operation lifecycle:");
+    println!("     - track SUBMITTED → ACCEPTED → APPLIED → COMMITTED → ACKED");
+    println!("  3. Identify where operations disappear:");
+    println!("     - before acceptance (queue overflow)?");
+    println!("     - during application (concurrency bug)?");
+    println!("     - during commit (state machine issue)?");
+    println!("  4. Test deterministic replay:");
+    println!("     - run failing sequence sequentially");
+    println!("     - compare state hash");
+    println!("  5. Instrument FeltDB internals to prove subsystem");
+    println!("\nCURRENT HYPOTHESIS:");
+    if let Some((_, fail_level)) = sharp_boundary {
+        println!("  Operation buffer or lock contention reaches limit at {} writers", fail_level);
+    }
+    println!("═══════════════════════════════════════════════════════════\n");
+
+    group.finish();
+}
+
+criterion_group!(benches, gate_14a_rc_root_cause);
+criterion_main!(benches);