@feltdb/core 0.8.4 → 0.8.6

package/dist/create/server-source/crates/feltdb/tests/compaction_stall_contract.rs

@@ -0,0 +1,272 @@
+//! Compaction must not stop the world on a timer.
+//!
+//! An incident on the managed deployment showed `/health` returning `200` while
+//! every application-scoped operation timed out or returned `502`, across
+//! several minutes, on four unrelated collections at once.
+//!
+//! Four collections failing together is the signature of a **global stall**,
+//! not of four collection-specific faults. The mechanism was here:
+//!
+//! > `compact_operation_log` takes the one lock that serializes every read and
+//! > every write, then serializes **every row in the database** to a temp file
+//! > and `fsync`s it — and a background task called it **every two seconds**.
+//!
+//! Pruning acknowledged operations from memory is cheap. Rewriting the log is
+//! not. This file holds them apart: pruning still happens whenever it can, and
+//! the rewrite waits until enough has accumulated to be worth the stall.
+//!
+//! See `docs/architecture/compaction-stall.md`.
+
+use feltdb::state_model::StateStore;
+use feltdb::{CompactionOutcome, CompactionPolicy, FeltDb};
+use serde_json::json;
+use std::sync::Arc;
+use std::time::Instant;
+use tempfile::TempDir;
+
+/// Acknowledge everything a database has produced, so a compaction has work.
+fn acknowledge_all(db: &Arc<FeltDb>) {
+    let versions = db.operation_versions().unwrap();
+    db.acknowledge_peer_versions("peer-1".to_string(), versions)
+        .unwrap();
+}
+
+fn write_n(db: &Arc<FeltDb>, n: usize) {
+    for index in 0..n {
+        db.insert(&format!("tasks:{index}"), json!({ "n": index }))
+            .unwrap();
+    }
+}
+
+// ---------------------------------------------------------------------------
+// The fix
+// ---------------------------------------------------------------------------
+
+/// **A timer-driven compaction prunes without rewriting the log.**
+///
+/// This is the change that stops the stall. The in-memory operation log still
+/// shrinks on every tick; the durable rewrite — the expensive half — waits.
+#[test]
+fn a_policy_driven_compaction_defers_the_rewrite() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("defer.log");
+    let db = Arc::new(FeltDb::open(&path).unwrap());
+    db.set_compaction_policy(CompactionPolicy {
+        rewrite_after_pruned: 1000,
+    });
+
+    write_n(&db, 20);
+    acknowledge_all(&db);
+    let before = std::fs::metadata(&path).unwrap().len();
+
+    let outcome = db
+        .maybe_compact_operation_log(&["peer-1".to_string()])
+        .unwrap();
+
+    match outcome {
+        CompactionOutcome::Deferred {
+            pruned,
+            pending_since_rewrite,
+        } => {
+            assert!(pruned > 0, "the cheap half still happened");
+            assert_eq!(pending_since_rewrite, pruned);
+        }
+        other => panic!("expected the rewrite to be deferred, got {other:?}"),
+    }
+    assert_eq!(
+        std::fs::metadata(&path).unwrap().len(),
+        before,
+        "the durable log was not rewritten"
+    );
+
+    // And the database is entirely usable.
+    let value: serde_json::Value = db.get("tasks:0").unwrap().unwrap();
+    assert_eq!(value["n"], json!(0));
+}
+
+/// **The rewrite happens once the accumulated gain earns it**, and the log
+/// actually shrinks when it does.
+#[test]
+fn the_rewrite_happens_once_the_threshold_is_reached() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("threshold.log");
+    let db = Arc::new(FeltDb::open(&path).unwrap());
+    db.set_compaction_policy(CompactionPolicy {
+        rewrite_after_pruned: 25,
+    });
+
+    let mut rewrites = 0;
+    let mut defers = 0;
+    for round in 0..6 {
+        for index in 0..10 {
+            db.update(
+                &format!("tasks:{}", round * 10 + index),
+                json!({ "n": index }),
+            )
+            .unwrap();
+        }
+        acknowledge_all(&db);
+        match db
+            .maybe_compact_operation_log(&["peer-1".to_string()])
+            .unwrap()
+        {
+            CompactionOutcome::Rewritten { .. } => rewrites += 1,
+            CompactionOutcome::Deferred { .. } => defers += 1,
+            CompactionOutcome::Idle => {}
+        }
+    }
+
+    assert!(defers > 0, "most ticks defer");
+    assert!(rewrites > 0, "and the rewrite still happens");
+    assert!(
+        defers > rewrites,
+        "the point is that deferring is the common case: {defers} deferred, {rewrites} rewritten"
+    );
+}
+
+/// **An explicit compaction still rewrites immediately.**
+///
+/// The policy governs the timer, not a caller who is asking for a rewrite now.
+/// Every existing caller of `compact_operation_log` keeps its behaviour.
+#[test]
+fn an_explicit_compaction_still_rewrites_now() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("explicit.log");
+    let db = Arc::new(FeltDb::open(&path).unwrap());
+    db.set_compaction_policy(CompactionPolicy {
+        rewrite_after_pruned: usize::MAX,
+    });
+
+    write_n(&db, 20);
+    for index in 0..20 {
+        db.update(&format!("tasks:{index}"), json!({"n": "updated"}))
+            .unwrap();
+    }
+    acknowledge_all(&db);
+    let before = std::fs::metadata(&path).unwrap().len();
+
+    let pruned = db.compact_operation_log(&["peer-1".to_string()]).unwrap();
+    assert!(pruned > 0);
+    assert!(
+        std::fs::metadata(&path).unwrap().len() < before,
+        "an explicit compaction rewrote and shrank the log"
+    );
+}
+
+/// **The deferred rewrite costs nothing in correctness.**
+///
+/// Pruning without rewriting leaves the durable log longer than it needs to be,
+/// and nothing else: the database reopens to the same state and the same
+/// history.
+#[test]
+fn deferring_the_rewrite_preserves_state_and_history() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("preserve.log");
+    let db = Arc::new(FeltDb::open(&path).unwrap());
+    db.set_compaction_policy(CompactionPolicy {
+        rewrite_after_pruned: usize::MAX,
+    });
+
+    db.insert("tasks:1", json!({"n": 0})).unwrap();
+    for n in 1..10 {
+        db.update("tasks:1", json!({ "n": n })).unwrap();
+    }
+    acknowledge_all(&db);
+    assert!(matches!(
+        db.maybe_compact_operation_log(&["peer-1".to_string()])
+            .unwrap(),
+        CompactionOutcome::Deferred { .. }
+    ));
+
+    let store = StateStore::with_feltdb(db.clone()).unwrap();
+    let history_before = store.history_digest();
+    let state_before = db.state_digest().unwrap();
+    drop(store);
+    drop(db);
+
+    let reopened = Arc::new(FeltDb::open(&path).unwrap());
+    let store = StateStore::with_feltdb(reopened.clone()).unwrap();
+    assert_eq!(reopened.state_digest().unwrap(), state_before);
+    assert_eq!(store.history_digest(), history_before);
+}
+
+/// Nothing to prune is not a rewrite either — the pre-existing early return.
+#[test]
+fn an_idle_compaction_does_no_work() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("idle.log");
+    let db = Arc::new(FeltDb::open(&path).unwrap());
+    write_n(&db, 5);
+    let before = std::fs::read(&path).unwrap();
+
+    // No peer has acknowledged anything.
+    let outcome = db.maybe_compact_operation_log(&[]).unwrap();
+    assert_eq!(outcome, CompactionOutcome::Idle);
+    assert_eq!(outcome.pruned(), 0);
+    assert!(!outcome.rewrote_log());
+    assert_eq!(std::fs::read(&path).unwrap(), before);
+}
+
+// ---------------------------------------------------------------------------
+// The stall itself
+// ---------------------------------------------------------------------------
+
+/// **The rewrite is what stalls, and it grows with the database.**
+///
+/// Measured as an ordering rather than a number: a rewrite over a larger
+/// database holds the lock longer. That is why running it on a two-second timer
+/// degraded as the deployment grew, and why the default policy now waits.
+#[test]
+fn the_rewrite_cost_grows_with_the_database() {
+    let cost = |rows: usize| {
+        let directory = TempDir::new().unwrap();
+        let path = directory.path().join("cost.log");
+        let db = Arc::new(FeltDb::open(&path).unwrap());
+        write_n(&db, rows);
+        for index in 0..rows {
+            db.update(&format!("tasks:{index}"), json!({"n": "updated"}))
+                .unwrap();
+        }
+        acknowledge_all(&db);
+        let started = Instant::now();
+        db.compact_operation_log(&["peer-1".to_string()]).unwrap();
+        started.elapsed()
+    };
+
+    let small = cost(20);
+    let large = cost(400);
+    assert!(
+        large > small,
+        "a rewrite over more rows holds the lock longer: {large:?} vs {small:?}"
+    );
+}
+
+/// **The default policy does not rewrite on every tick.**
+///
+/// The regression this whole file exists to prevent. With the shipped default,
+/// a handful of acknowledged operations must not trigger a full log rewrite.
+#[test]
+fn the_default_policy_does_not_rewrite_on_a_handful_of_operations() {
+    assert_eq!(CompactionPolicy::default().rewrite_after_pruned, 1000);
+
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("default.log");
+    let db = Arc::new(FeltDb::open(&path).unwrap());
+    assert_eq!(db.compaction_policy(), CompactionPolicy::default());
+
+    write_n(&db, 30);
+    acknowledge_all(&db);
+    let before = std::fs::metadata(&path).unwrap().len();
+
+    // Ten ticks of the timer.
+    for _ in 0..10 {
+        db.maybe_compact_operation_log(&["peer-1".to_string()])
+            .unwrap();
+    }
+
+    assert_eq!(
+        std::fs::metadata(&path).unwrap().len(),
+        before,
+        "ten timer ticks over a small database rewrote the log zero times"
+    );
+}