@feltdb/core 0.15.0 → 0.16.0

package/dist/create/server-source/crates/feltdb/tests/crash_durability_contract.rs

@@ -1,467 +0,0 @@
-//! Crash and power-loss durability.
-//!
-//! The question is not whether `fsync` appears somewhere. It is:
-//!
-//! > **After a process dies at every relevant point in a mutation, what durable
-//! > states are actually possible, and are all of them valid under FeltDB's
-//! > contract?**
-//!
-//! # How every crash point is reached
-//!
-//! Killing a process at a chosen instruction is not deterministic. Truncating a
-//! durable log at every byte offset is, and it is strictly stronger: it reaches
-//! every boundary a crash could land on, including boundaries *inside* a single
-//! write, which a signal cannot target. The exhaustive harness below opens the
-//! database at all of them.
-//!
-//! A real process abort is also tested, because truncation models the durable
-//! bytes and not the operating system's behaviour when a process dies.
-//!
-//! # What is deliberately not claimed
-//!
-//! `append_event` writes and flushes to the operating system. It does **not**
-//! `fsync`; only `append_transaction` does. So process-crash durability is
-//! proven here and **stable-storage durability is not**. Those are different
-//! claims and this file does not let the first stand in for the second. See
-//! `docs/architecture/crash-durability.md`.
-
-use feltdb::state_model::StateStore;
-use feltdb::{AtomicMutation, FeltDb, FlowError};
-use serde_json::{json, Value};
-use std::collections::HashMap;
-use std::path::Path;
-use std::sync::Arc;
-use tempfile::TempDir;
-
-/// What a database looks like after recovering from a crash.
-#[derive(Debug, PartialEq)]
-struct Recovered {
-    /// The value of `tasks:1`, if the record survived.
-    value: Option<i64>,
-    /// How many revisions its history holds.
-    revisions: usize,
-}
-
-fn inspect(db: &Arc<FeltDb>) -> Recovered {
-    let store = StateStore::with_feltdb(db.clone()).unwrap();
-    let history = store.history_of("tasks:1");
-
-    // Invariants that must hold at every crash point, checked here so no
-    // caller can forget them.
-    assert!(
-        history
-            .windows(2)
-            .all(|pair| pair[1].parent_id.as_ref() == Some(&pair[0].id)),
-        "ancestry is a chain at every crash point"
-    );
-    assert!(
-        history
-            .iter()
-            .enumerate()
-            .all(|(index, revision)| revision.sequence == index as u64),
-        "sequences are contiguous from zero at every crash point"
-    );
-    if let Some(first) = history.first() {
-        assert_eq!(first.parent_id, None, "the first revision begins history");
-    }
-
-    Recovered {
-        value: db
-            .get::<Value>("tasks:1")
-            .unwrap()
-            .map(|value| value["n"].as_i64().unwrap()),
-        revisions: history.len(),
-    }
-}
-
-/// Build a log by a known mutation sequence and return its bytes.
-fn log_of_three_writes(path: &Path) -> Vec<u8> {
-    let db = Arc::new(FeltDb::open(path).unwrap());
-    db.insert("tasks:1", json!({"n": 0})).unwrap();
-    db.update("tasks:1", json!({"n": 1})).unwrap();
-    db.update("tasks:1", json!({"n": 2})).unwrap();
-    drop(db);
-    std::fs::read(path).unwrap()
-}
-
-// ---------------------------------------------------------------------------
-// D1 — every crash point
-// ---------------------------------------------------------------------------
-
-/// **Every crash point leaves a clean prefix of the mutation sequence.**
-///
-/// Three writes, and the durable log cut at all 4,314 byte offsets. Every one
-/// opens, and every recovered database is one of exactly four states:
-///
-/// ```text
-///   nothing        no record, no history
-///   n = 0          one revision
-///   n = 1          two revisions
-///   n = 2          three revisions
-/// ```
-///
-/// There is no state in which a record exists without the revision recording
-/// it, none in which a revision exists for a record that was never written, and
-/// none with broken ancestry or a gap in the sequence. A crash truncates
-/// history; it cannot corrupt or invent it.
-#[test]
-fn every_crash_point_leaves_a_clean_prefix() {
-    let directory = TempDir::new().unwrap();
-    let path = directory.path().join("crash.log");
-    let full = log_of_three_writes(&path);
-
-    let legitimate = [
-        Recovered {
-            value: None,
-            revisions: 0,
-        },
-        Recovered {
-            value: Some(0),
-            revisions: 1,
-        },
-        Recovered {
-            value: Some(1),
-            revisions: 2,
-        },
-        Recovered {
-            value: Some(2),
-            revisions: 3,
-        },
-    ];
-
-    let mut seen = std::collections::BTreeSet::new();
-    for cut in 0..=full.len() {
-        // In place: an instance's identity is derived from its path, so a crash
-        // has to be simulated on the same file.
-        std::fs::write(&path, &full[..cut]).unwrap();
-        let db = Arc::new(
-            FeltDb::open(&path).unwrap_or_else(|error| panic!("cut {cut} did not open: {error}")),
-        );
-        let recovered = inspect(&db);
-        assert!(
-            legitimate.contains(&recovered),
-            "cut {cut} recovered to {recovered:?}, which is not a prefix of the writes"
-        );
-        seen.insert(format!("{recovered:?}"));
-    }
-
-    assert_eq!(
-        seen.len(),
-        4,
-        "all four prefixes are reachable, and nothing else is"
-    );
-}
-
-/// **No crash produces a revision for a record that was never written.**
-///
-/// The ordering is what guarantees this: a record is made durable before the
-/// revision recording it. A crash between them loses the revision, never the
-/// other way round — so recovery can omit a historical fact, and can never
-/// assert one that did not happen.
-///
-/// The omission is then repaired, which is the next test.
-#[test]
-fn no_crash_point_produces_a_phantom_revision() {
-    let directory = TempDir::new().unwrap();
-    let path = directory.path().join("phantom.log");
-    let full = log_of_three_writes(&path);
-
-    for cut in 0..=full.len() {
-        std::fs::write(&path, &full[..cut]).unwrap();
-        let db = Arc::new(FeltDb::open(&path).unwrap());
-        let recovered = inspect(&db);
-        let implied = recovered.value.map(|n| n as usize + 1).unwrap_or(0);
-        assert!(
-            recovered.revisions <= implied,
-            "cut {cut}: {} revisions for a record that reached only {:?}",
-            recovered.revisions,
-            recovered.value
-        );
-    }
-}
-
-/// **A revision lost to a crash is rebuilt from the operation that produced it.**
-///
-/// The crash window between appending a record and appending its revision used
-/// to leave an authority holding less history than its own operation stream
-/// implied. It no longer can: a revision is a deterministic function of the
-/// operation — resource, content, parent, sequence and authority all travel
-/// with it — so recovery reconstructs it.
-///
-/// Nothing is invented. The identity is recomputed from its parts and discarded
-/// if it does not follow from them, exactly as a replicating peer does.
-#[test]
-fn a_revision_lost_to_a_crash_is_rebuilt_from_its_operation() {
-    let directory = TempDir::new().unwrap();
-    let path = directory.path().join("rebuild.log");
-    let full = log_of_three_writes(&path);
-
-    for cut in 0..=full.len() {
-        std::fs::write(&path, &full[..cut]).unwrap();
-        let db = Arc::new(FeltDb::open(&path).unwrap());
-        let recovered = inspect(&db);
-        let implied = recovered.value.map(|n| n as usize + 1).unwrap_or(0);
-        assert_eq!(
-            recovered.revisions, implied,
-            "cut {cut}: a surviving record must carry its history"
-        );
-    }
-}
-
-// ---------------------------------------------------------------------------
-// D1 — a real process death
-// ---------------------------------------------------------------------------
-
-/// **A committed write survives the process dying.**
-///
-/// Truncation models durable bytes; this models the operating system. The child
-/// re-runs this binary, writes, and calls `abort()` — a real `SIGABRT` with no
-/// unwinding, no destructors and no flush on the way out.
-#[test]
-fn a_committed_write_survives_process_death() {
-    if std::env::var("FELTDB_CRASH_CHILD").is_ok() {
-        let path = std::env::var("FELTDB_CRASH_PATH").unwrap();
-        let db = FeltDb::open(&path).unwrap();
-        db.insert("tasks:1", json!({"n": 42})).unwrap();
-        db.update("tasks:1", json!({"n": 43})).unwrap();
-        std::process::abort();
-    }
-
-    let directory = TempDir::new().unwrap();
-    let path = directory.path().join("abort.log");
-    let outcome = std::process::Command::new(std::env::current_exe().unwrap())
-        .args(["a_committed_write_survives_process_death", "--exact"])
-        .env("FELTDB_CRASH_CHILD", "1")
-        .env("FELTDB_CRASH_PATH", &path)
-        .output()
-        .expect("the child runs");
-    assert!(
-        !outcome.status.success(),
-        "the child must actually have died, not returned"
-    );
-
-    let db = Arc::new(FeltDb::open(&path).expect("the database opens after the abort"));
-    let recovered = inspect(&db);
-    assert_eq!(
-        recovered,
-        Recovered {
-            value: Some(43),
-            revisions: 2
-        },
-        "both writes and both revisions survived the process dying"
-    );
-}
-
-// ---------------------------------------------------------------------------
-// D2 — transaction atomicity
-// ---------------------------------------------------------------------------
-
-/// **A transaction is all-or-nothing at every crash point.**
-///
-/// A transaction is one durable record carrying every row it touches, so a
-/// crash can only land before or after it. Never a subset.
-#[test]
-fn a_transaction_is_all_or_nothing_at_every_crash_point() {
-    let directory = TempDir::new().unwrap();
-    let path = directory.path().join("txn.log");
-    let db = Arc::new(FeltDb::open(&path).unwrap());
-    db.apply_atomic_transaction(
-        "t1",
-        None,
-        &[],
-        &[
-            AtomicMutation {
-                capability: "acct".into(),
-                key: "acct:a".into(),
-                value: Some(json!({"balance": 10})),
-            },
-            AtomicMutation {
-                capability: "acct".into(),
-                key: "acct:b".into(),
-                value: Some(json!({"balance": 20})),
-            },
-            AtomicMutation {
-                capability: "acct".into(),
-                key: "acct:c".into(),
-                value: Some(json!({"balance": 30})),
-            },
-        ],
-        None,
-    )
-    .expect("the transaction commits");
-    drop(db);
-    let full = std::fs::read(&path).unwrap();
-
-    let mut committed = 0;
-    let mut absent = 0;
-    for cut in 0..=full.len() {
-        std::fs::write(&path, &full[..cut]).unwrap();
-        let db = FeltDb::open(&path).unwrap();
-        let present: Vec<bool> = ["acct:a", "acct:b", "acct:c"]
-            .iter()
-            .map(|key| db.get::<Value>(key).unwrap().is_some())
-            .collect();
-        let count = present.iter().filter(|p| **p).count();
-        assert!(
-            count == 0 || count == 3,
-            "cut {cut}: a transaction left {count} of 3 rows — {present:?}"
-        );
-        if count == 3 {
-            committed += 1;
-        } else {
-            absent += 1;
-        }
-    }
-    assert!(committed > 0 && absent > 0, "both outcomes are reachable");
-}
-
-// ---------------------------------------------------------------------------
-// Where durability meets replication
-// ---------------------------------------------------------------------------
-
-/// **A crash-recovered database replicates identically.**
-///
-/// This is where durability, replay, history and replication meet. An authority
-/// recovered from a crash must not hold a history that disagrees with what its
-/// own operation stream produces on a peer — otherwise a crash silently forks
-/// the historical record.
-///
-/// Checked at every crash point, on both digests.
-#[test]
-fn a_crash_recovered_database_replicates_identically() {
-    let directory = TempDir::new().unwrap();
-    let path = directory.path().join("recovered.log");
-    let full = log_of_three_writes(&path);
-
-    for cut in (0..=full.len()).step_by(37) {
-        std::fs::write(&path, &full[..cut]).unwrap();
-        let origin = Arc::new(FeltDb::open(&path).unwrap());
-        let origin_store = StateStore::with_feltdb(origin.clone()).unwrap();
-
-        let peer_path = directory.path().join(format!("peer{cut}.log"));
-        let peer = Arc::new(FeltDb::open(&peer_path).unwrap());
-        for operation in origin.operations_since(&HashMap::new()).unwrap() {
-            peer.apply_remote_operation(operation)
-                .unwrap_or_else(|error| panic!("cut {cut}: replication failed: {error}"));
-        }
-        let peer_store = StateStore::with_feltdb(peer.clone()).unwrap();
-
-        assert_eq!(
-            peer.state_digest().unwrap(),
-            origin.state_digest().unwrap(),
-            "cut {cut}: state diverged"
-        );
-        assert_eq!(
-            peer_store.history_digest(),
-            origin_store.history_digest(),
-            "cut {cut}: a crash must not fork the historical record"
-        );
-        let _ = std::fs::remove_file(&peer_path);
-    }
-}
-
-/// A database recovered from a crash accepts writes and continues its history
-/// without reusing a sequence.
-#[test]
-fn a_recovered_database_continues_its_history() {
-    let directory = TempDir::new().unwrap();
-    let path = directory.path().join("continue.log");
-    let full = log_of_three_writes(&path);
-
-    for cut in (0..=full.len()).step_by(53) {
-        std::fs::write(&path, &full[..cut]).unwrap();
-        let db = Arc::new(FeltDb::open(&path).unwrap());
-        let before = inspect(&db).revisions;
-        db.update("tasks:1", json!({"n": 99})).unwrap();
-
-        let store = StateStore::with_feltdb(db.clone()).unwrap();
-        let history = store.history_of("tasks:1");
-        assert_eq!(history.len(), before + 1, "cut {cut}");
-        let mut sequences: Vec<u64> = history.iter().map(|r| r.sequence).collect();
-        let count = sequences.len();
-        sequences.dedup();
-        assert_eq!(sequences.len(), count, "cut {cut}: a sequence was reused");
-    }
-}
-
-// ---------------------------------------------------------------------------
-// D3 — the boundary this does not cross
-// ---------------------------------------------------------------------------
-
-/// **Single-record durability is flush, not fsync — and that is the claim.**
-///
-/// A record reaches the file before the call returns, which is what makes
-/// process-crash durability hold. It is **not** forced to stable storage, so
-/// what survives an operating-system or power failure is not established by
-/// anything here.
-///
-/// This test asserts only what it can — that the bytes are visible outside the
-/// process before success is reported — and deliberately makes no claim about
-/// power loss. The readiness matrix records that as Unproven, and
-/// `docs/architecture/crash-durability.md` states the boundary.
-#[test]
-fn a_write_is_durable_to_the_operating_system_before_it_returns() {
-    let directory = TempDir::new().unwrap();
-    let path = directory.path().join("visible.log");
-    let db = FeltDb::open(&path).unwrap();
-    db.insert("tasks:1", json!({"n": 7})).unwrap();
-
-    // Read the file behind the database's back, with it still open.
-    let log = std::fs::read_to_string(&path).unwrap();
-    assert!(
-        log.contains(r#""n":7"#),
-        "the record reached the file before the write returned"
-    );
-    assert!(
-        log.contains("state:revision:"),
-        "and so did the revision recording it"
-    );
-}
-
-/// An open that had to discard an incomplete final append says so, and a clean
-/// one says that instead.
-///
-/// Recovery being *visible* is what lets an operator tell "my database is fine"
-/// from "my database is fine and my last write is gone".
-#[test]
-fn recovery_after_a_crash_is_reported_rather_than_silent() {
-    let directory = TempDir::new().unwrap();
-    let path = directory.path().join("reported.log");
-    let full = log_of_three_writes(&path);
-
-    // A cut inside the final record: recoverable, and reported.
-    std::fs::write(&path, &full[..full.len() - 20]).unwrap();
-    let db = FeltDb::open(&path).unwrap();
-    assert!(
-        !db.log_recovery().is_clean(),
-        "a crash-recovered open is not reported as clean"
-    );
-    drop(db);
-
-    // Reopening the repaired log is clean.
-    let db = FeltDb::open(&path).unwrap();
-    assert!(db.log_recovery().is_clean());
-}
-
-/// A crash cannot leave a database that refuses to open.
-///
-/// Corruption refuses; an interrupted write recovers. This checks that a
-/// truncated log — the only thing a crash can produce — always falls in the
-/// second class.
-#[test]
-fn a_crash_never_produces_a_database_that_refuses_to_open() {
-    let directory = TempDir::new().unwrap();
-    let path = directory.path().join("openable.log");
-    let full = log_of_three_writes(&path);
-
-    for cut in 0..=full.len() {
-        std::fs::write(&path, &full[..cut]).unwrap();
-        match FeltDb::open(&path) {
-            Ok(_) => {}
-            Err(FlowError::CorruptLogLine(corruption)) => {
-                panic!("cut {cut} was treated as corruption: {corruption}")
-            }
-            Err(error) => panic!("cut {cut} failed to open: {error}"),
-        }
-    }
-}