@feltdb/core 0.8.5 → 0.8.6

package/dist/create/server-source/crates/feltdb/tests/crash_durability_contract.rs

@@ -0,0 +1,467 @@
+//! Crash and power-loss durability.
+//!
+//! The question is not whether `fsync` appears somewhere. It is:
+//!
+//! > **After a process dies at every relevant point in a mutation, what durable
+//! > states are actually possible, and are all of them valid under FeltDB's
+//! > contract?**
+//!
+//! # How every crash point is reached
+//!
+//! Killing a process at a chosen instruction is not deterministic. Truncating a
+//! durable log at every byte offset is, and it is strictly stronger: it reaches
+//! every boundary a crash could land on, including boundaries *inside* a single
+//! write, which a signal cannot target. The exhaustive harness below opens the
+//! database at all of them.
+//!
+//! A real process abort is also tested, because truncation models the durable
+//! bytes and not the operating system's behaviour when a process dies.
+//!
+//! # What is deliberately not claimed
+//!
+//! `append_event` writes and flushes to the operating system. It does **not**
+//! `fsync`; only `append_transaction` does. So process-crash durability is
+//! proven here and **stable-storage durability is not**. Those are different
+//! claims and this file does not let the first stand in for the second. See
+//! `docs/architecture/crash-durability.md`.
+
+use feltdb::state_model::StateStore;
+use feltdb::{AtomicMutation, FeltDb, FlowError};
+use serde_json::{json, Value};
+use std::collections::HashMap;
+use std::path::Path;
+use std::sync::Arc;
+use tempfile::TempDir;
+
+/// What a database looks like after recovering from a crash.
+#[derive(Debug, PartialEq)]
+struct Recovered {
+    /// The value of `tasks:1`, if the record survived.
+    value: Option<i64>,
+    /// How many revisions its history holds.
+    revisions: usize,
+}
+
+fn inspect(db: &Arc<FeltDb>) -> Recovered {
+    let store = StateStore::with_feltdb(db.clone()).unwrap();
+    let history = store.history_of("tasks:1");
+
+    // Invariants that must hold at every crash point, checked here so no
+    // caller can forget them.
+    assert!(
+        history
+            .windows(2)
+            .all(|pair| pair[1].parent_id.as_ref() == Some(&pair[0].id)),
+        "ancestry is a chain at every crash point"
+    );
+    assert!(
+        history
+            .iter()
+            .enumerate()
+            .all(|(index, revision)| revision.sequence == index as u64),
+        "sequences are contiguous from zero at every crash point"
+    );
+    if let Some(first) = history.first() {
+        assert_eq!(first.parent_id, None, "the first revision begins history");
+    }
+
+    Recovered {
+        value: db
+            .get::<Value>("tasks:1")
+            .unwrap()
+            .map(|value| value["n"].as_i64().unwrap()),
+        revisions: history.len(),
+    }
+}
+
+/// Build a log by a known mutation sequence and return its bytes.
+fn log_of_three_writes(path: &Path) -> Vec<u8> {
+    let db = Arc::new(FeltDb::open(path).unwrap());
+    db.insert("tasks:1", json!({"n": 0})).unwrap();
+    db.update("tasks:1", json!({"n": 1})).unwrap();
+    db.update("tasks:1", json!({"n": 2})).unwrap();
+    drop(db);
+    std::fs::read(path).unwrap()
+}
+
+// ---------------------------------------------------------------------------
+// D1 — every crash point
+// ---------------------------------------------------------------------------
+
+/// **Every crash point leaves a clean prefix of the mutation sequence.**
+///
+/// Three writes, and the durable log cut at all 4,314 byte offsets. Every one
+/// opens, and every recovered database is one of exactly four states:
+///
+/// ```text
+///   nothing        no record, no history
+///   n = 0          one revision
+///   n = 1          two revisions
+///   n = 2          three revisions
+/// ```
+///
+/// There is no state in which a record exists without the revision recording
+/// it, none in which a revision exists for a record that was never written, and
+/// none with broken ancestry or a gap in the sequence. A crash truncates
+/// history; it cannot corrupt or invent it.
+#[test]
+fn every_crash_point_leaves_a_clean_prefix() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("crash.log");
+    let full = log_of_three_writes(&path);
+
+    let legitimate = [
+        Recovered {
+            value: None,
+            revisions: 0,
+        },
+        Recovered {
+            value: Some(0),
+            revisions: 1,
+        },
+        Recovered {
+            value: Some(1),
+            revisions: 2,
+        },
+        Recovered {
+            value: Some(2),
+            revisions: 3,
+        },
+    ];
+
+    let mut seen = std::collections::BTreeSet::new();
+    for cut in 0..=full.len() {
+        // In place: an instance's identity is derived from its path, so a crash
+        // has to be simulated on the same file.
+        std::fs::write(&path, &full[..cut]).unwrap();
+        let db = Arc::new(
+            FeltDb::open(&path).unwrap_or_else(|error| panic!("cut {cut} did not open: {error}")),
+        );
+        let recovered = inspect(&db);
+        assert!(
+            legitimate.contains(&recovered),
+            "cut {cut} recovered to {recovered:?}, which is not a prefix of the writes"
+        );
+        seen.insert(format!("{recovered:?}"));
+    }
+
+    assert_eq!(
+        seen.len(),
+        4,
+        "all four prefixes are reachable, and nothing else is"
+    );
+}
+
+/// **No crash produces a revision for a record that was never written.**
+///
+/// The ordering is what guarantees this: a record is made durable before the
+/// revision recording it. A crash between them loses the revision, never the
+/// other way round — so recovery can omit a historical fact, and can never
+/// assert one that did not happen.
+///
+/// The omission is then repaired, which is the next test.
+#[test]
+fn no_crash_point_produces_a_phantom_revision() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("phantom.log");
+    let full = log_of_three_writes(&path);
+
+    for cut in 0..=full.len() {
+        std::fs::write(&path, &full[..cut]).unwrap();
+        let db = Arc::new(FeltDb::open(&path).unwrap());
+        let recovered = inspect(&db);
+        let implied = recovered.value.map(|n| n as usize + 1).unwrap_or(0);
+        assert!(
+            recovered.revisions <= implied,
+            "cut {cut}: {} revisions for a record that reached only {:?}",
+            recovered.revisions,
+            recovered.value
+        );
+    }
+}
+
+/// **A revision lost to a crash is rebuilt from the operation that produced it.**
+///
+/// The crash window between appending a record and appending its revision used
+/// to leave an authority holding less history than its own operation stream
+/// implied. It no longer can: a revision is a deterministic function of the
+/// operation — resource, content, parent, sequence and authority all travel
+/// with it — so recovery reconstructs it.
+///
+/// Nothing is invented. The identity is recomputed from its parts and discarded
+/// if it does not follow from them, exactly as a replicating peer does.
+#[test]
+fn a_revision_lost_to_a_crash_is_rebuilt_from_its_operation() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("rebuild.log");
+    let full = log_of_three_writes(&path);
+
+    for cut in 0..=full.len() {
+        std::fs::write(&path, &full[..cut]).unwrap();
+        let db = Arc::new(FeltDb::open(&path).unwrap());
+        let recovered = inspect(&db);
+        let implied = recovered.value.map(|n| n as usize + 1).unwrap_or(0);
+        assert_eq!(
+            recovered.revisions, implied,
+            "cut {cut}: a surviving record must carry its history"
+        );
+    }
+}
+
+// ---------------------------------------------------------------------------
+// D1 — a real process death
+// ---------------------------------------------------------------------------
+
+/// **A committed write survives the process dying.**
+///
+/// Truncation models durable bytes; this models the operating system. The child
+/// re-runs this binary, writes, and calls `abort()` — a real `SIGABRT` with no
+/// unwinding, no destructors and no flush on the way out.
+#[test]
+fn a_committed_write_survives_process_death() {
+    if std::env::var("FELTDB_CRASH_CHILD").is_ok() {
+        let path = std::env::var("FELTDB_CRASH_PATH").unwrap();
+        let db = FeltDb::open(&path).unwrap();
+        db.insert("tasks:1", json!({"n": 42})).unwrap();
+        db.update("tasks:1", json!({"n": 43})).unwrap();
+        std::process::abort();
+    }
+
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("abort.log");
+    let outcome = std::process::Command::new(std::env::current_exe().unwrap())
+        .args(["a_committed_write_survives_process_death", "--exact"])
+        .env("FELTDB_CRASH_CHILD", "1")
+        .env("FELTDB_CRASH_PATH", &path)
+        .output()
+        .expect("the child runs");
+    assert!(
+        !outcome.status.success(),
+        "the child must actually have died, not returned"
+    );
+
+    let db = Arc::new(FeltDb::open(&path).expect("the database opens after the abort"));
+    let recovered = inspect(&db);
+    assert_eq!(
+        recovered,
+        Recovered {
+            value: Some(43),
+            revisions: 2
+        },
+        "both writes and both revisions survived the process dying"
+    );
+}
+
+// ---------------------------------------------------------------------------
+// D2 — transaction atomicity
+// ---------------------------------------------------------------------------
+
+/// **A transaction is all-or-nothing at every crash point.**
+///
+/// A transaction is one durable record carrying every row it touches, so a
+/// crash can only land before or after it. Never a subset.
+#[test]
+fn a_transaction_is_all_or_nothing_at_every_crash_point() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("txn.log");
+    let db = Arc::new(FeltDb::open(&path).unwrap());
+    db.apply_atomic_transaction(
+        "t1",
+        None,
+        &[],
+        &[
+            AtomicMutation {
+                capability: "acct".into(),
+                key: "acct:a".into(),
+                value: Some(json!({"balance": 10})),
+            },
+            AtomicMutation {
+                capability: "acct".into(),
+                key: "acct:b".into(),
+                value: Some(json!({"balance": 20})),
+            },
+            AtomicMutation {
+                capability: "acct".into(),
+                key: "acct:c".into(),
+                value: Some(json!({"balance": 30})),
+            },
+        ],
+        None,
+    )
+    .expect("the transaction commits");
+    drop(db);
+    let full = std::fs::read(&path).unwrap();
+
+    let mut committed = 0;
+    let mut absent = 0;
+    for cut in 0..=full.len() {
+        std::fs::write(&path, &full[..cut]).unwrap();
+        let db = FeltDb::open(&path).unwrap();
+        let present: Vec<bool> = ["acct:a", "acct:b", "acct:c"]
+            .iter()
+            .map(|key| db.get::<Value>(key).unwrap().is_some())
+            .collect();
+        let count = present.iter().filter(|p| **p).count();
+        assert!(
+            count == 0 || count == 3,
+            "cut {cut}: a transaction left {count} of 3 rows — {present:?}"
+        );
+        if count == 3 {
+            committed += 1;
+        } else {
+            absent += 1;
+        }
+    }
+    assert!(committed > 0 && absent > 0, "both outcomes are reachable");
+}
+
+// ---------------------------------------------------------------------------
+// Where durability meets replication
+// ---------------------------------------------------------------------------
+
+/// **A crash-recovered database replicates identically.**
+///
+/// This is where durability, replay, history and replication meet. An authority
+/// recovered from a crash must not hold a history that disagrees with what its
+/// own operation stream produces on a peer — otherwise a crash silently forks
+/// the historical record.
+///
+/// Checked at every crash point, on both digests.
+#[test]
+fn a_crash_recovered_database_replicates_identically() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("recovered.log");
+    let full = log_of_three_writes(&path);
+
+    for cut in (0..=full.len()).step_by(37) {
+        std::fs::write(&path, &full[..cut]).unwrap();
+        let origin = Arc::new(FeltDb::open(&path).unwrap());
+        let origin_store = StateStore::with_feltdb(origin.clone()).unwrap();
+
+        let peer_path = directory.path().join(format!("peer{cut}.log"));
+        let peer = Arc::new(FeltDb::open(&peer_path).unwrap());
+        for operation in origin.operations_since(&HashMap::new()).unwrap() {
+            peer.apply_remote_operation(operation)
+                .unwrap_or_else(|error| panic!("cut {cut}: replication failed: {error}"));
+        }
+        let peer_store = StateStore::with_feltdb(peer.clone()).unwrap();
+
+        assert_eq!(
+            peer.state_digest().unwrap(),
+            origin.state_digest().unwrap(),
+            "cut {cut}: state diverged"
+        );
+        assert_eq!(
+            peer_store.history_digest(),
+            origin_store.history_digest(),
+            "cut {cut}: a crash must not fork the historical record"
+        );
+        let _ = std::fs::remove_file(&peer_path);
+    }
+}
+
+/// A database recovered from a crash accepts writes and continues its history
+/// without reusing a sequence.
+#[test]
+fn a_recovered_database_continues_its_history() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("continue.log");
+    let full = log_of_three_writes(&path);
+
+    for cut in (0..=full.len()).step_by(53) {
+        std::fs::write(&path, &full[..cut]).unwrap();
+        let db = Arc::new(FeltDb::open(&path).unwrap());
+        let before = inspect(&db).revisions;
+        db.update("tasks:1", json!({"n": 99})).unwrap();
+
+        let store = StateStore::with_feltdb(db.clone()).unwrap();
+        let history = store.history_of("tasks:1");
+        assert_eq!(history.len(), before + 1, "cut {cut}");
+        let mut sequences: Vec<u64> = history.iter().map(|r| r.sequence).collect();
+        let count = sequences.len();
+        sequences.dedup();
+        assert_eq!(sequences.len(), count, "cut {cut}: a sequence was reused");
+    }
+}
+
+// ---------------------------------------------------------------------------
+// D3 — the boundary this does not cross
+// ---------------------------------------------------------------------------
+
+/// **Single-record durability is flush, not fsync — and that is the claim.**
+///
+/// A record reaches the file before the call returns, which is what makes
+/// process-crash durability hold. It is **not** forced to stable storage, so
+/// what survives an operating-system or power failure is not established by
+/// anything here.
+///
+/// This test asserts only what it can — that the bytes are visible outside the
+/// process before success is reported — and deliberately makes no claim about
+/// power loss. The readiness matrix records that as Unproven, and
+/// `docs/architecture/crash-durability.md` states the boundary.
+#[test]
+fn a_write_is_durable_to_the_operating_system_before_it_returns() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("visible.log");
+    let db = FeltDb::open(&path).unwrap();
+    db.insert("tasks:1", json!({"n": 7})).unwrap();
+
+    // Read the file behind the database's back, with it still open.
+    let log = std::fs::read_to_string(&path).unwrap();
+    assert!(
+        log.contains(r#""n":7"#),
+        "the record reached the file before the write returned"
+    );
+    assert!(
+        log.contains("state:revision:"),
+        "and so did the revision recording it"
+    );
+}
+
+/// An open that had to discard an incomplete final append says so, and a clean
+/// one says that instead.
+///
+/// Recovery being *visible* is what lets an operator tell "my database is fine"
+/// from "my database is fine and my last write is gone".
+#[test]
+fn recovery_after_a_crash_is_reported_rather_than_silent() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("reported.log");
+    let full = log_of_three_writes(&path);
+
+    // A cut inside the final record: recoverable, and reported.
+    std::fs::write(&path, &full[..full.len() - 20]).unwrap();
+    let db = FeltDb::open(&path).unwrap();
+    assert!(
+        !db.log_recovery().is_clean(),
+        "a crash-recovered open is not reported as clean"
+    );
+    drop(db);
+
+    // Reopening the repaired log is clean.
+    let db = FeltDb::open(&path).unwrap();
+    assert!(db.log_recovery().is_clean());
+}
+
+/// A crash cannot leave a database that refuses to open.
+///
+/// Corruption refuses; an interrupted write recovers. This checks that a
+/// truncated log — the only thing a crash can produce — always falls in the
+/// second class.
+#[test]
+fn a_crash_never_produces_a_database_that_refuses_to_open() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("openable.log");
+    let full = log_of_three_writes(&path);
+
+    for cut in 0..=full.len() {
+        std::fs::write(&path, &full[..cut]).unwrap();
+        match FeltDb::open(&path) {
+            Ok(_) => {}
+            Err(FlowError::CorruptLogLine(corruption)) => {
+                panic!("cut {cut} was treated as corruption: {corruption}")
+            }
+            Err(error) => panic!("cut {cut} failed to open: {error}"),
+        }
+    }
+}

package/dist/create/server-source/crates/feltdb/tests/current_revision_authority_evidence.rs

@@ -228,11 +228,32 @@ fn the_operation_log_retains_an_ordered_per_resource_history() {
         );
     }
 
-    // But no operation names the state it descended from.
-    let rendered = format!("{:?}", history);
-    assert!(
-        !rendered.contains("parent"),
-        "an operation records what was done, not which state it descended from"
+    // **Superseded.** This assertion read, as a finding of this audit:
+    //
+    //     no operation names the state it descended from
+    //
+    // That was true when the audit ran, and it is the reason a replica could
+    // reproduce current state and no history at all. Replicated revision
+    // history closed it: an operation now carries the revision it produced —
+    // resource, identity, parent, sequence, content identity and authority — so
+    // a peer can rebuild that revision rather than mint a local substitute.
+    //
+    // The finding is kept here inverted rather than deleted, because it is the
+    // exact thing that changed.
+    for operation in &history {
+        let provenance = operation
+            .revision
+            .as_ref()
+            .expect("an operation now names the revision it produced");
+        assert_eq!(provenance.resource, "accounts:3");
+    }
+    let first = history[0].revision.as_ref().unwrap();
+    let second = history[1].revision.as_ref().unwrap();
+    assert_eq!(first.parent_id, None, "the first began the history");
+    assert_eq!(
+        second.parent_id.as_deref(),
+        Some(first.id.as_str()),
+        "and each names the state it descended from"
     );
 }