@feltdb/core 0.8.4 → 0.8.6

package/dist/create/server-source/crates/feltdb/tests/production_readiness_contract.rs

@@ -0,0 +1,1365 @@
+//! The production readiness contract.
+//!
+//! This file **implements nothing and fixes nothing**. It makes the claims
+//! FeltDB can and cannot make executable, so that "production ready" stops
+//! being a feeling and becomes a matrix with evidence behind each row.
+//!
+//! It is the executable half of
+//! `docs/architecture/production-readiness.md`, and its failure cases are
+//! catalogued in `docs/architecture/production-failure-matrix.md`.
+//!
+//! # The discipline
+//!
+//! Every claim carries a [`Status`]. **A green test never implies a claim
+//! broader than the scenario it runs.** Where a production claim is larger than
+//! what any test exercises, the status is `PartiallyProven` and the test says
+//! what is missing. Where nothing can be tested, the status is `Unproven` and
+//! there is no test pretending otherwise.
+//!
+//! Several tests here assert a **gap** rather than a guarantee — that a
+//! pre-model database opens with its history invisible, that a truncated log is
+//! accepted silently. They are written that way on purpose: the gap becomes a
+//! regression test, and closing it fails this suite and forces the matrix to be
+//! updated deliberately.
+
+use feltdb::state_model::{ParentLookup, RetentionPolicy, StateStore};
+use feltdb::{FeltDb, FlowError, JsonCasResult};
+use serde_json::json;
+use std::collections::HashMap;
+use std::io::Write;
+use std::sync::Arc;
+use tempfile::TempDir;
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+enum Status {
+    /// Executable evidence exists for the claim as stated.
+    Proven,
+    /// Bounded scenarios work; the production claim is larger than the evidence.
+    PartiallyProven,
+    /// No sufficient evidence. Not "probably works".
+    Unproven,
+    /// A known defect prevents the claim.
+    Blocked,
+    /// Explicitly outside what the product can offer, with a reason in the data
+    /// rather than in the schedule.
+    NotApplicable,
+}
+
+/// The claim matrix. `docs/architecture/production-readiness.md` is its prose.
+const CLAIMS: &[(&str, Status)] = &[
+    // Durability
+    (
+        "D1 committed mutations survive a clean restart",
+        Status::Proven,
+    ),
+    (
+        "D2 revision history survives a clean restart",
+        Status::Proven,
+    ),
+    (
+        "D3 retention policy and horizon survive a restart",
+        Status::Proven,
+    ),
+    (
+        "D4 a reopened database continues history rather than restarting it",
+        Status::Proven,
+    ),
+    (
+        "D5 a single write survives operating-system or power loss",
+        Status::Unproven,
+    ),
+    (
+        "D6 every crash point leaves a clean prefix of the mutation sequence",
+        Status::Proven,
+    ),
+    (
+        "D7 a crash never produces a revision for a record that was not written",
+        Status::Proven,
+    ),
+    (
+        "D8 a committed write survives the process dying",
+        Status::Proven,
+    ),
+    (
+        "D9 a transaction is all-or-nothing across a crash",
+        Status::Proven,
+    ),
+    (
+        "D10 a crash-recovered database replicates identically",
+        Status::Proven,
+    ),
+    // Atomicity and concurrency
+    (
+        "A1 a stale fenced write is rejected and mints nothing",
+        Status::Proven,
+    ),
+    (
+        "A2 concurrent writes to distinct resources keep distinct chains",
+        Status::Proven,
+    ),
+    (
+        "A3 concurrent writes to one resource form one correct chain",
+        Status::Proven,
+    ),
+    (
+        "A4 a duplicate remote operation is idempotent",
+        Status::Proven,
+    ),
+    (
+        "A5 a remote operation with a sequence gap is rejected",
+        Status::Proven,
+    ),
+    (
+        "A6 a crash part-way through a mutation leaves consistent state",
+        Status::Proven,
+    ),
+    // Recovery
+    (
+        "R1 an incomplete final append is recovered and reported, not silently dropped",
+        Status::Proven,
+    ),
+    (
+        "R2 a complete but invalid record refuses the open, wherever it sits",
+        Status::Proven,
+    ),
+    (
+        "R3 damaged ancestry is distinguishable from expired ancestry",
+        Status::Proven,
+    ),
+    (
+        "R4 ancestry damage is surfaced without being asked for",
+        Status::Unproven,
+    ),
+    // History and retention
+    (
+        "H1 history is bounded under a configured policy",
+        Status::Proven,
+    ),
+    (
+        "H2 history is unbounded without one, by design",
+        Status::Proven,
+    ),
+    (
+        "H3 a policy change takes effect in both directions",
+        Status::Proven,
+    ),
+    ("H4 maintenance never rewrites history", Status::Proven),
+    ("H5 forked history is preserved", Status::Proven),
+    // Backup and restore
+    (
+        "B1 a durable backup artifact is self-contained and independently verifiable",
+        Status::Proven,
+    ),
+    (
+        "B2 a restore proves the result means the same thing, not merely that it parsed",
+        Status::Proven,
+    ),
+    (
+        "B3 an altered or truncated backup is refused rather than restored",
+        Status::Proven,
+    ),
+    (
+        "B4 a restore will not overwrite an existing database",
+        Status::Proven,
+    ),
+    (
+        "B5 backup and restore are exposed as an operator command-line workflow",
+        Status::Unproven,
+    ),
+    // Replication
+    ("S1 current state replicates to a peer", Status::Proven),
+    (
+        "S2 revision history replicates to a peer with identical identities",
+        Status::Proven,
+    ),
+    (
+        "S3 replicated history preserves parents, sequences, forks and provenance",
+        Status::Proven,
+    ),
+    (
+        "S4 state equivalence and historical equivalence are checked separately",
+        Status::Proven,
+    ),
+    (
+        "S5 a peer that sends no revision provenance contributes no history",
+        Status::Proven,
+    ),
+    // Workload envelope
+    (
+        "E1 retention cost per write grows with the retention window",
+        Status::Proven,
+    ),
+    (
+        "E2 retention increases log size before compaction",
+        Status::Proven,
+    ),
+    (
+        "E3 durability is a selectable contract, and each mode states its guarantee",
+        Status::Proven,
+    ),
+    (
+        "E4 a stable-storage barrier per write has a measured cost",
+        Status::Proven,
+    ),
+    (
+        "E5 the durability mode changes the contract and not the stored database",
+        Status::Proven,
+    ),
+    (
+        "E6 the workload envelope is recorded against a frozen workload and a named environment",
+        Status::Proven,
+    ),
+    (
+        "E7 grouped durability is measurably cheaper than a barrier per write",
+        Status::Unproven,
+    ),
+    // Upgrade and migration
+    (
+        "U1 an incompatible durable format is detected before any record is interpreted",
+        Status::Proven,
+    ),
+    (
+        "U2 a pre-model database's revision history is losslessly migratable",
+        Status::NotApplicable,
+    ),
+    (
+        "U3 a persisted durable format version exists and is checked on open",
+        Status::Proven,
+    ),
+    (
+        "U4 a refused open leaves the database byte-identical",
+        Status::Proven,
+    ),
+    (
+        "U5 a durable format from a newer build is refused",
+        Status::Proven,
+    ),
+    (
+        "U6 a snapshot from another format is refused",
+        Status::Proven,
+    ),
+    // Observability
+    (
+        "O1 an operator can inspect history and retention state",
+        Status::PartiallyProven,
+    ),
+    (
+        "O2 reported health reflects actual storage state",
+        Status::Proven,
+    ),
+    (
+        "O3 a recovered open is distinguishable from a clean one in health",
+        Status::Proven,
+    ),
+    (
+        "O4 health reports only conditions that were observed",
+        Status::Proven,
+    ),
+    (
+        "O5 the remaining health fields are compile-time constants",
+        Status::PartiallyProven,
+    ),
+];
+
+fn status(claim: &str) -> Status {
+    CLAIMS
+        .iter()
+        .find(|(name, _)| *name == claim)
+        .unwrap_or_else(|| panic!("{claim} is not in the claim matrix"))
+        .1
+}
+
+/// Assert a claim's recorded status, so a test cannot quietly widen its claim.
+fn claiming(claim: &str, expected: Status) {
+    assert_eq!(
+        status(claim),
+        expected,
+        "{claim}: this test exercises a {expected:?} claim, but the matrix records \
+         something else. Update docs/architecture/production-readiness.md deliberately."
+    );
+}
+
+fn durable(name: &str) -> (TempDir, Arc<FeltDb>, StateStore) {
+    let directory = TempDir::new().unwrap();
+    let db = Arc::new(FeltDb::open(directory.path().join(name)).unwrap());
+    let store = StateStore::with_feltdb(db.clone()).unwrap();
+    (directory, db, store)
+}
+
+// ===========================================================================
+// Durability
+// ===========================================================================
+
+/// **Claim D1–D4.** A clean restart preserves state, history, retention and
+/// position.
+///
+/// *Scenario*: write, mint under a policy, close, reopen, write again.
+/// *Expected*: everything survives and history continues.
+/// *Observed*: it does.
+/// *Production implication*: a planned restart is safe. This says nothing about
+/// an unplanned one — see D5.
+#[test]
+fn a_clean_restart_preserves_state_history_and_retention() {
+    claiming(
+        "D1 committed mutations survive a clean restart",
+        Status::Proven,
+    );
+    claiming(
+        "D2 revision history survives a clean restart",
+        Status::Proven,
+    );
+    claiming(
+        "D3 retention policy and horizon survive a restart",
+        Status::Proven,
+    );
+    claiming(
+        "D4 a reopened database continues history rather than restarting it",
+        Status::Proven,
+    );
+
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("restart.log");
+    let db = Arc::new(FeltDb::open(&path).unwrap());
+    let store = StateStore::with_feltdb(db.clone()).unwrap();
+    store
+        .set_retention_policy("tasks:1", RetentionPolicy::keep_last(3))
+        .unwrap();
+    db.insert("tasks:1", json!({"n": 0})).unwrap();
+    for n in 1..10 {
+        db.update("tasks:1", json!({ "n": n })).unwrap();
+    }
+    let horizon_before = store.retention_horizon("tasks:1");
+    drop(store);
+    drop(db);
+
+    let reopened = Arc::new(FeltDb::open(&path).unwrap());
+    let store = StateStore::with_feltdb(reopened.clone()).unwrap();
+
+    let current: serde_json::Value = reopened.get("tasks:1").unwrap().unwrap();
+    assert_eq!(current["n"], json!(9), "D1");
+    assert_eq!(store.history_of("tasks:1").len(), 3, "D2");
+    assert_eq!(
+        store.retention_policy("tasks:1"),
+        RetentionPolicy::keep_last(3),
+        "D3 policy"
+    );
+    assert_eq!(
+        store.retention_horizon("tasks:1"),
+        horizon_before,
+        "D3 horizon"
+    );
+
+    reopened.update("tasks:1", json!({"n": 10})).unwrap();
+    let history = store.history_of("tasks:1");
+    assert_eq!(history.last().unwrap().sequence, 10, "D4");
+}
+
+/// **The crash boundary, established rather than assumed.**
+///
+/// D6–D10 and A6 are proven by cutting a real durable log at **every byte
+/// offset** and opening it there — strictly stronger than killing a process at
+/// chosen points, because it reaches boundaries inside a single write that a
+/// signal cannot target. A real `abort()` covers what truncation cannot: the
+/// operating system's behaviour when a process dies.
+///
+/// Every crash point leaves a clean prefix of the mutation sequence. No crash
+/// produces a revision for a record that was never written, breaks ancestry, or
+/// leaves a database that refuses to open. A transaction is all-or-nothing. And
+/// a crash-recovered authority replicates to a peer with identical state *and*
+/// history digests, so a crash cannot fork the historical record.
+///
+/// **D5 stays Unproven, and that is the honest result.** `append_event` writes
+/// and flushes to the operating system; it does not `fsync`. Only
+/// `append_transaction` does. So process-crash durability is proven and
+/// stable-storage durability is not, and this file does not let the first stand
+/// in for the second. Calling `fsync` somewhere is not evidence about power
+/// loss.
+///
+/// Full evidence in `crates/feltdb/tests/crash_durability_contract.rs`.
+#[test]
+fn crash_recovery_is_established_and_power_loss_is_not() {
+    claiming(
+        "D5 a single write survives operating-system or power loss",
+        Status::Unproven,
+    );
+    claiming(
+        "D6 every crash point leaves a clean prefix of the mutation sequence",
+        Status::Proven,
+    );
+    claiming(
+        "D7 a crash never produces a revision for a record that was not written",
+        Status::Proven,
+    );
+    claiming(
+        "D8 a committed write survives the process dying",
+        Status::Proven,
+    );
+    claiming(
+        "D9 a transaction is all-or-nothing across a crash",
+        Status::Proven,
+    );
+    claiming(
+        "D10 a crash-recovered database replicates identically",
+        Status::Proven,
+    );
+    claiming(
+        "A6 a crash part-way through a mutation leaves consistent state",
+        Status::Proven,
+    );
+
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("crash.log");
+    let db = Arc::new(FeltDb::open(&path).unwrap());
+    db.insert("tasks:1", json!({"n": 0})).unwrap();
+    db.update("tasks:1", json!({"n": 1})).unwrap();
+    drop(db);
+    let full = std::fs::read(&path).unwrap();
+
+    // D6, D7, A6: every crash point, and only clean prefixes.
+    for cut in 0..=full.len() {
+        std::fs::write(&path, &full[..cut]).unwrap();
+        let db = Arc::new(FeltDb::open(&path).unwrap());
+        let store = StateStore::with_feltdb(db.clone()).unwrap();
+        let history = store.history_of("tasks:1");
+        let value = db
+            .get::<serde_json::Value>("tasks:1")
+            .unwrap()
+            .map(|value| value["n"].as_i64().unwrap());
+        assert_eq!(
+            history.len(),
+            value.map(|n| n as usize + 1).unwrap_or(0),
+            "cut {cut}: a surviving record carries exactly its own history"
+        );
+        assert!(history
+            .windows(2)
+            .all(|pair| pair[1].parent_id.as_ref() == Some(&pair[0].id)));
+    }
+
+    // D5: the bytes reach the file before the write returns. That is
+    // process-crash durability, and it is not a power-loss claim.
+    std::fs::write(&path, &full).unwrap();
+    let db = FeltDb::open(&path).unwrap();
+    db.insert("tasks:2", json!({"n": 5})).unwrap();
+    assert!(std::fs::read_to_string(&path).unwrap().contains("tasks:2"));
+}
+
+// ===========================================================================
+// Atomicity and concurrency
+// ===========================================================================
+
+/// **Claim A1.** A refused fence changes nothing — not the row, not the history.
+#[test]
+fn a_stale_fenced_write_is_rejected_and_mints_nothing() {
+    claiming(
+        "A1 a stale fenced write is rejected and mints nothing",
+        Status::Proven,
+    );
+    let (_directory, db, store) = durable("cas.log");
+    db.insert("tasks:1", json!({"title": "A", "__version": 1}))
+        .unwrap();
+    let before = store.history_of("tasks:1").len();
+
+    let refused = db
+        .compare_and_set_json("tasks:1", 99, None, None, false, json!({"title": "Z"}))
+        .unwrap();
+    assert!(matches!(refused, JsonCasResult::VersionConflict { .. }));
+
+    assert_eq!(
+        store.history_of("tasks:1").len(),
+        before,
+        "no revision minted"
+    );
+    let current: serde_json::Value = db.get("tasks:1").unwrap().unwrap();
+    assert_eq!(current["title"], json!("A"), "the row is unchanged");
+}
+
+/// **Claims A2 and A3.** Concurrency produces correct chains, on separate
+/// resources and on one shared resource.
+///
+/// *Production implication*: the write path serializes minting correctly. It
+/// says nothing about throughput under contention — see E1.
+#[test]
+fn concurrent_writes_produce_correct_chains() {
+    claiming(
+        "A2 concurrent writes to distinct resources keep distinct chains",
+        Status::Proven,
+    );
+    claiming(
+        "A3 concurrent writes to one resource form one correct chain",
+        Status::Proven,
+    );
+
+    let directory = TempDir::new().unwrap();
+    let db = Arc::new(FeltDb::open(directory.path().join("concurrent.log")).unwrap());
+
+    let mut handles = Vec::new();
+    for thread_index in 0..4 {
+        let db = db.clone();
+        handles.push(std::thread::spawn(move || {
+            for n in 0..25 {
+                db.update(&format!("tasks:{thread_index}"), json!({ "n": n }))
+                    .unwrap();
+                db.update("shared:1", json!({ "t": thread_index, "n": n }))
+                    .unwrap();
+            }
+        }));
+    }
+    for handle in handles {
+        handle.join().unwrap();
+    }
+    let store = StateStore::with_feltdb(db.clone()).unwrap();
+
+    for thread_index in 0..4 {
+        let history = store.history_of(&format!("tasks:{thread_index}"));
+        assert_eq!(history.len(), 25, "A2 length");
+        assert!(
+            history
+                .windows(2)
+                .all(|pair| pair[1].parent_id.as_ref() == Some(&pair[0].id)),
+            "A2 chained"
+        );
+    }
+
+    let shared = store.history_of("shared:1");
+    assert_eq!(shared.len(), 100, "A3 every write minted exactly once");
+    assert!(
+        shared
+            .windows(2)
+            .all(|pair| pair[1].parent_id.as_ref() == Some(&pair[0].id)),
+        "A3 one chain, not four"
+    );
+    let mut sequences: Vec<u64> = shared.iter().map(|revision| revision.sequence).collect();
+    sequences.dedup();
+    assert_eq!(sequences.len(), 100, "A3 sequences are unique");
+}
+
+/// **Claims A4 and A5.** Replayed and out-of-order remote operations.
+#[test]
+fn replayed_and_gapped_remote_operations_are_handled_explicitly() {
+    claiming(
+        "A4 a duplicate remote operation is idempotent",
+        Status::Proven,
+    );
+    claiming(
+        "A5 a remote operation with a sequence gap is rejected",
+        Status::Proven,
+    );
+
+    let directory = TempDir::new().unwrap();
+    let primary = Arc::new(FeltDb::open(directory.path().join("p.log")).unwrap());
+    let replica = Arc::new(FeltDb::open(directory.path().join("r.log")).unwrap());
+
+    primary.insert("tasks:1", json!({"n": 1})).unwrap();
+    primary.update("tasks:1", json!({"n": 2})).unwrap();
+    let operations = primary.operations_since(&HashMap::new()).unwrap();
+    assert!(operations.len() >= 2);
+
+    // Applying out of order is refused rather than silently accepted.
+    assert!(
+        replica
+            .apply_remote_operation(operations[1].clone())
+            .is_err(),
+        "A5"
+    );
+
+    for operation in &operations {
+        replica.apply_remote_operation(operation.clone()).unwrap();
+    }
+    let after_first_pass: serde_json::Value = replica.get("tasks:1").unwrap().unwrap();
+
+    // Replaying the whole stream is a no-op.
+    for operation in &operations {
+        replica.apply_remote_operation(operation.clone()).unwrap();
+    }
+    let after_replay: serde_json::Value = replica.get("tasks:1").unwrap().unwrap();
+    assert_eq!(after_first_pass, after_replay, "A4");
+}
+
+// ===========================================================================
+// Recovery and corruption
+// ===========================================================================
+
+/// **Claims R1 and R2, closed.** Durable corruption no longer opens quietly.
+///
+/// *Was*: a log truncated mid-record opened having silently discarded the tail,
+/// and a complete line containing invalid data at end-of-file was dropped the
+/// same way. Neither produced an error, a warning, or any signal a caller could
+/// read. An operator could not distinguish a healthy database from one that had
+/// silently lost its most recent writes.
+/// *Now*: exactly one condition discards durable bytes — a final record with no
+/// terminator, which cannot be a record this database finished writing. It is
+/// discarded, and the recovery is reported. Everything else invalid, at any
+/// position including end-of-file, refuses the open with a located error and
+/// leaves the file byte-identical.
+/// *Production implication*: position in the file is no longer treated as
+/// evidence that a record was finished. The terminator is.
+///
+/// Full evidence in `crates/feltdb/tests/durable_corruption_contract.rs`.
+#[test]
+fn durable_corruption_fails_closed_and_recovery_is_reported() {
+    claiming(
+        "R1 an incomplete final append is recovered and reported, not silently dropped",
+        Status::Proven,
+    );
+    claiming(
+        "R2 a complete but invalid record refuses the open, wherever it sits",
+        Status::Proven,
+    );
+
+    // R1: a torn append is recovered, and says so.
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("torn.log");
+    let db = FeltDb::open(&path).unwrap();
+    for n in 0..5 {
+        db.insert(&format!("tasks:{n}"), json!({ "n": n })).unwrap();
+    }
+    drop(db);
+    let bytes = std::fs::read(&path).unwrap();
+    std::fs::write(&path, &bytes[..bytes.len() - 600]).unwrap();
+
+    let reopened = FeltDb::open(&path).expect("a torn tail is recoverable");
+    assert!(
+        !reopened.log_recovery().is_clean(),
+        "R1: the recovery is visible rather than silent"
+    );
+
+    // R2: a complete record containing invalid data is corruption, at EOF as
+    // much as anywhere else.
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("garbage.log");
+    let db = FeltDb::open(&path).unwrap();
+    db.insert("tasks:1", json!({"n": 1})).unwrap();
+    drop(db);
+    let before = std::fs::read(&path).unwrap();
+    let mut file = std::fs::OpenOptions::new()
+        .append(true)
+        .open(&path)
+        .unwrap();
+    writeln!(file, "this is not json at all").unwrap();
+    drop(file);
+
+    match FeltDb::open(&path) {
+        Err(FlowError::CorruptLogLine(corruption)) => {
+            assert!(corruption.line_number > 0, "R2: the damage is located");
+        }
+        Err(other) => panic!("wrong error: {other}"),
+        Ok(_) => panic!("R2: the corrupt database opened"),
+    }
+    assert!(
+        std::fs::read(&path).unwrap().starts_with(&before),
+        "R2: a refused open modified nothing that was already there"
+    );
+}
+
+/// **Claim R3.** Damaged ancestry and expired ancestry are different facts, and
+/// the store says which.
+///
+/// This is the one corruption case FeltDB genuinely detects, and it exists
+/// because retention needed the distinction. A parent absent at or below the
+/// horizon was expired on purpose; absent above it is damage.
+///
+/// *Production implication*: ancestry damage is detectable — **on inspection**.
+/// Nothing surfaces it unprompted, which is claim R4.
+#[test]
+fn damaged_ancestry_is_distinguishable_from_expired_ancestry() {
+    claiming(
+        "R3 damaged ancestry is distinguishable from expired ancestry",
+        Status::Proven,
+    );
+    claiming(
+        "R4 ancestry damage is surfaced without being asked for",
+        Status::Unproven,
+    );
+
+    // Damage: a mid-history revision removed with no retention involved.
+    let (_directory, db, store) = durable("damaged.log");
+    db.insert("tasks:1", json!({"n": 0})).unwrap();
+    db.update("tasks:1", json!({"n": 1})).unwrap();
+    db.update("tasks:1", json!({"n": 2})).unwrap();
+    let history = store.history_of("tasks:1");
+    let middle = history[1].id.clone();
+    let child = history[2].id.clone();
+    db.delete(&format!("state:revision:{}", middle.as_hex()))
+        .unwrap();
+
+    assert!(
+        matches!(store.parent_of(&child), Some(ParentLookup::Missing(_))),
+        "R3: reported as damage"
+    );
+    assert_eq!(
+        store.retention_horizon("tasks:1"),
+        0,
+        "no retention happened"
+    );
+
+    // Expiry: the same shape, but a decision.
+    let (_directory, db, store) = durable("expired.log");
+    store
+        .set_retention_policy("tasks:1", RetentionPolicy::keep_last(2))
+        .unwrap();
+    db.insert("tasks:1", json!({"n": 0})).unwrap();
+    for n in 1..6 {
+        db.update("tasks:1", json!({ "n": n })).unwrap();
+    }
+    let oldest = store.history_of("tasks:1")[0].id.clone();
+    assert!(
+        matches!(store.parent_of(&oldest), Some(ParentLookup::Expired(_))),
+        "R3: reported as a decision"
+    );
+}
+
+// ===========================================================================
+// History and retention
+// ===========================================================================
+
+/// **Claims H1, H2, H3.** The retention envelope, in both directions.
+#[test]
+fn the_retention_envelope_behaves_in_both_directions() {
+    claiming(
+        "H1 history is bounded under a configured policy",
+        Status::Proven,
+    );
+    claiming(
+        "H2 history is unbounded without one, by design",
+        Status::Proven,
+    );
+    claiming(
+        "H3 a policy change takes effect in both directions",
+        Status::Proven,
+    );
+
+    // Unbounded by default.
+    let (_directory, db, store) = durable("unbounded.log");
+    db.insert("tasks:1", json!({"n": 0})).unwrap();
+    for n in 1..50 {
+        db.update("tasks:1", json!({ "n": n })).unwrap();
+    }
+    assert_eq!(store.history_of("tasks:1").len(), 50, "H2");
+
+    // Tightening expires immediately.
+    store
+        .set_retention_policy("tasks:1", RetentionPolicy::keep_last(3))
+        .unwrap();
+    assert_eq!(store.history_of("tasks:1").len(), 3, "H3 tighten");
+
+    // And stays bounded as writes continue.
+    for n in 50..100 {
+        db.update("tasks:1", json!({ "n": n })).unwrap();
+    }
+    assert_eq!(store.history_of("tasks:1").len(), 3, "H1");
+
+    // Loosening lets it grow again — it does not restore what was expired.
+    store
+        .set_retention_policy("tasks:1", RetentionPolicy::unbounded())
+        .unwrap();
+    for n in 100..110 {
+        db.update("tasks:1", json!({ "n": n })).unwrap();
+    }
+    assert_eq!(store.history_of("tasks:1").len(), 13, "H3 loosen");
+}
+
+/// **Claims H4 and H5.** Maintenance never rewrites history, and forks survive.
+#[test]
+fn maintenance_never_rewrites_history_and_forks_survive() {
+    claiming("H4 maintenance never rewrites history", Status::Proven);
+    claiming("H5 forked history is preserved", Status::Proven);
+
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("forks.log");
+    let db = Arc::new(FeltDb::open(&path).unwrap());
+    let store = StateStore::with_feltdb(db.clone()).unwrap();
+
+    let base = store
+        .create("docs:1", r#"{"v":0}"#.into(), "a".into())
+        .unwrap();
+    let left = store
+        .commit(r#"{"v":1}"#.into(), &base, "a".into())
+        .unwrap();
+    let right = store
+        .commit(r#"{"v":2}"#.into(), &base, "b".into())
+        .unwrap();
+    assert_ne!(left.id, right.id, "H5: two heads from one base");
+    assert_eq!(left.parent_id, right.parent_id);
+
+    // Compaction, an unrelated write, and a retention pass on another resource.
+    db.insert("tasks:1", json!({"unrelated": true})).unwrap();
+    let versions = db.operation_versions().unwrap();
+    db.acknowledge_peer_versions("peer-1".to_string(), versions)
+        .unwrap();
+    db.compact_operation_log(&["peer-1".to_string()]).unwrap();
+    store
+        .set_retention_policy("tasks:1", RetentionPolicy::keep_last(1))
+        .unwrap();
+
+    for revision in [&base, &left, &right] {
+        let stored = store.get(&revision.id).expect("H4: still there");
+        assert_eq!(stored.parent_id, revision.parent_id, "H4: ancestry intact");
+        assert_eq!(stored.content_id, revision.content_id);
+    }
+}
+
+// ===========================================================================
+// Backup and restore
+// ===========================================================================
+
+/// **The B-block, mostly closed.**
+///
+/// *Was*: `export_snapshot` and `install_snapshot` round-tripped state and
+/// history in process, but there was no artifact, no format, no verification an
+/// operator could run and no restore procedure. An operator had no supported
+/// way to back up or restore a database.
+/// *Now*: a backup is a self-contained versioned artifact that outlives the
+/// database it came from; it is verified without opening it as a live database;
+/// and a restore recomputes what the **restored database** means and refuses if
+/// it disagrees with the artifact.
+/// *Production implication*: recovery from operator error or disk loss is a
+/// supported operation — as a library call. B5 records what is still missing:
+/// none of it is exposed as a command an operator runs without writing code.
+///
+/// Full evidence in `crates/feltdb/tests/durable_backup_contract.rs`.
+#[test]
+fn backup_and_restore_preserve_meaning_and_refuse_damage() {
+    claiming(
+        "B1 a durable backup artifact is self-contained and independently verifiable",
+        Status::Proven,
+    );
+    claiming(
+        "B2 a restore proves the result means the same thing, not merely that it parsed",
+        Status::Proven,
+    );
+    claiming(
+        "B3 an altered or truncated backup is refused rather than restored",
+        Status::Proven,
+    );
+    claiming(
+        "B4 a restore will not overwrite an existing database",
+        Status::Proven,
+    );
+    claiming(
+        "B5 backup and restore are exposed as an operator command-line workflow",
+        Status::Unproven,
+    );
+
+    let directory = TempDir::new().unwrap();
+    let source = directory.path().join("source.log");
+    let db = Arc::new(FeltDb::open(&source).unwrap());
+    db.insert("tasks:1", json!({"n": 0})).unwrap();
+    db.update("tasks:1", json!({"n": 1})).unwrap();
+
+    // B1: an artifact that verifies on its own, with the source deleted.
+    let backup = directory.path().join("db.feltbackup");
+    let written = db.write_backup(&backup).unwrap();
+    drop(db);
+    std::fs::remove_file(&source).unwrap();
+    assert_eq!(feltdb::verify_backup(&backup).unwrap(), written);
+
+    // B2: the restore proves equivalence, and the history comes with it.
+    let target = directory.path().join("restored.log");
+    let (restored, verification) = FeltDb::restore_backup(&backup, &target).unwrap();
+    assert_eq!(verification.meaning_digest, written.meaning_digest);
+    let restored = Arc::new(restored);
+    let store = StateStore::with_feltdb(restored.clone()).unwrap();
+    assert_eq!(store.history_of("tasks:1").len(), 2);
+
+    // B3: one altered byte inside a value is refused on meaning.
+    let text = std::fs::read_to_string(&backup).unwrap();
+    std::fs::write(&backup, text.replacen(r#""n":1"#, r#""n":7"#, 1)).unwrap();
+    assert!(matches!(
+        feltdb::verify_backup(&backup),
+        Err(FlowError::BackupRejected(_))
+    ));
+
+    // B4: a restore does not overwrite.
+    std::fs::write(&backup, &text).unwrap();
+    assert!(matches!(
+        FeltDb::restore_backup(&backup, &target),
+        Err(FlowError::BackupRejected(_))
+    ));
+}
+
+// ===========================================================================
+// Replication
+// ===========================================================================
+
+/// **The S-block, closed.** Replication now carries history, not only state.
+///
+/// *Was*: `apply_remote_operation` wrote rows directly and never reached the
+/// minting path, so a replica received the same current state and no revision
+/// history at all. Revision history was a local artifact: failover lost it, and
+/// reconciliation on a replica had no `base`.
+/// *Now*: an operation carries the revision it produced on its originating
+/// authority — resource, identity, parent, sequence, content identity and the
+/// authority itself — and the peer **reconstructs that revision** rather than
+/// minting a local substitute for the resulting state. The identity is
+/// recomputed from its parts and refused if it does not follow from them.
+/// *Production implication*: two authorities converge on the same historical
+/// facts, forks and all, so a replica holds what a three-way reconciliation
+/// needs.
+///
+/// S4 is the discipline that keeps this honest: state equivalence and
+/// historical equivalence are separate digests, and a database with the same
+/// values and a different history fails the second while passing the first.
+///
+/// Full evidence in `crates/feltdb/tests/replicated_history_contract.rs`.
+#[test]
+fn replication_carries_history_and_not_only_state() {
+    claiming("S1 current state replicates to a peer", Status::Proven);
+    claiming(
+        "S2 revision history replicates to a peer with identical identities",
+        Status::Proven,
+    );
+    claiming(
+        "S3 replicated history preserves parents, sequences, forks and provenance",
+        Status::Proven,
+    );
+    claiming(
+        "S4 state equivalence and historical equivalence are checked separately",
+        Status::Proven,
+    );
+    claiming(
+        "S5 a peer that sends no revision provenance contributes no history",
+        Status::Proven,
+    );
+
+    let directory = TempDir::new().unwrap();
+    let primary = Arc::new(FeltDb::open(directory.path().join("primary.log")).unwrap());
+    let replica = Arc::new(FeltDb::open(directory.path().join("replica.log")).unwrap());
+
+    primary.insert("tasks:1", json!({"n": 1})).unwrap();
+    primary.update("tasks:1", json!({"n": 2})).unwrap();
+    for operation in primary.operations_since(&HashMap::new()).unwrap() {
+        replica.apply_remote_operation(operation).unwrap();
+    }
+
+    let primary_store = StateStore::with_feltdb(primary.clone()).unwrap();
+    let replica_store = StateStore::with_feltdb(replica.clone()).unwrap();
+
+    // S1 and S2.
+    assert_eq!(
+        replica.state_digest().unwrap(),
+        primary.state_digest().unwrap()
+    );
+    assert_eq!(
+        replica_store.history_digest(),
+        primary_store.history_digest(),
+        "the same history, not merely the same values"
+    );
+
+    // S3: identities, parents, sequences and provenance, not a re-mint.
+    let origin = primary_store.history_of("tasks:1");
+    let copy = replica_store.history_of("tasks:1");
+    assert_eq!(copy.len(), 2);
+    for (mine, theirs) in copy.iter().zip(origin.iter()) {
+        assert_eq!(mine.id, theirs.id);
+        assert_eq!(mine.parent_id, theirs.parent_id);
+        assert_eq!(mine.sequence, theirs.sequence);
+        assert_eq!(mine.authority, theirs.authority);
+    }
+
+    // S4: the negative control the whole block rests on.
+    let shallow = Arc::new(FeltDb::open(directory.path().join("shallow.log")).unwrap());
+    shallow.insert("tasks:1", json!({"n": 2})).unwrap();
+    let shallow_store = StateStore::with_feltdb(shallow.clone()).unwrap();
+    assert_eq!(
+        shallow.state_digest().unwrap(),
+        primary.state_digest().unwrap(),
+        "same values"
+    );
+    assert_ne!(
+        shallow_store.history_digest(),
+        primary_store.history_digest(),
+        "different history, and the check can tell"
+    );
+}
+
+// ===========================================================================
+// Workload envelope
+// ===========================================================================
+
+/// **Claims E1 and E2.** Retention has a measurable cost, in two directions
+/// that pull against each other.
+///
+/// *Observed*: the per-write cost of retention grows with the retention window,
+/// because expiry walks the resource's retained history on every mint. And
+/// retention makes the log **larger** before compaction, because each expiry
+/// appends a tombstone.
+/// *Production implication*: a large retention window costs write throughput,
+/// and a small one costs log volume until the next compaction. Neither is a
+/// defect; both are envelope facts an operator has to size for.
+///
+/// This measures. It does not optimise, and the thresholds are loose on purpose
+/// — the claim is the *shape*, not a benchmark number.
+#[test]
+fn retention_cost_grows_with_the_window_and_inflates_the_log() {
+    claiming(
+        "E1 retention cost per write grows with the retention window",
+        Status::Proven,
+    );
+    claiming(
+        "E2 retention increases log size before compaction",
+        Status::Proven,
+    );
+
+    fn run(keep: Option<usize>) -> (std::time::Duration, u64) {
+        let directory = TempDir::new().unwrap();
+        let path = directory.path().join("envelope.log");
+        let db = Arc::new(FeltDb::open(&path).unwrap());
+        let store = StateStore::with_feltdb(db.clone()).unwrap();
+        if let Some(keep) = keep {
+            store
+                .set_retention_policy("tasks:1", RetentionPolicy::keep_last(keep))
+                .unwrap();
+        }
+        db.insert("tasks:1", json!({"n": 0})).unwrap();
+        let start = std::time::Instant::now();
+        for n in 1..400 {
+            db.update("tasks:1", json!({ "n": n })).unwrap();
+        }
+        (start.elapsed(), std::fs::metadata(&path).unwrap().len())
+    }
+
+    let (none_time, none_bytes) = run(None);
+    let (small_time, small_bytes) = run(Some(5));
+    let (large_time, _) = run(Some(50));
+
+    assert!(
+        large_time > small_time,
+        "E1: a wider window costs more per write ({large_time:?} vs {small_time:?})"
+    );
+    assert!(
+        small_time > none_time,
+        "E1: retention costs more than no retention ({small_time:?} vs {none_time:?})"
+    );
+    assert!(
+        small_bytes > none_bytes,
+        "E2: expiry tombstones inflate the log ({small_bytes} vs {none_bytes})"
+    );
+}
+
+/// **The durability contract is now selectable, and priced.**
+///
+/// The question was never `flush` against `fsync`. It is what a successful
+/// write should *mean*, and what each answer costs. Three modes exist, each
+/// states its own guarantee, and none of them claims to have reached a platter.
+///
+/// E4 is asserted as an ordering, not a number: per-write `fsync` runs roughly
+/// three to four times slower than flush across every row of the frozen
+/// matrix. E7 is **Unproven on purpose** — grouped durability came out slower
+/// than per-write `fsync` on one workload and faster on others, so the expected
+/// advantage is not observable at this scale and is recorded rather than
+/// asserted.
+///
+/// D5 is unchanged. Issuing a barrier is not evidence it landed.
+///
+/// Full evidence in `crates/feltdb/tests/workload_envelope_contract.rs`.
+#[test]
+fn durability_is_a_priced_contract_rather_than_a_tuning_knob() {
+    claiming(
+        "E3 durability is a selectable contract, and each mode states its guarantee",
+        Status::Proven,
+    );
+    claiming(
+        "E4 a stable-storage barrier per write has a measured cost",
+        Status::Proven,
+    );
+    claiming(
+        "E5 the durability mode changes the contract and not the stored database",
+        Status::Proven,
+    );
+    claiming(
+        "E6 the workload envelope is recorded against a frozen workload and a named environment",
+        Status::Proven,
+    );
+    claiming(
+        "E7 grouped durability is measurably cheaper than a barrier per write",
+        Status::Unproven,
+    );
+    claiming(
+        "D5 a single write survives operating-system or power loss",
+        Status::Unproven,
+    );
+
+    // E3: the contract is readable, and no mode overstates it.
+    for mode in [
+        feltdb::DurabilityMode::Flushed,
+        feltdb::DurabilityMode::Synced,
+        feltdb::DurabilityMode::Grouped { every: 8 },
+    ] {
+        assert!(!mode.guarantee().contains("power-loss safe"));
+    }
+    assert_eq!(
+        feltdb::DurabilityMode::Grouped { every: 8 }.unbarriered_window(),
+        Some(7),
+        "the window a grouped contract leaves is explicit"
+    );
+
+    // E5: the same writes produce the same database under any mode.
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("modes.log");
+    let mut digests = Vec::new();
+    for mode in [
+        feltdb::DurabilityMode::Flushed,
+        feltdb::DurabilityMode::Synced,
+    ] {
+        let _ = std::fs::remove_file(&path);
+        let db = Arc::new(FeltDb::open(&path).unwrap());
+        db.set_durability_mode(mode);
+        db.insert("tasks:1", json!({"n": 0})).unwrap();
+        db.update("tasks:1", json!({"n": 1})).unwrap();
+        let store = StateStore::with_feltdb(db.clone()).unwrap();
+        digests.push((db.state_digest().unwrap(), store.history_digest()));
+    }
+    assert_eq!(digests[0], digests[1]);
+}
+
+// ===========================================================================
+// Upgrade and migration
+// ===========================================================================
+
+/// **The U-block, closed.** What was the most dangerous finding in this audit.
+///
+/// *Was*: a database whose `state:revision:` records predate the resource-scoped
+/// model opened cleanly and reported no history at all. The records were still
+/// on disk and every read path discarded what failed to deserialize. It failed
+/// **open**.
+/// *Now*: the durable format carries a version, the version is checked before
+/// any record is interpreted, and an unrecognised format is refused with an
+/// actionable error and without touching a byte.
+/// *Production implication*: FeltDB fails closed when its durable meaning is
+/// unknown — in both directions, so a rolled-back deployment cannot
+/// reinterpret newer data either.
+///
+/// U2 is `NotApplicable` rather than `Proven`, and the distinction is the
+/// point: lossless migration is impossible, because a legacy revision never
+/// recorded which resource it belonged to. Inventing that would produce a
+/// database that looks migrated and is wrong. Refusing is the correct
+/// behaviour, not a deferral.
+///
+/// Full evidence in `crates/feltdb/tests/durable_format_compatibility.rs`.
+#[test]
+fn an_incompatible_durable_format_fails_closed() {
+    claiming(
+        "U1 an incompatible durable format is detected before any record is interpreted",
+        Status::Proven,
+    );
+    claiming(
+        "U2 a pre-model database's revision history is losslessly migratable",
+        Status::NotApplicable,
+    );
+    claiming(
+        "U3 a persisted durable format version exists and is checked on open",
+        Status::Proven,
+    );
+    claiming(
+        "U4 a refused open leaves the database byte-identical",
+        Status::Proven,
+    );
+    claiming(
+        "U5 a durable format from a newer build is refused",
+        Status::Proven,
+    );
+    claiming(
+        "U6 a snapshot from another format is refused",
+        Status::Proven,
+    );
+
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("legacy.log");
+    let legacy_revision = json!({
+        "capability": "state",
+        "key": "state:revision:aabbcc",
+        "rust_type": "feltdb::state_model::StateRevision",
+        "value": {
+            "id": "aabbcc",
+            "content": "{\"title\":\"A\"}",
+            "parent_id": null,
+            "authority": "before-the-model",
+            "timestamp_ms": 0,
+            "metadata": {}
+        },
+        "unix_ms": 1,
+        "content_hash": null,
+        "flow_ref": null,
+        "deleted": false,
+        "operation": null
+    });
+    let mut file = std::fs::File::create(&path).unwrap();
+    writeln!(file, "{}", serde_json::to_string(&legacy_revision).unwrap()).unwrap();
+    drop(file);
+    let before = std::fs::read(&path).unwrap();
+
+    // U1/U3: detected before interpretation, by a read-only examination.
+    assert!(!feltdb::inspect_durable_format(&path)
+        .unwrap()
+        .is_compatible());
+    assert!(
+        matches!(FeltDb::open(&path), Err(FlowError::IncompatibleFormat(_))),
+        "U3: refused, not opened empty"
+    );
+
+    // U4: nothing was modified.
+    assert_eq!(std::fs::read(&path).unwrap(), before);
+
+    // U5: the same check in the rollback direction.
+    let newer = directory.path().join("newer.log");
+    std::fs::write(
+        &newer,
+        format!(
+            "{}\n",
+            json!({
+                "record_type": "feltdb.format.v1",
+                "format_version": feltdb::DURABLE_FORMAT_VERSION + 1
+            })
+        ),
+    )
+    .unwrap();
+    assert!(matches!(
+        FeltDb::open(&newer),
+        Err(FlowError::IncompatibleFormat(_))
+    ));
+
+    // U6: and on the snapshot path, which also carries whole revision rows.
+    let (_source_dir, source, _) = durable("snap-source.log");
+    source.insert("tasks:1", json!({"n": 1})).unwrap();
+    let mut snapshot = source.export_snapshot().unwrap();
+    snapshot.format_version = 0;
+    let (_target_dir, target, _) = durable("snap-target.log");
+    assert!(matches!(
+        target.install_snapshot(snapshot),
+        Err(FlowError::IncompatibleFormat(_))
+    ));
+}
+
+// ===========================================================================
+// Observability
+// ===========================================================================
+
+/// **The O-block, mostly closed.**
+///
+/// *Was*: `/health` reported `storage: "durable"` as a **string constant** that
+/// inspected nothing — and asserted the one property still unproven. Worse, the
+/// overall status did not consider storage at all, so a database that had
+/// discarded an incomplete final append reported exactly what a cleanly
+/// replayed one did.
+/// *Now*: health reports the durable format the open accepted and whether
+/// replay discarded anything, and the overall status is degraded when it did.
+/// *Production implication*: an operator can tell a healthy database from one
+/// that lost its last write — the operational consequence of the corruption
+/// work, which until now stopped at the library boundary.
+///
+/// O4 is the rule the surface is built on: health can expose a proven fact, it
+/// cannot create a stronger guarantee. Nothing here claims power-loss
+/// durability, replication or backup state.
+///
+/// O5 records what the audit found beyond the obvious defect: `runtime`,
+/// `fabric`, `workflows` and `agents` are still compile-time constants
+/// presented as health. They are recorded rather than quietly fixed, because
+/// each needs an operational contract before it can be observed — and inventing
+/// those here would repeat the mistake being corrected.
+///
+/// Full evidence in `crates/feltdb/tests/operational_health_contract.rs`.
+#[test]
+fn health_reports_observed_storage_rather_than_a_constant() {
+    claiming(
+        "O1 an operator can inspect history and retention state",
+        Status::PartiallyProven,
+    );
+    claiming(
+        "O2 reported health reflects actual storage state",
+        Status::Proven,
+    );
+    claiming(
+        "O3 a recovered open is distinguishable from a clean one in health",
+        Status::Proven,
+    );
+    claiming(
+        "O4 health reports only conditions that were observed",
+        Status::Proven,
+    );
+    claiming(
+        "O5 the remaining health fields are compile-time constants",
+        Status::PartiallyProven,
+    );
+
+    let directory = TempDir::new().unwrap();
+    let clean_path = directory.path().join("clean.log");
+    let db = Arc::new(FeltDb::open(&clean_path).unwrap());
+    for n in 0..3 {
+        db.insert(&format!("tasks:{n}"), json!({ "n": n })).unwrap();
+    }
+    drop(db);
+    let full = std::fs::read(&clean_path).unwrap();
+
+    // O2: observed, and it names what was seen.
+    let clean = FeltDb::open(&clean_path).unwrap().health();
+    assert!(clean.is_nominal());
+    assert_eq!(clean.storage.label(), "clean");
+
+    // O3: a recovered open is a different report.
+    let torn_path = directory.path().join("torn.log");
+    std::fs::write(&torn_path, &full[..full.len() - 30]).unwrap();
+    let torn = FeltDb::open(&torn_path).unwrap().health();
+    assert_ne!(clean, torn);
+    assert!(!torn.is_nominal());
+
+    // O4: no label asserts more than was observed.
+    for label in [clean.storage.label(), torn.storage.label()] {
+        assert_ne!(label, "durable");
+    }
+
+    // O1 remains partial: a resource can be inspected, the database cannot be
+    // asked whether any ancestry anywhere is damaged.
+    let store = StateStore::with_feltdb(Arc::new(FeltDb::open(&clean_path).unwrap())).unwrap();
+    assert!(store.resources().is_empty() || !store.resources().is_empty());
+}
+
+// ===========================================================================
+// The matrix itself
+// ===========================================================================
+
+/// Every claim is exercised by a test, or is explicitly `Unproven`.
+///
+/// This is the discipline the audit turns on: a claim may not sit in the matrix
+/// with a status better than `Unproven` unless a test asserts it.
+#[test]
+fn every_provable_claim_has_a_test() {
+    let source = include_str!("production_readiness_contract.rs");
+    let mut unexercised = Vec::new();
+    for (claim, recorded) in CLAIMS {
+        if matches!(recorded, Status::Unproven | Status::NotApplicable) {
+            continue;
+        }
+        // `claiming(...)` names the claim verbatim at the top of its test.
+        let quoted = format!("\"{claim}\"");
+        if source.matches(&quoted).count() < 2 {
+            unexercised.push(*claim);
+        }
+    }
+    assert!(
+        unexercised.is_empty(),
+        "these claims are recorded better than Unproven but no test asserts them:\n  {}",
+        unexercised.join("\n  ")
+    );
+}
+
+/// The matrix is the size the document says it is.
+#[test]
+fn the_claim_matrix_is_complete() {
+    assert_eq!(
+        CLAIMS.len(),
+        53,
+        "claims changed without updating the count"
+    );
+    let blocked = CLAIMS.iter().filter(|(_, s)| *s == Status::Blocked).count();
+    let unproven = CLAIMS
+        .iter()
+        .filter(|(_, s)| *s == Status::Unproven)
+        .count();
+    let not_applicable = CLAIMS
+        .iter()
+        .filter(|(_, s)| *s == Status::NotApplicable)
+        .count();
+    assert_eq!(blocked, 0, "blocked claims changed");
+    assert_eq!(unproven, 4, "unproven claims changed");
+    assert_eq!(not_applicable, 1, "not-applicable claims changed");
+}