@feltdb/core 0.8.3 → 0.8.5

package/dist/create/server-source/crates/feltdb/tests/production_readiness_contract.rs

@@ -0,0 +1,1032 @@
+//! The production readiness contract.
+//!
+//! This file **implements nothing and fixes nothing**. It makes the claims
+//! FeltDB can and cannot make executable, so that "production ready" stops
+//! being a feeling and becomes a matrix with evidence behind each row.
+//!
+//! It is the executable half of
+//! `docs/architecture/production-readiness.md`, and its failure cases are
+//! catalogued in `docs/architecture/production-failure-matrix.md`.
+//!
+//! # The discipline
+//!
+//! Every claim carries a [`Status`]. **A green test never implies a claim
+//! broader than the scenario it runs.** Where a production claim is larger than
+//! what any test exercises, the status is `PartiallyProven` and the test says
+//! what is missing. Where nothing can be tested, the status is `Unproven` and
+//! there is no test pretending otherwise.
+//!
+//! Several tests here assert a **gap** rather than a guarantee — that a
+//! pre-model database opens with its history invisible, that a truncated log is
+//! accepted silently. They are written that way on purpose: the gap becomes a
+//! regression test, and closing it fails this suite and forces the matrix to be
+//! updated deliberately.
+
+use feltdb::state_model::{ParentLookup, RetentionPolicy, StateStore};
+use feltdb::{FeltDb, FlowError, JsonCasResult};
+use serde_json::json;
+use std::collections::HashMap;
+use std::io::Write;
+use std::sync::Arc;
+use tempfile::TempDir;
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+enum Status {
+    /// Executable evidence exists for the claim as stated.
+    Proven,
+    /// Bounded scenarios work; the production claim is larger than the evidence.
+    PartiallyProven,
+    /// No sufficient evidence. Not "probably works".
+    Unproven,
+    /// A known defect prevents the claim.
+    Blocked,
+    /// Explicitly outside what the product can offer, with a reason in the data
+    /// rather than in the schedule.
+    NotApplicable,
+}
+
+/// The claim matrix. `docs/architecture/production-readiness.md` is its prose.
+const CLAIMS: &[(&str, Status)] = &[
+    // Durability
+    (
+        "D1 committed mutations survive a clean restart",
+        Status::Proven,
+    ),
+    (
+        "D2 revision history survives a clean restart",
+        Status::Proven,
+    ),
+    (
+        "D3 retention policy and horizon survive a restart",
+        Status::Proven,
+    ),
+    (
+        "D4 a reopened database continues history rather than restarting it",
+        Status::Proven,
+    ),
+    (
+        "D5 a single write survives operating-system or power loss",
+        Status::Unproven,
+    ),
+    // Atomicity and concurrency
+    (
+        "A1 a stale fenced write is rejected and mints nothing",
+        Status::Proven,
+    ),
+    (
+        "A2 concurrent writes to distinct resources keep distinct chains",
+        Status::Proven,
+    ),
+    (
+        "A3 concurrent writes to one resource form one correct chain",
+        Status::Proven,
+    ),
+    (
+        "A4 a duplicate remote operation is idempotent",
+        Status::Proven,
+    ),
+    (
+        "A5 a remote operation with a sequence gap is rejected",
+        Status::Proven,
+    ),
+    (
+        "A6 a crash part-way through a mutation leaves consistent state",
+        Status::Unproven,
+    ),
+    // Recovery
+    ("R1 a truncated log is detected", Status::Blocked),
+    ("R2 a malformed record is detected", Status::Blocked),
+    (
+        "R3 damaged ancestry is distinguishable from expired ancestry",
+        Status::Proven,
+    ),
+    (
+        "R4 damage is surfaced without being asked for",
+        Status::Unproven,
+    ),
+    // History and retention
+    (
+        "H1 history is bounded under a configured policy",
+        Status::Proven,
+    ),
+    (
+        "H2 history is unbounded without one, by design",
+        Status::Proven,
+    ),
+    (
+        "H3 a policy change takes effect in both directions",
+        Status::Proven,
+    ),
+    ("H4 maintenance never rewrites history", Status::Proven),
+    ("H5 forked history is preserved", Status::Proven),
+    // Backup and restore
+    (
+        "B1 an in-process snapshot round-trips state and history",
+        Status::PartiallyProven,
+    ),
+    (
+        "B2 an operator-facing backup, restore and verify workflow exists",
+        Status::Unproven,
+    ),
+    // Replication
+    ("S1 current state replicates to a peer", Status::Proven),
+    ("S2 revision history replicates to a peer", Status::Blocked),
+    // Workload envelope
+    (
+        "E1 retention cost per write grows with the retention window",
+        Status::Proven,
+    ),
+    (
+        "E2 retention increases log size before compaction",
+        Status::Proven,
+    ),
+    // Upgrade and migration
+    (
+        "U1 an incompatible durable format is detected before any record is interpreted",
+        Status::Proven,
+    ),
+    (
+        "U2 a pre-model database's revision history is losslessly migratable",
+        Status::NotApplicable,
+    ),
+    (
+        "U3 a persisted durable format version exists and is checked on open",
+        Status::Proven,
+    ),
+    (
+        "U4 a refused open leaves the database byte-identical",
+        Status::Proven,
+    ),
+    (
+        "U5 a durable format from a newer build is refused",
+        Status::Proven,
+    ),
+    (
+        "U6 a snapshot from another format is refused",
+        Status::Proven,
+    ),
+    // Observability
+    (
+        "O1 an operator can inspect history and retention state",
+        Status::PartiallyProven,
+    ),
+    (
+        "O2 reported health reflects actual storage state",
+        Status::Blocked,
+    ),
+];
+
+fn status(claim: &str) -> Status {
+    CLAIMS
+        .iter()
+        .find(|(name, _)| *name == claim)
+        .unwrap_or_else(|| panic!("{claim} is not in the claim matrix"))
+        .1
+}
+
+/// Assert a claim's recorded status, so a test cannot quietly widen its claim.
+fn claiming(claim: &str, expected: Status) {
+    assert_eq!(
+        status(claim),
+        expected,
+        "{claim}: this test exercises a {expected:?} claim, but the matrix records \
+         something else. Update docs/architecture/production-readiness.md deliberately."
+    );
+}
+
+fn durable(name: &str) -> (TempDir, Arc<FeltDb>, StateStore) {
+    let directory = TempDir::new().unwrap();
+    let db = Arc::new(FeltDb::open(directory.path().join(name)).unwrap());
+    let store = StateStore::with_feltdb(db.clone()).unwrap();
+    (directory, db, store)
+}
+
+// ===========================================================================
+// Durability
+// ===========================================================================
+
+/// **Claim D1–D4.** A clean restart preserves state, history, retention and
+/// position.
+///
+/// *Scenario*: write, mint under a policy, close, reopen, write again.
+/// *Expected*: everything survives and history continues.
+/// *Observed*: it does.
+/// *Production implication*: a planned restart is safe. This says nothing about
+/// an unplanned one — see D5.
+#[test]
+fn a_clean_restart_preserves_state_history_and_retention() {
+    claiming(
+        "D1 committed mutations survive a clean restart",
+        Status::Proven,
+    );
+    claiming(
+        "D2 revision history survives a clean restart",
+        Status::Proven,
+    );
+    claiming(
+        "D3 retention policy and horizon survive a restart",
+        Status::Proven,
+    );
+    claiming(
+        "D4 a reopened database continues history rather than restarting it",
+        Status::Proven,
+    );
+
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("restart.log");
+    let db = Arc::new(FeltDb::open(&path).unwrap());
+    let store = StateStore::with_feltdb(db.clone()).unwrap();
+    store
+        .set_retention_policy("tasks:1", RetentionPolicy::keep_last(3))
+        .unwrap();
+    db.insert("tasks:1", json!({"n": 0})).unwrap();
+    for n in 1..10 {
+        db.update("tasks:1", json!({ "n": n })).unwrap();
+    }
+    let horizon_before = store.retention_horizon("tasks:1");
+    drop(store);
+    drop(db);
+
+    let reopened = Arc::new(FeltDb::open(&path).unwrap());
+    let store = StateStore::with_feltdb(reopened.clone()).unwrap();
+
+    let current: serde_json::Value = reopened.get("tasks:1").unwrap().unwrap();
+    assert_eq!(current["n"], json!(9), "D1");
+    assert_eq!(store.history_of("tasks:1").len(), 3, "D2");
+    assert_eq!(
+        store.retention_policy("tasks:1"),
+        RetentionPolicy::keep_last(3),
+        "D3 policy"
+    );
+    assert_eq!(
+        store.retention_horizon("tasks:1"),
+        horizon_before,
+        "D3 horizon"
+    );
+
+    reopened.update("tasks:1", json!({"n": 10})).unwrap();
+    let history = store.history_of("tasks:1");
+    assert_eq!(history.last().unwrap().sequence, 10, "D4");
+}
+
+/// **Claim D5 is Unproven, and the reason is in the code rather than a guess.**
+///
+/// `append_event` writes a record and flushes it to the operating system. It
+/// does **not** fsync; only `append_transaction` does. So a single write
+/// survives the process dying, and its survival across an operating-system or
+/// power failure is not established by anything here.
+///
+/// Automatic revision minting uses the same path, so a revision inherits
+/// exactly this durability and no more.
+///
+/// This test asserts only what it can: that the process-death case holds. It
+/// deliberately does not assert the power-loss case, and the matrix says so.
+#[test]
+fn a_single_write_is_flushed_but_its_power_loss_behaviour_is_unproven() {
+    claiming(
+        "D5 a single write survives operating-system or power loss",
+        Status::Unproven,
+    );
+
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("flush.log");
+    let db = FeltDb::open(&path).unwrap();
+    db.insert("tasks:1", json!({"n": 1})).unwrap();
+    // Read the file behind the database's back: the record is already there.
+    let log = std::fs::read_to_string(&path).unwrap();
+    assert!(
+        log.contains("tasks:1"),
+        "the record reached the file before any close"
+    );
+}
+
+// ===========================================================================
+// Atomicity and concurrency
+// ===========================================================================
+
+/// **Claim A1.** A refused fence changes nothing — not the row, not the history.
+#[test]
+fn a_stale_fenced_write_is_rejected_and_mints_nothing() {
+    claiming(
+        "A1 a stale fenced write is rejected and mints nothing",
+        Status::Proven,
+    );
+    let (_directory, db, store) = durable("cas.log");
+    db.insert("tasks:1", json!({"title": "A", "__version": 1}))
+        .unwrap();
+    let before = store.history_of("tasks:1").len();
+
+    let refused = db
+        .compare_and_set_json("tasks:1", 99, None, None, false, json!({"title": "Z"}))
+        .unwrap();
+    assert!(matches!(refused, JsonCasResult::VersionConflict { .. }));
+
+    assert_eq!(
+        store.history_of("tasks:1").len(),
+        before,
+        "no revision minted"
+    );
+    let current: serde_json::Value = db.get("tasks:1").unwrap().unwrap();
+    assert_eq!(current["title"], json!("A"), "the row is unchanged");
+}
+
+/// **Claims A2 and A3.** Concurrency produces correct chains, on separate
+/// resources and on one shared resource.
+///
+/// *Production implication*: the write path serializes minting correctly. It
+/// says nothing about throughput under contention — see E1.
+#[test]
+fn concurrent_writes_produce_correct_chains() {
+    claiming(
+        "A2 concurrent writes to distinct resources keep distinct chains",
+        Status::Proven,
+    );
+    claiming(
+        "A3 concurrent writes to one resource form one correct chain",
+        Status::Proven,
+    );
+
+    let directory = TempDir::new().unwrap();
+    let db = Arc::new(FeltDb::open(directory.path().join("concurrent.log")).unwrap());
+
+    let mut handles = Vec::new();
+    for thread_index in 0..4 {
+        let db = db.clone();
+        handles.push(std::thread::spawn(move || {
+            for n in 0..25 {
+                db.update(&format!("tasks:{thread_index}"), json!({ "n": n }))
+                    .unwrap();
+                db.update("shared:1", json!({ "t": thread_index, "n": n }))
+                    .unwrap();
+            }
+        }));
+    }
+    for handle in handles {
+        handle.join().unwrap();
+    }
+    let store = StateStore::with_feltdb(db.clone()).unwrap();
+
+    for thread_index in 0..4 {
+        let history = store.history_of(&format!("tasks:{thread_index}"));
+        assert_eq!(history.len(), 25, "A2 length");
+        assert!(
+            history
+                .windows(2)
+                .all(|pair| pair[1].parent_id.as_ref() == Some(&pair[0].id)),
+            "A2 chained"
+        );
+    }
+
+    let shared = store.history_of("shared:1");
+    assert_eq!(shared.len(), 100, "A3 every write minted exactly once");
+    assert!(
+        shared
+            .windows(2)
+            .all(|pair| pair[1].parent_id.as_ref() == Some(&pair[0].id)),
+        "A3 one chain, not four"
+    );
+    let mut sequences: Vec<u64> = shared.iter().map(|revision| revision.sequence).collect();
+    sequences.dedup();
+    assert_eq!(sequences.len(), 100, "A3 sequences are unique");
+}
+
+/// **Claims A4 and A5.** Replayed and out-of-order remote operations.
+#[test]
+fn replayed_and_gapped_remote_operations_are_handled_explicitly() {
+    claiming(
+        "A4 a duplicate remote operation is idempotent",
+        Status::Proven,
+    );
+    claiming(
+        "A5 a remote operation with a sequence gap is rejected",
+        Status::Proven,
+    );
+
+    let directory = TempDir::new().unwrap();
+    let primary = Arc::new(FeltDb::open(directory.path().join("p.log")).unwrap());
+    let replica = Arc::new(FeltDb::open(directory.path().join("r.log")).unwrap());
+
+    primary.insert("tasks:1", json!({"n": 1})).unwrap();
+    primary.update("tasks:1", json!({"n": 2})).unwrap();
+    let operations = primary.operations_since(&HashMap::new()).unwrap();
+    assert!(operations.len() >= 2);
+
+    // Applying out of order is refused rather than silently accepted.
+    assert!(
+        replica
+            .apply_remote_operation(operations[1].clone())
+            .is_err(),
+        "A5"
+    );
+
+    for operation in &operations {
+        replica.apply_remote_operation(operation.clone()).unwrap();
+    }
+    let after_first_pass: serde_json::Value = replica.get("tasks:1").unwrap().unwrap();
+
+    // Replaying the whole stream is a no-op.
+    for operation in &operations {
+        replica.apply_remote_operation(operation.clone()).unwrap();
+    }
+    let after_replay: serde_json::Value = replica.get("tasks:1").unwrap().unwrap();
+    assert_eq!(after_first_pass, after_replay, "A4");
+}
+
+// ===========================================================================
+// Recovery and corruption
+// ===========================================================================
+
+/// **Claims R1 and R2 are Blocked, and this test records the defect.**
+///
+/// *Scenario*: truncate the log mid-record; append a line that is not JSON.
+/// *Expected of a database*: the damage is reported.
+/// *Observed*: both open successfully. The truncated tail record is dropped and
+/// the malformed line is skipped, with no error, no warning and no way for a
+/// caller to learn it happened.
+/// *Production implication*: **an operator cannot distinguish a healthy
+/// database from one that has silently lost its most recent writes.** This is
+/// the single largest recovery gap.
+///
+/// The assertions below encode the current behaviour, so closing the gap fails
+/// this test and forces the matrix to be updated.
+#[test]
+fn a_damaged_log_is_accepted_silently() {
+    claiming("R1 a truncated log is detected", Status::Blocked);
+    claiming("R2 a malformed record is detected", Status::Blocked);
+
+    // Truncation.
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("truncated.log");
+    let db = FeltDb::open(&path).unwrap();
+    for n in 0..5 {
+        db.insert(&format!("tasks:{n}"), json!({ "n": n })).unwrap();
+    }
+    drop(db);
+    let bytes = std::fs::read(&path).unwrap();
+    std::fs::write(&path, &bytes[..bytes.len() - 600]).unwrap();
+
+    let reopened = FeltDb::open(&path).expect("truncation does not fail the open");
+    let surviving = (0..5)
+        .filter(|n| {
+            reopened
+                .get::<serde_json::Value>(&format!("tasks:{n}"))
+                .unwrap()
+                .is_some()
+        })
+        .count();
+    assert!(
+        surviving < 5,
+        "records were lost — and nothing said so ({surviving}/5 present)"
+    );
+
+    // A line that is not JSON at all.
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("garbage.log");
+    let db = FeltDb::open(&path).unwrap();
+    db.insert("tasks:1", json!({"n": 1})).unwrap();
+    drop(db);
+    let mut file = std::fs::OpenOptions::new()
+        .append(true)
+        .open(&path)
+        .unwrap();
+    writeln!(file, "this is not json at all").unwrap();
+    drop(file);
+
+    let reopened = FeltDb::open(&path).expect("a malformed record does not fail the open");
+    assert!(
+        reopened
+            .get::<serde_json::Value>("tasks:1")
+            .unwrap()
+            .is_some(),
+        "earlier records are fine; the corrupt one simply vanished"
+    );
+}
+
+/// **Claim R3.** Damaged ancestry and expired ancestry are different facts, and
+/// the store says which.
+///
+/// This is the one corruption case FeltDB genuinely detects, and it exists
+/// because retention needed the distinction. A parent absent at or below the
+/// horizon was expired on purpose; absent above it is damage.
+///
+/// *Production implication*: ancestry damage is detectable — **on inspection**.
+/// Nothing surfaces it unprompted, which is claim R4.
+#[test]
+fn damaged_ancestry_is_distinguishable_from_expired_ancestry() {
+    claiming(
+        "R3 damaged ancestry is distinguishable from expired ancestry",
+        Status::Proven,
+    );
+    claiming(
+        "R4 damage is surfaced without being asked for",
+        Status::Unproven,
+    );
+
+    // Damage: a mid-history revision removed with no retention involved.
+    let (_directory, db, store) = durable("damaged.log");
+    db.insert("tasks:1", json!({"n": 0})).unwrap();
+    db.update("tasks:1", json!({"n": 1})).unwrap();
+    db.update("tasks:1", json!({"n": 2})).unwrap();
+    let history = store.history_of("tasks:1");
+    let middle = history[1].id.clone();
+    let child = history[2].id.clone();
+    db.delete(&format!("state:revision:{}", middle.as_hex()))
+        .unwrap();
+
+    assert!(
+        matches!(store.parent_of(&child), Some(ParentLookup::Missing(_))),
+        "R3: reported as damage"
+    );
+    assert_eq!(
+        store.retention_horizon("tasks:1"),
+        0,
+        "no retention happened"
+    );
+
+    // Expiry: the same shape, but a decision.
+    let (_directory, db, store) = durable("expired.log");
+    store
+        .set_retention_policy("tasks:1", RetentionPolicy::keep_last(2))
+        .unwrap();
+    db.insert("tasks:1", json!({"n": 0})).unwrap();
+    for n in 1..6 {
+        db.update("tasks:1", json!({ "n": n })).unwrap();
+    }
+    let oldest = store.history_of("tasks:1")[0].id.clone();
+    assert!(
+        matches!(store.parent_of(&oldest), Some(ParentLookup::Expired(_))),
+        "R3: reported as a decision"
+    );
+}
+
+// ===========================================================================
+// History and retention
+// ===========================================================================
+
+/// **Claims H1, H2, H3.** The retention envelope, in both directions.
+#[test]
+fn the_retention_envelope_behaves_in_both_directions() {
+    claiming(
+        "H1 history is bounded under a configured policy",
+        Status::Proven,
+    );
+    claiming(
+        "H2 history is unbounded without one, by design",
+        Status::Proven,
+    );
+    claiming(
+        "H3 a policy change takes effect in both directions",
+        Status::Proven,
+    );
+
+    // Unbounded by default.
+    let (_directory, db, store) = durable("unbounded.log");
+    db.insert("tasks:1", json!({"n": 0})).unwrap();
+    for n in 1..50 {
+        db.update("tasks:1", json!({ "n": n })).unwrap();
+    }
+    assert_eq!(store.history_of("tasks:1").len(), 50, "H2");
+
+    // Tightening expires immediately.
+    store
+        .set_retention_policy("tasks:1", RetentionPolicy::keep_last(3))
+        .unwrap();
+    assert_eq!(store.history_of("tasks:1").len(), 3, "H3 tighten");
+
+    // And stays bounded as writes continue.
+    for n in 50..100 {
+        db.update("tasks:1", json!({ "n": n })).unwrap();
+    }
+    assert_eq!(store.history_of("tasks:1").len(), 3, "H1");
+
+    // Loosening lets it grow again — it does not restore what was expired.
+    store
+        .set_retention_policy("tasks:1", RetentionPolicy::unbounded())
+        .unwrap();
+    for n in 100..110 {
+        db.update("tasks:1", json!({ "n": n })).unwrap();
+    }
+    assert_eq!(store.history_of("tasks:1").len(), 13, "H3 loosen");
+}
+
+/// **Claims H4 and H5.** Maintenance never rewrites history, and forks survive.
+#[test]
+fn maintenance_never_rewrites_history_and_forks_survive() {
+    claiming("H4 maintenance never rewrites history", Status::Proven);
+    claiming("H5 forked history is preserved", Status::Proven);
+
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("forks.log");
+    let db = Arc::new(FeltDb::open(&path).unwrap());
+    let store = StateStore::with_feltdb(db.clone()).unwrap();
+
+    let base = store
+        .create("docs:1", r#"{"v":0}"#.into(), "a".into())
+        .unwrap();
+    let left = store
+        .commit(r#"{"v":1}"#.into(), &base, "a".into())
+        .unwrap();
+    let right = store
+        .commit(r#"{"v":2}"#.into(), &base, "b".into())
+        .unwrap();
+    assert_ne!(left.id, right.id, "H5: two heads from one base");
+    assert_eq!(left.parent_id, right.parent_id);
+
+    // Compaction, an unrelated write, and a retention pass on another resource.
+    db.insert("tasks:1", json!({"unrelated": true})).unwrap();
+    let versions = db.operation_versions().unwrap();
+    db.acknowledge_peer_versions("peer-1".to_string(), versions)
+        .unwrap();
+    db.compact_operation_log(&["peer-1".to_string()]).unwrap();
+    store
+        .set_retention_policy("tasks:1", RetentionPolicy::keep_last(1))
+        .unwrap();
+
+    for revision in [&base, &left, &right] {
+        let stored = store.get(&revision.id).expect("H4: still there");
+        assert_eq!(stored.parent_id, revision.parent_id, "H4: ancestry intact");
+        assert_eq!(stored.content_id, revision.content_id);
+    }
+}
+
+// ===========================================================================
+// Backup and restore
+// ===========================================================================
+
+/// **Claim B1 is Partially Proven; B2 is Unproven.**
+///
+/// *Observed*: `export_snapshot` and `install_snapshot` do round-trip current
+/// state **and** revision history into an empty database.
+/// *What that is not*: a backup product. There is no operator command, no
+/// on-disk backup format, no integrity verification step a human can run, no
+/// incremental or scheduled backup, and no documented restore procedure. The
+/// snapshot is an in-process value used by replication.
+/// *Production implication*: **an operator has no supported way to back up or
+/// restore a FeltDB database today.** The mechanism a backup could be built on
+/// exists; the workflow does not.
+#[test]
+fn a_snapshot_round_trips_but_is_not_a_backup_workflow() {
+    claiming(
+        "B1 an in-process snapshot round-trips state and history",
+        Status::PartiallyProven,
+    );
+    claiming(
+        "B2 an operator-facing backup, restore and verify workflow exists",
+        Status::Unproven,
+    );
+
+    let (_source_dir, source, source_store) = durable("source.log");
+    source.insert("tasks:1", json!({"n": 0})).unwrap();
+    source.update("tasks:1", json!({"n": 1})).unwrap();
+    let history_before = source_store.history_of("tasks:1").len();
+    let snapshot = source.export_snapshot().unwrap();
+
+    let (_target_dir, target, target_store) = durable("target.log");
+    assert!(target.can_install_snapshot().unwrap());
+    target.install_snapshot(snapshot).unwrap();
+
+    let current: serde_json::Value = target.get("tasks:1").unwrap().unwrap();
+    assert_eq!(current["n"], json!(1), "B1 current state");
+    assert_eq!(
+        target_store.history_of("tasks:1").len(),
+        history_before,
+        "B1 revision history"
+    );
+}
+
+// ===========================================================================
+// Replication
+// ===========================================================================
+
+/// **Claim S2 is Blocked, and this is the open question #306 left.**
+///
+/// *Scenario*: a primary takes a write; the replica applies the operation.
+/// *Expected of the production claim*: the replica has the same history.
+/// *Observed*: the replica has the same **current state** and **no revision
+/// history at all**. `apply_remote_operation` writes rows directly and never
+/// reaches the minting path.
+/// *Production implication*: **revision history is a local artifact.** A
+/// failover to a replica loses all history; reconciliation on a replica has no
+/// `base`; and the two nodes disagree about what happened even though they
+/// agree about what is.
+///
+/// The revision model makes this closable — identity is a function of resource,
+/// content, parent and sequence, all of which the replica has — but nothing
+/// closes it today.
+#[test]
+fn a_replica_receives_state_but_no_history() {
+    claiming("S1 current state replicates to a peer", Status::Proven);
+    claiming("S2 revision history replicates to a peer", Status::Blocked);
+
+    let directory = TempDir::new().unwrap();
+    let primary = Arc::new(FeltDb::open(directory.path().join("primary.log")).unwrap());
+    let replica = Arc::new(FeltDb::open(directory.path().join("replica.log")).unwrap());
+
+    primary.insert("tasks:1", json!({"n": 1})).unwrap();
+    primary.update("tasks:1", json!({"n": 2})).unwrap();
+    for operation in primary.operations_since(&HashMap::new()).unwrap() {
+        replica.apply_remote_operation(operation).unwrap();
+    }
+
+    let current: serde_json::Value = replica.get("tasks:1").unwrap().unwrap();
+    assert_eq!(current["n"], json!(2), "S1: state arrived");
+
+    let primary_history = StateStore::with_feltdb(primary.clone())
+        .unwrap()
+        .history_of("tasks:1")
+        .len();
+    let replica_history = StateStore::with_feltdb(replica.clone())
+        .unwrap()
+        .history_of("tasks:1")
+        .len();
+    assert_eq!(primary_history, 2, "the primary recorded two revisions");
+    assert_eq!(replica_history, 0, "S2: the replica recorded none");
+}
+
+// ===========================================================================
+// Workload envelope
+// ===========================================================================
+
+/// **Claims E1 and E2.** Retention has a measurable cost, in two directions
+/// that pull against each other.
+///
+/// *Observed*: the per-write cost of retention grows with the retention window,
+/// because expiry walks the resource's retained history on every mint. And
+/// retention makes the log **larger** before compaction, because each expiry
+/// appends a tombstone.
+/// *Production implication*: a large retention window costs write throughput,
+/// and a small one costs log volume until the next compaction. Neither is a
+/// defect; both are envelope facts an operator has to size for.
+///
+/// This measures. It does not optimise, and the thresholds are loose on purpose
+/// — the claim is the *shape*, not a benchmark number.
+#[test]
+fn retention_cost_grows_with_the_window_and_inflates_the_log() {
+    claiming(
+        "E1 retention cost per write grows with the retention window",
+        Status::Proven,
+    );
+    claiming(
+        "E2 retention increases log size before compaction",
+        Status::Proven,
+    );
+
+    fn run(keep: Option<usize>) -> (std::time::Duration, u64) {
+        let directory = TempDir::new().unwrap();
+        let path = directory.path().join("envelope.log");
+        let db = Arc::new(FeltDb::open(&path).unwrap());
+        let store = StateStore::with_feltdb(db.clone()).unwrap();
+        if let Some(keep) = keep {
+            store
+                .set_retention_policy("tasks:1", RetentionPolicy::keep_last(keep))
+                .unwrap();
+        }
+        db.insert("tasks:1", json!({"n": 0})).unwrap();
+        let start = std::time::Instant::now();
+        for n in 1..400 {
+            db.update("tasks:1", json!({ "n": n })).unwrap();
+        }
+        (start.elapsed(), std::fs::metadata(&path).unwrap().len())
+    }
+
+    let (none_time, none_bytes) = run(None);
+    let (small_time, small_bytes) = run(Some(5));
+    let (large_time, _) = run(Some(50));
+
+    assert!(
+        large_time > small_time,
+        "E1: a wider window costs more per write ({large_time:?} vs {small_time:?})"
+    );
+    assert!(
+        small_time > none_time,
+        "E1: retention costs more than no retention ({small_time:?} vs {none_time:?})"
+    );
+    assert!(
+        small_bytes > none_bytes,
+        "E2: expiry tombstones inflate the log ({small_bytes} vs {none_bytes})"
+    );
+}
+
+// ===========================================================================
+// Upgrade and migration
+// ===========================================================================
+
+/// **The U-block, closed.** What was the most dangerous finding in this audit.
+///
+/// *Was*: a database whose `state:revision:` records predate the resource-scoped
+/// model opened cleanly and reported no history at all. The records were still
+/// on disk and every read path discarded what failed to deserialize. It failed
+/// **open**.
+/// *Now*: the durable format carries a version, the version is checked before
+/// any record is interpreted, and an unrecognised format is refused with an
+/// actionable error and without touching a byte.
+/// *Production implication*: FeltDB fails closed when its durable meaning is
+/// unknown — in both directions, so a rolled-back deployment cannot
+/// reinterpret newer data either.
+///
+/// U2 is `NotApplicable` rather than `Proven`, and the distinction is the
+/// point: lossless migration is impossible, because a legacy revision never
+/// recorded which resource it belonged to. Inventing that would produce a
+/// database that looks migrated and is wrong. Refusing is the correct
+/// behaviour, not a deferral.
+///
+/// Full evidence in `crates/feltdb/tests/durable_format_compatibility.rs`.
+#[test]
+fn an_incompatible_durable_format_fails_closed() {
+    claiming(
+        "U1 an incompatible durable format is detected before any record is interpreted",
+        Status::Proven,
+    );
+    claiming(
+        "U2 a pre-model database's revision history is losslessly migratable",
+        Status::NotApplicable,
+    );
+    claiming(
+        "U3 a persisted durable format version exists and is checked on open",
+        Status::Proven,
+    );
+    claiming(
+        "U4 a refused open leaves the database byte-identical",
+        Status::Proven,
+    );
+    claiming(
+        "U5 a durable format from a newer build is refused",
+        Status::Proven,
+    );
+    claiming(
+        "U6 a snapshot from another format is refused",
+        Status::Proven,
+    );
+
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("legacy.log");
+    let legacy_revision = json!({
+        "capability": "state",
+        "key": "state:revision:aabbcc",
+        "rust_type": "feltdb::state_model::StateRevision",
+        "value": {
+            "id": "aabbcc",
+            "content": "{\"title\":\"A\"}",
+            "parent_id": null,
+            "authority": "before-the-model",
+            "timestamp_ms": 0,
+            "metadata": {}
+        },
+        "unix_ms": 1,
+        "content_hash": null,
+        "flow_ref": null,
+        "deleted": false,
+        "operation": null
+    });
+    let mut file = std::fs::File::create(&path).unwrap();
+    writeln!(file, "{}", serde_json::to_string(&legacy_revision).unwrap()).unwrap();
+    drop(file);
+    let before = std::fs::read(&path).unwrap();
+
+    // U1/U3: detected before interpretation, by a read-only examination.
+    assert!(!feltdb::inspect_durable_format(&path)
+        .unwrap()
+        .is_compatible());
+    assert!(
+        matches!(FeltDb::open(&path), Err(FlowError::IncompatibleFormat(_))),
+        "U3: refused, not opened empty"
+    );
+
+    // U4: nothing was modified.
+    assert_eq!(std::fs::read(&path).unwrap(), before);
+
+    // U5: the same check in the rollback direction.
+    let newer = directory.path().join("newer.log");
+    std::fs::write(
+        &newer,
+        format!(
+            "{}\n",
+            json!({
+                "record_type": "feltdb.format.v1",
+                "format_version": feltdb::DURABLE_FORMAT_VERSION + 1
+            })
+        ),
+    )
+    .unwrap();
+    assert!(matches!(
+        FeltDb::open(&newer),
+        Err(FlowError::IncompatibleFormat(_))
+    ));
+
+    // U6: and on the snapshot path, which also carries whole revision rows.
+    let (_source_dir, source, _) = durable("snap-source.log");
+    source.insert("tasks:1", json!({"n": 1})).unwrap();
+    let mut snapshot = source.export_snapshot().unwrap();
+    snapshot.format_version = 0;
+    let (_target_dir, target, _) = durable("snap-target.log");
+    assert!(matches!(
+        target.install_snapshot(snapshot),
+        Err(FlowError::IncompatibleFormat(_))
+    ));
+}
+
+// ===========================================================================
+// Observability
+// ===========================================================================
+
+/// **Claim O1 is Partially Proven; O2 is Blocked.**
+///
+/// *Observed*: an operator with the library API can inspect a resource's
+/// history, its retention policy and horizon, and resolve any parent to
+/// `Root` / `Revision` / `Expired` / `Missing`. That is a real inspection
+/// surface.
+/// *What is missing*: nothing is aggregated. There is no count of resources
+/// under retention, no record of retention activity, no counter for rejected
+/// writes or conflicts, and no way to ask "is any ancestry damaged?" short of
+/// walking every revision.
+/// *And O2*: the server's `/health` reports `storage: "durable"` as a **string
+/// constant**. It does not inspect storage. A database with a truncated log and
+/// silently missing records reports itself healthy.
+/// *Production implication*: an operator can investigate a resource they already
+/// suspect, and cannot discover a problem they do not.
+#[test]
+fn an_operator_can_inspect_a_resource_but_not_the_database() {
+    claiming(
+        "O1 an operator can inspect history and retention state",
+        Status::PartiallyProven,
+    );
+    claiming(
+        "O2 reported health reflects actual storage state",
+        Status::Blocked,
+    );
+
+    let (_directory, db, store) = durable("observe.log");
+    store
+        .set_retention_policy("tasks:1", RetentionPolicy::keep_last(2))
+        .unwrap();
+    db.insert("tasks:1", json!({"n": 0})).unwrap();
+    for n in 1..6 {
+        db.update("tasks:1", json!({ "n": n })).unwrap();
+    }
+
+    // What an operator *can* see, per resource.
+    assert_eq!(store.resources(), vec!["tasks:1".to_string()]);
+    assert_eq!(store.history_of("tasks:1").len(), 2);
+    assert_eq!(
+        store.retention_policy("tasks:1"),
+        RetentionPolicy::keep_last(2)
+    );
+    assert!(store.retention_horizon("tasks:1") > 0);
+    let oldest = store.history_of("tasks:1")[0].id.clone();
+    assert!(store.parent_of(&oldest).is_some());
+
+    // What no API reports: how many revisions were expired, when, or whether
+    // any resource anywhere has damaged ancestry. Establishing either means
+    // walking every resource by hand, which is why O1 is only partly proven.
+}
+
+// ===========================================================================
+// The matrix itself
+// ===========================================================================
+
+/// Every claim is exercised by a test, or is explicitly `Unproven`.
+///
+/// This is the discipline the audit turns on: a claim may not sit in the matrix
+/// with a status better than `Unproven` unless a test asserts it.
+#[test]
+fn every_provable_claim_has_a_test() {
+    let source = include_str!("production_readiness_contract.rs");
+    let mut unexercised = Vec::new();
+    for (claim, recorded) in CLAIMS {
+        if matches!(recorded, Status::Unproven | Status::NotApplicable) {
+            continue;
+        }
+        // `claiming(...)` names the claim verbatim at the top of its test.
+        let quoted = format!("\"{claim}\"");
+        if source.matches(&quoted).count() < 2 {
+            unexercised.push(*claim);
+        }
+    }
+    assert!(
+        unexercised.is_empty(),
+        "these claims are recorded better than Unproven but no test asserts them:\n  {}",
+        unexercised.join("\n  ")
+    );
+}
+
+/// The matrix is the size the document says it is.
+#[test]
+fn the_claim_matrix_is_complete() {
+    assert_eq!(
+        CLAIMS.len(),
+        34,
+        "claims changed without updating the count"
+    );
+    let blocked = CLAIMS.iter().filter(|(_, s)| *s == Status::Blocked).count();
+    let unproven = CLAIMS
+        .iter()
+        .filter(|(_, s)| *s == Status::Unproven)
+        .count();
+    let not_applicable = CLAIMS
+        .iter()
+        .filter(|(_, s)| *s == Status::NotApplicable)
+        .count();
+    assert_eq!(blocked, 4, "blocked claims changed");
+    assert_eq!(unproven, 4, "unproven claims changed");
+    assert_eq!(not_applicable, 1, "not-applicable claims changed");
+}