@feltdb/core 0.8.4 → 0.8.6

package/dist/create/server-source/crates/feltdb/tests/durable_backup_contract.rs

@@ -0,0 +1,445 @@
+//! Durable backup, verification and restore.
+//!
+//! The question this answers is not "can we serialize the database" — the
+//! in-process snapshot already round-tripped. It is:
+//!
+//! > **Can an operator produce a durable backup artifact, verify it
+//! > independently, restore it into a fresh database, and prove that the
+//! > restored database has the same authoritative state and history?**
+//!
+//! The distinction the whole file turns on:
+//!
+//! > **Verification must not merely prove that the file parses. It must prove
+//! > that the resulting database means the same thing.**
+//!
+//! So a backup carries two digests. The content digest proves the bytes are
+//! intact. The **meaning** digest covers what the database *is* — every live
+//! record, the revision graph with its resources, parents, sequences and
+//! authorities, and every retention policy and horizon — and is recomputed from
+//! the *restored database* and compared. A restore that produced a
+//! plausible-but-different database fails there.
+//!
+//! See `docs/architecture/durable-backup.md`.
+
+use feltdb::state_model::{ParentLookup, RetentionPolicy, StateStore};
+use feltdb::{verify_backup, FeltDb, FlowError, BACKUP_VERSION, DURABLE_FORMAT_VERSION};
+use serde_json::{json, Value};
+use std::path::Path;
+use std::sync::Arc;
+use tempfile::TempDir;
+
+/// A database with everything a backup has to carry: plain state, a bounded
+/// history, an unbounded one, a fork, and expired ancestry.
+fn populated(directory: &Path) -> (Arc<FeltDb>, StateStore) {
+    let db = Arc::new(FeltDb::open(directory.join("source.log")).unwrap());
+    let store = StateStore::with_feltdb(db.clone()).unwrap();
+
+    // A resource under retention, with history expired behind a horizon.
+    store
+        .set_retention_policy("tasks:1", RetentionPolicy::keep_last(3))
+        .unwrap();
+    db.insert("tasks:1", json!({"n": 0})).unwrap();
+    for n in 1..12 {
+        db.update("tasks:1", json!({ "n": n })).unwrap();
+    }
+
+    // A resource with unbounded history.
+    db.insert("tasks:2", json!({"n": 0})).unwrap();
+    for n in 1..4 {
+        db.update("tasks:2", json!({ "n": n })).unwrap();
+    }
+
+    // A forked history, committed explicitly.
+    let base = store
+        .create("docs:1", r#"{"v":0}"#.into(), "alice".into())
+        .unwrap();
+    store
+        .commit(r#"{"v":1}"#.into(), &base, "alice".into())
+        .unwrap();
+    store
+        .commit(r#"{"v":2}"#.into(), &base, "bob".into())
+        .unwrap();
+
+    (db, store)
+}
+
+/// Everything about a database a backup is supposed to preserve.
+#[derive(Debug, PartialEq)]
+struct Meaning {
+    current: Vec<(String, String)>,
+    resources: Vec<String>,
+    revisions: Vec<(String, String, Option<String>, u64, String, String)>,
+    retention: Vec<(String, Option<usize>, u64)>,
+}
+
+fn meaning(db: &Arc<FeltDb>, store: &StateStore) -> Meaning {
+    let mut current = Vec::new();
+    for key in ["tasks:1", "tasks:2"] {
+        if let Some(value) = db.get::<Value>(key).unwrap() {
+            current.push((key.to_string(), value.to_string()));
+        }
+    }
+    let resources = store.resources();
+    let mut revisions = Vec::new();
+    for resource in &resources {
+        for revision in store.history_of(resource) {
+            revisions.push((
+                revision.resource.clone(),
+                revision.id.as_hex().to_string(),
+                revision
+                    .parent_id
+                    .as_ref()
+                    .map(|id| id.as_hex().to_string()),
+                revision.sequence,
+                revision.content_id.as_hex().to_string(),
+                revision.authority.clone(),
+            ));
+        }
+    }
+    revisions.sort();
+    let retention = resources
+        .iter()
+        .map(|resource| {
+            (
+                resource.clone(),
+                store.retention_policy(resource).keep_last,
+                store.retention_horizon(resource),
+            )
+        })
+        .collect();
+    Meaning {
+        current,
+        resources: resources.clone(),
+        revisions,
+        retention,
+    }
+}
+
+fn expect_rejected(outcome: Result<impl Sized, FlowError>) -> feltdb::BackupProblem {
+    match outcome {
+        Err(FlowError::BackupRejected(problem)) => *problem,
+        Err(other) => panic!("expected a backup rejection, got {other}"),
+        Ok(_) => panic!("the damaged backup was accepted — this is the defect"),
+    }
+}
+
+// ---------------------------------------------------------------------------
+// The artifact
+// ---------------------------------------------------------------------------
+
+/// A backup is a self-contained artifact, and it does not need the database it
+/// came from.
+#[test]
+fn a_backup_is_self_contained_and_outlives_its_source() {
+    let directory = TempDir::new().unwrap();
+    let (db, store) = populated(directory.path());
+    let expected = meaning(&db, &store);
+
+    let backup = directory.path().join("db.feltbackup");
+    let written = db.write_backup(&backup).unwrap();
+    assert_eq!(written.backup_version, BACKUP_VERSION);
+    assert_eq!(written.format_version, DURABLE_FORMAT_VERSION);
+    assert!(written.rows > 0);
+
+    // The source is gone entirely.
+    drop(store);
+    drop(db);
+    std::fs::remove_file(directory.path().join("source.log")).unwrap();
+
+    // The artifact still verifies, and still restores.
+    let verified = verify_backup(&backup).unwrap();
+    assert_eq!(verified, written);
+
+    let target = directory.path().join("restored.log");
+    let (restored, verification) = FeltDb::restore_backup(&backup, &target).unwrap();
+    assert_eq!(verification.meaning_digest, written.meaning_digest);
+
+    let restored = Arc::new(restored);
+    let restored_store = StateStore::with_feltdb(restored.clone()).unwrap();
+    assert_eq!(meaning(&restored, &restored_store), expected);
+}
+
+/// Everything the model was built to hold survives a round trip: resource
+/// identity, revision identity, ancestry, forks, retention and provenance.
+#[test]
+fn a_restore_preserves_state_history_forks_retention_and_provenance() {
+    let directory = TempDir::new().unwrap();
+    let (db, store) = populated(directory.path());
+    let backup = directory.path().join("db.feltbackup");
+    db.write_backup(&backup).unwrap();
+
+    let target = directory.path().join("restored.log");
+    let (restored, _) = FeltDb::restore_backup(&backup, &target).unwrap();
+    let restored = Arc::new(restored);
+    let restored_store = StateStore::with_feltdb(restored.clone()).unwrap();
+
+    // Current state.
+    let current: Value = restored.get("tasks:1").unwrap().unwrap();
+    assert_eq!(current["n"], json!(11));
+
+    // Retention state, including the horizon.
+    assert_eq!(
+        restored_store.retention_policy("tasks:1"),
+        RetentionPolicy::keep_last(3)
+    );
+    assert_eq!(
+        restored_store.retention_horizon("tasks:1"),
+        store.retention_horizon("tasks:1")
+    );
+
+    // The bounded history is exactly what it was, expiry included.
+    let before = store.history_of("tasks:1");
+    let after = restored_store.history_of("tasks:1");
+    assert_eq!(after.len(), 3);
+    assert_eq!(
+        after.iter().map(|r| r.id.clone()).collect::<Vec<_>>(),
+        before.iter().map(|r| r.id.clone()).collect::<Vec<_>>(),
+        "revision identities are preserved, not recomputed"
+    );
+    assert_eq!(
+        after[0].sequence, before[0].sequence,
+        "sequences survive, so history did not restart"
+    );
+
+    // Ancestry, and the expiry boundary that goes with it.
+    assert!(matches!(
+        restored_store.parent_of(&after[0].id),
+        Some(ParentLookup::Expired(_))
+    ));
+    assert!(matches!(
+        restored_store.parent_of(&after[1].id),
+        Some(ParentLookup::Revision(_))
+    ));
+
+    // The fork, both heads and their shared base.
+    let docs = restored_store.history_of("docs:1");
+    assert_eq!(docs.len(), 3, "base plus two heads");
+    let heads: Vec<_> = docs.iter().filter(|r| r.sequence == 1).collect();
+    assert_eq!(heads.len(), 2, "the fork survived");
+    assert_eq!(heads[0].parent_id, heads[1].parent_id);
+
+    // Authority provenance.
+    let mut authorities: Vec<&str> = heads.iter().map(|r| r.authority.as_str()).collect();
+    authorities.sort();
+    assert_eq!(authorities, vec!["alice", "bob"]);
+}
+
+/// A restored database is a live database: it continues history rather than
+/// restarting it.
+#[test]
+fn a_restored_database_continues_history() {
+    let directory = TempDir::new().unwrap();
+    let (db, store) = populated(directory.path());
+    let head_before = store.head_of("tasks:2").unwrap();
+    let backup = directory.path().join("db.feltbackup");
+    db.write_backup(&backup).unwrap();
+
+    let target = directory.path().join("restored.log");
+    let (restored, _) = FeltDb::restore_backup(&backup, &target).unwrap();
+    let restored = Arc::new(restored);
+    restored.update("tasks:2", json!({"n": 99})).unwrap();
+
+    let restored_store = StateStore::with_feltdb(restored.clone()).unwrap();
+    let head_after = restored_store.head_of("tasks:2").unwrap();
+    assert_eq!(head_after.sequence, head_before.sequence + 1);
+    assert_eq!(head_after.parent_id.as_ref(), Some(&head_before.id));
+}
+
+// ---------------------------------------------------------------------------
+// Verification, without a live database
+// ---------------------------------------------------------------------------
+
+/// Verification reads the artifact and never opens or writes a database.
+#[test]
+fn verification_inspects_the_artifact_without_mutating_it() {
+    let directory = TempDir::new().unwrap();
+    let (db, _store) = populated(directory.path());
+    let backup = directory.path().join("db.feltbackup");
+    let written = db.write_backup(&backup).unwrap();
+
+    let before = std::fs::read(&backup).unwrap();
+    let verified = verify_backup(&backup).unwrap();
+    assert_eq!(verified.meaning_digest, written.meaning_digest);
+    assert_eq!(
+        std::fs::read(&backup).unwrap(),
+        before,
+        "verifying must not modify the artifact"
+    );
+
+    // And nothing was restored as a side effect.
+    assert!(!directory.path().join("restored.log").exists());
+}
+
+/// **The negative control.** One altered byte must refuse, not restore a
+/// plausible-but-wrong database.
+#[test]
+fn a_single_altered_byte_is_refused() {
+    let directory = TempDir::new().unwrap();
+    let (db, _store) = populated(directory.path());
+    let backup = directory.path().join("db.feltbackup");
+    db.write_backup(&backup).unwrap();
+
+    // Change one digit inside a stored value: still valid JSON, still a valid
+    // record, and a different database.
+    let text = std::fs::read_to_string(&backup).unwrap();
+    let altered = text.replacen(r#""n":11"#, r#""n":12"#, 1);
+    assert_ne!(altered, text, "the fixture must actually change something");
+    std::fs::write(&backup, &altered).unwrap();
+
+    let problem = expect_rejected(verify_backup(&backup));
+    assert!(
+        problem.reason.contains("digest") || problem.reason.contains("mean"),
+        "rejected on meaning, not on syntax: {}",
+        problem.reason
+    );
+
+    let target = directory.path().join("restored.log");
+    expect_rejected(FeltDb::restore_backup(&backup, &target));
+    assert!(
+        !target.exists(),
+        "a refused restore leaves no half-built database"
+    );
+}
+
+/// A truncated artifact is refused, at the header and in the body.
+#[test]
+fn a_truncated_backup_is_refused() {
+    let directory = TempDir::new().unwrap();
+    let (db, _store) = populated(directory.path());
+    let backup = directory.path().join("db.feltbackup");
+    db.write_backup(&backup).unwrap();
+    let full = std::fs::read_to_string(&backup).unwrap();
+
+    // Body cut short: the header's record count no longer matches.
+    let mut lines: Vec<&str> = full.lines().collect();
+    lines.truncate(lines.len() - 2);
+    std::fs::write(&backup, lines.join("\n")).unwrap();
+    let problem = expect_rejected(verify_backup(&backup));
+    assert!(
+        problem.reason.contains("declares"),
+        "the count mismatch is what catches it: {}",
+        problem.reason
+    );
+
+    // Header cut short.
+    std::fs::write(&backup, &full[..40]).unwrap();
+    let problem = expect_rejected(verify_backup(&backup));
+    assert_eq!(problem.line_number, Some(1));
+
+    // Nothing at all.
+    std::fs::write(&backup, "").unwrap();
+    let problem = expect_rejected(verify_backup(&backup));
+    assert!(problem.reason.contains("empty"));
+}
+
+/// A malformed record inside the artifact is refused, and located.
+#[test]
+fn a_malformed_record_in_a_backup_is_refused() {
+    let directory = TempDir::new().unwrap();
+    let (db, _store) = populated(directory.path());
+    let backup = directory.path().join("db.feltbackup");
+    db.write_backup(&backup).unwrap();
+
+    let mut lines: Vec<String> = std::fs::read_to_string(&backup)
+        .unwrap()
+        .lines()
+        .map(str::to_string)
+        .collect();
+    lines[2] = "{not valid json}".to_string();
+    std::fs::write(&backup, lines.join("\n")).unwrap();
+
+    let problem = expect_rejected(verify_backup(&backup));
+    assert_eq!(problem.line_number, Some(3));
+}
+
+/// A backup from another backup version, or another durable format, is refused.
+#[test]
+fn an_incompatible_backup_version_is_refused() {
+    let directory = TempDir::new().unwrap();
+    let (db, _store) = populated(directory.path());
+    let backup = directory.path().join("db.feltbackup");
+    db.write_backup(&backup).unwrap();
+    let full = std::fs::read_to_string(&backup).unwrap();
+
+    for (field, replacement) in [
+        ("\"backup_version\":1", "\"backup_version\":2"),
+        (
+            &format!("\"format_version\":{DURABLE_FORMAT_VERSION}"),
+            &format!("\"format_version\":{}", DURABLE_FORMAT_VERSION + 1),
+        ),
+    ] {
+        let altered = full.replacen(field, replacement, 1);
+        assert_ne!(altered, full, "fixture must change {field}");
+        std::fs::write(&backup, &altered).unwrap();
+        let problem = expect_rejected(verify_backup(&backup));
+        assert_eq!(problem.line_number, Some(1));
+        assert!(problem.reason.contains("version"));
+    }
+}
+
+/// A file that is not a FeltDB backup is refused rather than guessed at.
+#[test]
+fn a_foreign_file_is_not_mistaken_for_a_backup() {
+    let directory = TempDir::new().unwrap();
+    let backup = directory.path().join("not-a-backup.feltbackup");
+    std::fs::write(&backup, "{\"hello\":\"world\"}\n").unwrap();
+    let problem = expect_rejected(verify_backup(&backup));
+    assert_eq!(problem.line_number, Some(1));
+}
+
+// ---------------------------------------------------------------------------
+// Restore safety
+// ---------------------------------------------------------------------------
+
+/// Restore refuses to write over an existing database.
+///
+/// A restore that could overwrite the thing an operator is trying to recover is
+/// not a recovery tool.
+#[test]
+fn a_restore_will_not_overwrite_an_existing_database() {
+    let directory = TempDir::new().unwrap();
+    let (db, _store) = populated(directory.path());
+    let backup = directory.path().join("db.feltbackup");
+    db.write_backup(&backup).unwrap();
+
+    let occupied = directory.path().join("occupied.log");
+    let existing = FeltDb::open(&occupied).unwrap();
+    existing
+        .insert("tasks:9", json!({"important": true}))
+        .unwrap();
+    let before = std::fs::read(&occupied).unwrap();
+    drop(existing);
+
+    let problem = expect_rejected(FeltDb::restore_backup(&backup, &occupied));
+    assert!(problem.reason.contains("already exists"));
+    assert_eq!(
+        std::fs::read(&occupied).unwrap(),
+        before,
+        "the existing database was not touched"
+    );
+}
+
+/// A restored database is itself backup-able, and the second backup means the
+/// same as the first.
+///
+/// This is the property that makes a backup chain trustworthy: restoring and
+/// re-backing up is not lossy.
+#[test]
+fn a_backup_of_a_restore_means_the_same_thing() {
+    let directory = TempDir::new().unwrap();
+    let (db, _store) = populated(directory.path());
+    let first = directory.path().join("first.feltbackup");
+    let original = db.write_backup(&first).unwrap();
+
+    let target = directory.path().join("restored.log");
+    let (restored, _) = FeltDb::restore_backup(&first, &target).unwrap();
+
+    let second = directory.path().join("second.feltbackup");
+    let again = restored.write_backup(&second).unwrap();
+
+    assert_eq!(
+        again.meaning_digest, original.meaning_digest,
+        "a round trip does not change what the database means"
+    );
+    assert_eq!(again.rows, original.rows);
+}