@feltdb/core 0.8.5 → 0.8.7

package/dist/create/server-source/crates/feltdb/tests/durable_corruption_contract.rs

@@ -0,0 +1,518 @@
+//! Corruption detection and fail-closed recovery.
+//!
+//! The invariant this file enforces:
+//!
+//! > **A database open may recover from a documented, provably incomplete final
+//! > write, but it must never silently discard bytes that could represent a
+//! > committed durable record.**
+//!
+//! Three classes, and the boundary between them is the whole subject:
+//!
+//! | condition | behaviour |
+//! | --- | --- |
+//! | valid complete record | replayed |
+//! | final record with **no terminator** | discarded, and the recovery is reported |
+//! | anything else invalid, anywhere | **open refused** |
+//!
+//! The dangerous idea this rejects is that "last line" means "safe to discard".
+//! A *complete* record that is invalid is corruption wherever it sits, and
+//! treating it as a torn write would convert a disk fault into apparently
+//! successful data loss merely because it happened at the end of the file.
+//!
+//! See `docs/architecture/durable-corruption.md`.
+
+use feltdb::{FeltDb, FlowError, LogRecovery};
+use serde_json::{json, Value};
+use std::io::Write;
+use std::path::Path;
+use tempfile::TempDir;
+
+fn format_record() -> String {
+    json!({"record_type": "feltdb.format.v1", "format_version": feltdb::DURABLE_FORMAT_VERSION})
+        .to_string()
+}
+
+fn row(key: &str, n: i64) -> String {
+    json!({
+        "capability": key.split(':').next().unwrap(),
+        "key": key,
+        "rust_type": "serde_json::value::Value",
+        "value": {"n": n},
+        "unix_ms": 1,
+        "content_hash": null,
+        "flow_ref": null,
+        "deleted": false,
+        "operation": null
+    })
+    .to_string()
+}
+
+/// A transaction as the log stores it: **one record carrying all its rows**.
+fn transaction(id: &str, keys: &[(&str, i64)], revision: u64) -> String {
+    let rows: Vec<Value> = keys
+        .iter()
+        .map(|(key, n)| serde_json::from_str::<Value>(&row(key, *n)).unwrap())
+        .collect();
+    json!({
+        "record_type": "feltdb.transaction.v1",
+        "transaction_id": id,
+        "payload_hash": null,
+        "state_before": revision,
+        "state_after": revision + 1,
+        "base_revision": revision,
+        "commit_revision": revision + 1,
+        "rows": rows,
+        "audit": null
+    })
+    .to_string()
+}
+
+/// Write a log verbatim. `terminate_last` controls whether the final record
+/// gets its newline — the difference between a finished append and a torn one.
+fn write_log(path: &Path, records: &[String], terminate_last: bool) {
+    let mut file = std::fs::File::create(path).unwrap();
+    for (index, record) in records.iter().enumerate() {
+        if index + 1 == records.len() && !terminate_last {
+            write!(file, "{record}").unwrap();
+        } else {
+            writeln!(file, "{record}").unwrap();
+        }
+    }
+}
+
+fn present(db: &FeltDb, key: &str) -> bool {
+    db.get::<Value>(key).unwrap().is_some()
+}
+
+/// Open, and assert the durable bytes are unchanged whatever the outcome.
+fn open_without_touching(path: &Path) -> Result<FeltDb, FlowError> {
+    let before = std::fs::read(path).unwrap();
+    let outcome = FeltDb::open(path);
+    if outcome.is_err() {
+        assert_eq!(
+            std::fs::read(path).unwrap(),
+            before,
+            "a refused open must leave the database byte-identical"
+        );
+    }
+    outcome
+}
+
+fn expect_corrupt(path: &Path) -> feltdb::LogCorruption {
+    match open_without_touching(path) {
+        Err(FlowError::CorruptLogLine(corruption)) => *corruption,
+        Err(other) => panic!("expected corruption, got {other}"),
+        Ok(_) => panic!("the corrupt database opened — this is the defect"),
+    }
+}
+
+// ---------------------------------------------------------------------------
+// C1 — a clean log
+// ---------------------------------------------------------------------------
+
+/// A valid log replays completely and reports that it did.
+#[test]
+fn a_clean_log_replays_completely_and_reports_clean() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("clean.log");
+    let db = FeltDb::open(&path).unwrap();
+    for n in 1..=3 {
+        db.insert(&format!("tasks:{n}"), json!({ "n": n })).unwrap();
+    }
+    drop(db);
+
+    let before = std::fs::read(&path).unwrap();
+    let reopened = FeltDb::open(&path).unwrap();
+    for n in 1..=3 {
+        assert!(present(&reopened, &format!("tasks:{n}")));
+    }
+    assert_eq!(reopened.log_recovery(), LogRecovery::Clean);
+    assert!(reopened.log_recovery().is_clean());
+    assert_eq!(
+        std::fs::read(&path).unwrap(),
+        before,
+        "a clean open changes nothing"
+    );
+}
+
+// ---------------------------------------------------------------------------
+// C2 / C7 / C8 — the one thing that may be discarded
+// ---------------------------------------------------------------------------
+
+/// An unterminated final record is a torn append: discarded, and **reported**.
+///
+/// This is the only condition under which durable bytes are dropped. The
+/// database is usable; its last write did not land; and the difference between
+/// that and a clean open survives past an internal debug message.
+#[test]
+fn an_unterminated_final_record_is_recovered_and_reported() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("torn.log");
+    write_log(
+        &path,
+        &[
+            format_record(),
+            row("tasks:1", 1),
+            row("tasks:2", 2),
+            row("tasks:3", 3),
+        ],
+        false,
+    );
+    let torn_offset = std::fs::read_to_string(&path)
+        .unwrap()
+        .rfind(&row("tasks:3", 3))
+        .unwrap() as u64;
+
+    let db = FeltDb::open(&path).unwrap();
+
+    // Everything committed before it is intact.
+    assert!(present(&db, "tasks:1"));
+    assert!(present(&db, "tasks:2"));
+    // The incomplete record was not replayed.
+    assert!(!present(&db, "tasks:3"));
+
+    // C7: the recovery is observable, with a location.
+    match db.log_recovery() {
+        LogRecovery::RecoveredTornTail {
+            byte_offset,
+            discarded_bytes,
+        } => {
+            assert_eq!(byte_offset, torn_offset);
+            assert!(discarded_bytes > 0);
+        }
+        other => panic!("expected a reported torn-tail recovery, got {other}"),
+    }
+
+    // C8: and it is distinguishable from a clean open, which is the fact the
+    // health contract will later need.
+    assert!(
+        !db.log_recovery().is_clean(),
+        "an open that discarded data must not look clean"
+    );
+}
+
+/// A partially written record is discarded whole, not half-applied.
+#[test]
+fn a_partial_final_record_is_discarded_whole() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("partial.log");
+    let complete = row("tasks:3", 3);
+    let half = complete[..complete.len() / 2].to_string();
+    write_log(
+        &path,
+        &[format_record(), row("tasks:1", 1), row("tasks:2", 2), half],
+        false,
+    );
+
+    let db = FeltDb::open(&path).unwrap();
+    assert!(present(&db, "tasks:1"));
+    assert!(present(&db, "tasks:2"));
+    assert!(!present(&db, "tasks:3"));
+    assert!(!db.log_recovery().is_clean());
+}
+
+/// After a torn tail is discarded, the log is a valid database again and the
+/// next open is clean.
+///
+/// This is why the truncation happens: without it the next append would splice
+/// onto a partial record and turn a recoverable torn write into permanent
+/// corruption. It is a repair, and it is reported rather than silent.
+#[test]
+fn a_recovered_log_reopens_cleanly_and_accepts_writes() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("reopen.log");
+    let complete = row("tasks:3", 3);
+    write_log(
+        &path,
+        &[
+            format_record(),
+            row("tasks:1", 1),
+            complete[..complete.len() / 2].to_string(),
+        ],
+        false,
+    );
+
+    let db = FeltDb::open(&path).unwrap();
+    assert!(!db.log_recovery().is_clean());
+    db.insert("tasks:9", json!({"n": 9})).unwrap();
+    drop(db);
+
+    let reopened = FeltDb::open(&path).unwrap();
+    assert_eq!(
+        reopened.log_recovery(),
+        LogRecovery::Clean,
+        "the repair leaves a clean log behind it"
+    );
+    assert!(present(&reopened, "tasks:1"));
+    assert!(present(&reopened, "tasks:9"));
+}
+
+// ---------------------------------------------------------------------------
+// C3 — a complete final record is never "safe to discard"
+// ---------------------------------------------------------------------------
+
+/// **The dangerous case.** A terminated final line containing invalid data is
+/// corruption, not a torn write.
+///
+/// It was silently dropped before, which converted a disk fault at end-of-file
+/// into an apparently successful open with data missing. Position in the file
+/// is not evidence about whether a record was finished — the **terminator** is.
+#[test]
+fn a_complete_but_invalid_final_record_refuses_the_open() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("eof-garbage.log");
+    write_log(
+        &path,
+        &[
+            format_record(),
+            row("tasks:1", 1),
+            row("tasks:2", 2),
+            "{not valid json}".to_string(),
+        ],
+        true,
+    );
+
+    let corruption = expect_corrupt(&path);
+    assert_eq!(corruption.line_number, 4);
+    assert!(corruption.reason.contains("not valid JSON"));
+}
+
+/// The same, for a final record that is valid JSON and not a durable record.
+#[test]
+fn a_complete_final_record_of_the_wrong_shape_refuses_the_open() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("eof-shape.log");
+    write_log(
+        &path,
+        &[
+            format_record(),
+            row("tasks:1", 1),
+            json!({"hello": "world"}).to_string(),
+        ],
+        true,
+    );
+
+    let corruption = expect_corrupt(&path);
+    assert_eq!(corruption.line_number, 3);
+    assert!(corruption.reason.contains("durable record"));
+}
+
+// ---------------------------------------------------------------------------
+// C4 / C5 — corruption anywhere else
+// ---------------------------------------------------------------------------
+
+/// Corruption in the middle refuses the open. It never replays what surrounds
+/// it and skips the damage.
+#[test]
+fn corruption_in_the_middle_refuses_the_open() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("middle.log");
+    write_log(
+        &path,
+        &[
+            format_record(),
+            row("tasks:1", 1),
+            "{not valid json}".to_string(),
+            row("tasks:3", 3),
+        ],
+        true,
+    );
+
+    let corruption = expect_corrupt(&path);
+    assert_eq!(corruption.line_number, 3, "the damage is located");
+    assert!(corruption.byte_offset > 0);
+    assert!(corruption.excerpt.contains("not valid json"));
+}
+
+/// **Semantic corruption**, which is the more important half: records that
+/// parse as JSON but are not valid durable records.
+#[test]
+fn semantically_invalid_records_refuse_the_open() {
+    let cases: Vec<(&str, String)> = vec![
+        (
+            "missing required fields",
+            json!({"capability": "tasks", "key": "tasks:2", "value": {"n": 2}}).to_string(),
+        ),
+        (
+            "a record kind this build does not know",
+            json!({"record_type": "feltdb.something.v9", "payload": 1}).to_string(),
+        ),
+        (
+            "a row whose value field is absent",
+            json!({
+                "capability": "tasks", "key": "tasks:2",
+                "rust_type": "serde_json::value::Value",
+                "unix_ms": 1, "content_hash": null, "flow_ref": null,
+                "deleted": false, "operation": null
+            })
+            .to_string(),
+        ),
+    ];
+
+    for (label, damaged) in cases {
+        let directory = TempDir::new().unwrap();
+        let path = directory.path().join("semantic.log");
+        write_log(
+            &path,
+            &[
+                format_record(),
+                row("tasks:1", 1),
+                damaged,
+                row("tasks:3", 3),
+            ],
+            true,
+        );
+        let corruption = expect_corrupt(&path);
+        assert_eq!(corruption.line_number, 3, "{label}");
+    }
+}
+
+// ---------------------------------------------------------------------------
+// C6 — refusal is never destructive
+// ---------------------------------------------------------------------------
+
+/// **A refused open does not repair the database on its way out.**
+///
+/// The decisive case is a log with *both* a torn tail and corruption earlier in
+/// it. Truncating the tail is legitimate only when the rest of the log replayed;
+/// doing it before discovering the corruption would mean a database that
+/// refuses to open has still been modified by the attempt.
+#[test]
+fn a_refused_open_does_not_truncate_a_torn_tail() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("both.log");
+    let complete = row("tasks:9", 9);
+    write_log(
+        &path,
+        &[
+            format_record(),
+            row("tasks:1", 1),
+            "{not valid json}".to_string(),
+            complete[..complete.len() / 2].to_string(),
+        ],
+        false,
+    );
+    let before = std::fs::read(&path).unwrap();
+
+    let corruption = expect_corrupt(&path);
+    assert_eq!(corruption.line_number, 3);
+    assert_eq!(
+        std::fs::read(&path).unwrap(),
+        before,
+        "the torn tail was not truncated by a refused open"
+    );
+}
+
+/// Format inspection is read-only too, even on a database with a torn tail.
+///
+/// This closes a hole in the format-versioning work: inspection shares the log
+/// reader, and a reader that repaired as it went would have made the
+/// byte-identical guarantee conditional on the file having no torn tail.
+#[test]
+fn format_inspection_does_not_modify_a_torn_log() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("inspect.log");
+    let complete = row("tasks:2", 2);
+    write_log(
+        &path,
+        &[
+            format_record(),
+            row("tasks:1", 1),
+            complete[..complete.len() / 2].to_string(),
+        ],
+        false,
+    );
+    let before = std::fs::read(&path).unwrap();
+
+    let compatibility = feltdb::inspect_durable_format(&path).unwrap();
+    assert!(compatibility.is_compatible());
+    assert_eq!(
+        std::fs::read(&path).unwrap(),
+        before,
+        "inspecting a database must never modify it"
+    );
+}
+
+// ---------------------------------------------------------------------------
+// C10 — transaction boundaries
+// ---------------------------------------------------------------------------
+
+/// A transaction is **one durable record**, so a torn transaction is discarded
+/// whole and none of its mutations is exposed.
+///
+/// This result derives from the existing commit protocol rather than being
+/// invented here: `append_transaction` writes the whole transaction — every row
+/// and its commit revision — as a single line, before any in-memory state
+/// changes. There is no window in which some of a transaction's rows are
+/// durable and the rest are not.
+#[test]
+fn a_torn_transaction_exposes_none_of_its_mutations() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("torn-txn.log");
+    let committed = transaction("t1", &[("tasks:1", 1), ("tasks:2", 2)], 0);
+    let torn = transaction("t2", &[("tasks:3", 3), ("tasks:4", 4)], 1);
+
+    write_log(
+        &path,
+        &[
+            format_record(),
+            committed,
+            torn[..torn.len() / 2].to_string(),
+        ],
+        false,
+    );
+
+    let db = FeltDb::open(&path).unwrap();
+    assert!(present(&db, "tasks:1"), "the committed transaction applied");
+    assert!(present(&db, "tasks:2"));
+    assert!(
+        !present(&db, "tasks:3") && !present(&db, "tasks:4"),
+        "the torn transaction applied none of its rows — not even the first"
+    );
+    assert!(!db.log_recovery().is_clean());
+}
+
+/// A complete transaction followed by a torn ordinary record keeps the
+/// transaction and drops only the record.
+#[test]
+fn a_complete_transaction_survives_a_torn_record_after_it() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("txn-then-torn.log");
+    let committed = transaction("t1", &[("tasks:1", 1), ("tasks:2", 2)], 0);
+    let trailing = row("tasks:5", 5);
+
+    write_log(
+        &path,
+        &[
+            format_record(),
+            committed,
+            trailing[..trailing.len() / 2].to_string(),
+        ],
+        false,
+    );
+
+    let db = FeltDb::open(&path).unwrap();
+    assert!(present(&db, "tasks:1"));
+    assert!(present(&db, "tasks:2"));
+    assert!(!present(&db, "tasks:5"));
+}
+
+/// A **complete** but malformed transaction record refuses the open, exactly
+/// like any other complete invalid record.
+#[test]
+fn a_malformed_complete_transaction_refuses_the_open() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("bad-txn.log");
+    write_log(
+        &path,
+        &[
+            format_record(),
+            row("tasks:1", 1),
+            json!({"record_type": "feltdb.transaction.v1", "transaction_id": "t1"}).to_string(),
+        ],
+        true,
+    );
+
+    let corruption = expect_corrupt(&path);
+    assert_eq!(corruption.line_number, 3);
+}