@feltdb/core 0.8.5 → 0.8.7

package/dist/create/server-source/crates/feltdb/tests/managed_incident_regression.rs

@@ -0,0 +1,191 @@
+//! The managed incident, composed.
+//!
+//! The four remediations each have their own contract: `compaction_stall_contract`
+//! covers the deferred rewrite, `bounded_read_contract` covers scoped reads, and
+//! the server crate's `admission_tests` cover the refusal. Every one of them
+//! exercises its fix in isolation, on an idle database.
+//!
+//! The incident was not any one of them in isolation. It was their composition:
+//! a populated collection, a timer firing compaction into the single lock that
+//! serializes every read and write, and scoped requests arriving throughout.
+//! A database that satisfies each contract separately can still fail the shape
+//! that actually happened, so the shape gets its own test.
+//!
+//! # What this file does not establish
+//!
+//! **It does not measure latency, and it does not prove the incident cannot
+//! recur.** No assertion here bounds how long a read waits, and every read
+//! still queues behind the same mutex. That is claim `M6`, and it is recorded
+//! in the certification matrix as `Unproven` precisely because a test like this
+//! one is not evidence for it. A timing assertion added here would either be
+//! flaky or be tuned until it passed, and both are worse than an honest gap.
+//!
+//! What it does establish is that under the composed load, the timer does not
+//! rewrite, scoped reads stay bounded, and no read observes a torn or missing
+//! database.
+
+use feltdb::{CompactionOutcome, CompactionPolicy, FeltDb};
+use serde_json::json;
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::sync::Arc;
+use tempfile::TempDir;
+
+const ROWS: usize = 800;
+
+/// A database shaped like the affected instance: one large collection, every
+/// operation acknowledged so a compaction always has work to do.
+fn incident_shaped() -> (TempDir, Arc<FeltDb>) {
+    let directory = TempDir::new().unwrap();
+    let db = Arc::new(FeltDb::open(directory.path().join("incident.log")).unwrap());
+    db.set_compaction_policy(CompactionPolicy::default());
+
+    for index in 0..ROWS {
+        db.insert(
+            &format!("docs:{index:05}"),
+            json!({ "n": index, "body": "x".repeat(200) }),
+        )
+        .unwrap();
+    }
+    let versions = db.operation_versions().unwrap();
+    db.acknowledge_peer_versions("peer-1".to_string(), versions)
+        .unwrap();
+
+    (directory, db)
+}
+
+/// **Scoped reads stay correct and bounded while the compaction timer runs.**
+///
+/// The compacting thread calls the same entry point the server's timer calls,
+/// continuously, for the whole run. The reading thread does what the handlers
+/// now do: a keyed lookup and a limited search, never a materializing scan.
+///
+/// Three properties are asserted, and each one is a way the incident could
+/// return:
+///
+/// 1. every read returns the right answer — a compaction running concurrently
+///    never exposes a partially pruned database;
+/// 2. every bounded search visits exactly its limit — the amplifier is still
+///    absent under load, not just on an idle database;
+/// 3. the timer performs no durable rewrite — the expensive half stays deferred
+///    when the policy says it should, and does not creep back in because
+///    something else about the workload changed.
+#[test]
+fn scoped_reads_stay_correct_and_bounded_while_compaction_runs() {
+    let (_directory, db) = incident_shaped();
+
+    let stop = Arc::new(AtomicBool::new(false));
+    let rewrites = Arc::new(AtomicUsize::new(0));
+    let ticks = Arc::new(AtomicUsize::new(0));
+
+    let compactor = {
+        let db = db.clone();
+        let stop = stop.clone();
+        let rewrites = rewrites.clone();
+        let ticks = ticks.clone();
+        std::thread::spawn(move || {
+            while !stop.load(Ordering::Relaxed) {
+                let outcome = db
+                    .maybe_compact_operation_log(&["peer-1".to_string()])
+                    .expect("a timer-driven compaction does not fail");
+                ticks.fetch_add(1, Ordering::Relaxed);
+                if matches!(outcome, CompactionOutcome::Rewritten { .. }) {
+                    rewrites.fetch_add(1, Ordering::Relaxed);
+                }
+            }
+        })
+    };
+
+    for round in 0..200 {
+        let key = format!("docs:{:05}", round % ROWS);
+        let row = db
+            .get_collection_record("docs", &key)
+            .unwrap()
+            .unwrap_or_else(|| panic!("{key} is readable while compaction runs"));
+        assert_eq!(row.value["n"], json!(round % ROWS));
+
+        let visited = AtomicUsize::new(0);
+        let matched = db
+            .query_collection("docs", Some(10), |row| {
+                visited.fetch_add(1, Ordering::Relaxed);
+                row.value["body"].is_string()
+            })
+            .unwrap();
+        assert_eq!(matched.len(), 10, "the limit is honoured under load");
+        assert_eq!(
+            visited.load(Ordering::Relaxed),
+            10,
+            "and the search did not walk the collection"
+        );
+    }
+
+    stop.store(true, Ordering::Relaxed);
+    compactor.join().unwrap();
+
+    assert!(
+        ticks.load(Ordering::Relaxed) > 0,
+        "the compaction thread actually ran"
+    );
+    assert_eq!(
+        rewrites.load(Ordering::Relaxed),
+        0,
+        "the timer rewrote the durable log during the run; the deferred-rewrite \
+         fix does not hold under the composed workload"
+    );
+}
+
+/// **The composed run leaves the database intact.**
+///
+/// The incident's worst plausible outcome was not slowness, it was a database
+/// damaged by a rewrite interleaved with traffic. After the concurrent run the
+/// full collection is still present, still correct, and still there after a
+/// reopen — including once a rewrite is forced to happen.
+#[test]
+fn a_concurrent_compaction_leaves_the_database_intact() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("intact.log");
+    let db = Arc::new(FeltDb::open(&path).unwrap());
+    db.set_compaction_policy(CompactionPolicy::default());
+
+    for index in 0..ROWS {
+        db.insert(&format!("docs:{index:05}"), json!({ "n": index }))
+            .unwrap();
+    }
+    let versions = db.operation_versions().unwrap();
+    db.acknowledge_peer_versions("peer-1".to_string(), versions)
+        .unwrap();
+
+    let stop = Arc::new(AtomicBool::new(false));
+    let compactor = {
+        let db = db.clone();
+        let stop = stop.clone();
+        std::thread::spawn(move || {
+            while !stop.load(Ordering::Relaxed) {
+                db.maybe_compact_operation_log(&["peer-1".to_string()])
+                    .unwrap();
+            }
+        })
+    };
+
+    for index in ROWS..ROWS + 200 {
+        db.insert(&format!("docs:{index:05}"), json!({ "n": index }))
+            .unwrap();
+    }
+
+    stop.store(true, Ordering::Relaxed);
+    compactor.join().unwrap();
+
+    // An explicit compaction still rewrites — the deferral is a policy, not a
+    // disablement — and the rewrite happens over a database that concurrent
+    // traffic has just been mutating.
+    db.compact_operation_log(&["peer-1".to_string()]).unwrap();
+    drop(db);
+
+    let reopened = FeltDb::open(&path).unwrap();
+    for index in 0..ROWS + 200 {
+        let value: serde_json::Value = reopened
+            .get(&format!("docs:{index:05}"))
+            .unwrap()
+            .unwrap_or_else(|| panic!("docs:{index:05} survived the composed run"));
+        assert_eq!(value["n"], json!(index));
+    }
+}

package/dist/create/server-source/crates/feltdb/tests/operational_health_contract.rs

@@ -0,0 +1,278 @@
+//! Operational health.
+//!
+//! Everything from the format work onward proved what FeltDB *does*. This file
+//! asks a different question:
+//!
+//! > **Can an operator reliably tell what FeltDB knows about itself?**
+//!
+//! The rule the whole surface is built on:
+//!
+//! > **Health can expose a proven fact. It cannot create a stronger
+//! > guarantee.**
+//!
+//! So `DatabaseHealth` reports the durable format an open accepted and whether
+//! replay discarded an incomplete final append, and nothing else. There is no
+//! replication health, no backup freshness and no durability claim, because
+//! none of those has an operational contract yet and inventing one here would
+//! reintroduce exactly the defect this replaces: a reassuring label that
+//! observes nothing.
+//!
+//! See `docs/architecture/operational-health.md`.
+
+use feltdb::{
+    inspect_durable_format, DatabaseHealth, DurableFormat, FeltDb, FlowError, LogRecovery,
+    StorageHealth, DURABLE_FORMAT_VERSION,
+};
+use serde_json::{json, Value};
+use std::io::Write;
+use std::path::Path;
+use tempfile::TempDir;
+
+fn three_writes(path: &Path) -> Vec<u8> {
+    let db = FeltDb::open(path).unwrap();
+    for n in 0..3 {
+        db.insert(&format!("tasks:{n}"), json!({ "n": n })).unwrap();
+    }
+    drop(db);
+    std::fs::read(path).unwrap()
+}
+
+// ---------------------------------------------------------------------------
+// The five questions health has to answer
+// ---------------------------------------------------------------------------
+
+/// **A clean startup reports clean.**
+#[test]
+fn a_clean_open_reports_clean_storage() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("clean.log");
+    three_writes(&path);
+
+    let db = FeltDb::open(&path).unwrap();
+    let health = db.health();
+
+    assert_eq!(health.storage, StorageHealth::Clean);
+    assert_eq!(health.recovery, LogRecovery::Clean);
+    assert_eq!(
+        health.durable_format,
+        DurableFormat::Versioned(DURABLE_FORMAT_VERSION)
+    );
+    assert!(health.is_nominal());
+    assert_eq!(health.storage.label(), "clean");
+}
+
+/// **The regression that matters.** A database that discarded a torn tail must
+/// not report the same health as one that replayed cleanly.
+///
+/// This is the operational consequence of the corruption work: the distinction
+/// between a clean open and a recovered one was established there, and it is
+/// worth nothing to an operator unless it reaches them.
+#[test]
+fn a_recovered_open_does_not_report_the_same_health_as_a_clean_one() {
+    let directory = TempDir::new().unwrap();
+    let clean_path = directory.path().join("clean.log");
+    let torn_path = directory.path().join("torn.log");
+    let full = three_writes(&clean_path);
+
+    // The same database, with its final append cut short.
+    std::fs::write(&torn_path, &full[..full.len() - 30]).unwrap();
+
+    let clean = FeltDb::open(&clean_path).unwrap().health();
+    let torn = FeltDb::open(&torn_path).unwrap().health();
+
+    assert_ne!(
+        clean, torn,
+        "a database that lost its last write must not look identical to one that did not"
+    );
+    assert!(clean.is_nominal());
+    assert!(!torn.is_nominal(), "recovered is not nominal");
+    assert_ne!(clean.storage.label(), torn.storage.label());
+
+    match torn.storage {
+        StorageHealth::RecoveredIncompleteWrite {
+            byte_offset,
+            discarded_bytes,
+        } => {
+            assert!(byte_offset > 0, "the operator is told where");
+            assert!(discarded_bytes > 0, "and how much");
+        }
+        other => panic!("expected a reported recovery, got {other:?}"),
+    }
+
+    // And it says so in words an operator can act on.
+    let described = torn.storage.to_string();
+    assert!(described.contains("incomplete append"));
+    assert!(described.contains("last write did not land"));
+}
+
+/// **A corrupt database does not become healthy.** It does not open at all.
+///
+/// The ambiguity being removed here is "healthy process, unhealthy database":
+/// there is no state in which a server is serving a database whose durable log
+/// failed to replay.
+#[test]
+fn a_corrupt_database_never_reaches_a_health_report() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("corrupt.log");
+    three_writes(&path);
+
+    let mut file = std::fs::OpenOptions::new()
+        .append(true)
+        .open(&path)
+        .unwrap();
+    writeln!(file, "this is not json at all").unwrap();
+    drop(file);
+
+    match FeltDb::open(&path) {
+        Err(FlowError::CorruptLogLine(_)) => {}
+        Err(other) => panic!("wrong error: {other}"),
+        Ok(_) => panic!("a corrupt database opened and could have reported health"),
+    }
+}
+
+/// **An incompatible database stays refused**, so a running server cannot be
+/// advertising one.
+#[test]
+fn an_incompatible_database_never_reaches_a_health_report() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("future.log");
+    std::fs::write(
+        &path,
+        format!(
+            "{}\n",
+            json!({
+                "record_type": "feltdb.format.v1",
+                "format_version": DURABLE_FORMAT_VERSION + 1
+            })
+        ),
+    )
+    .unwrap();
+
+    assert!(!inspect_durable_format(&path).unwrap().is_compatible());
+    assert!(matches!(
+        FeltDb::open(&path),
+        Err(FlowError::IncompatibleFormat(_))
+    ));
+}
+
+/// Health survives the thing it describes: reopening a repaired log reports
+/// clean again, because the condition is genuinely gone.
+///
+/// A status that stayed degraded forever would be as useless as one that never
+/// was.
+#[test]
+fn health_reflects_the_current_open_not_the_database_s_past() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("repaired.log");
+    let full = three_writes(&path);
+    std::fs::write(&path, &full[..full.len() - 30]).unwrap();
+
+    let recovered = FeltDb::open(&path).unwrap();
+    assert!(!recovered.health().is_nominal());
+    drop(recovered);
+
+    let reopened = FeltDb::open(&path).unwrap();
+    assert!(
+        reopened.health().is_nominal(),
+        "the incomplete append is gone, so the condition is too"
+    );
+}
+
+/// A database with no torn tail but written before format versioning is
+/// reported as what it is, rather than as an error or as current.
+#[test]
+fn an_unversioned_database_reports_its_actual_format() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("unversioned.log");
+    std::fs::write(
+        &path,
+        format!(
+            "{}\n",
+            json!({
+                "capability": "tasks", "key": "tasks:1",
+                "rust_type": "serde_json::value::Value", "value": {"n": 1},
+                "unix_ms": 1, "content_hash": null, "flow_ref": null,
+                "deleted": false, "operation": null
+            })
+        ),
+    )
+    .unwrap();
+
+    let health = FeltDb::open(&path).unwrap().health();
+    assert_eq!(health.durable_format, DurableFormat::Unversioned);
+    assert!(health.is_nominal(), "unversioned is not unhealthy");
+}
+
+// ---------------------------------------------------------------------------
+// What health deliberately does not say
+// ---------------------------------------------------------------------------
+
+/// **Health does not manufacture a guarantee the database does not have.**
+///
+/// The defect being replaced was `storage: "durable"` — a string constant that
+/// inspected nothing and asserted the one property still unproven. The
+/// replacement is a small closed set of observed conditions, and every label in
+/// it names what was seen rather than what would be reassuring.
+///
+/// In particular nothing here claims durability against power loss, which
+/// remains unproven; nothing claims the database is replicated, backed up, or
+/// within any capacity; and there is no "probably fine".
+#[test]
+fn health_reports_only_conditions_that_were_observed() {
+    let directory = TempDir::new().unwrap();
+    let clean = directory.path().join("a.log");
+    let torn = directory.path().join("b.log");
+    let full = three_writes(&clean);
+    std::fs::write(&torn, &full[..full.len() - 30]).unwrap();
+
+    let labels: Vec<&str> = [&clean, &torn]
+        .iter()
+        .map(|path| FeltDb::open(path).unwrap().health().storage.label())
+        .collect();
+
+    // Exactly two conditions, both naming an observation.
+    assert_eq!(labels, vec!["clean", "recovered-incomplete-write"]);
+    for label in &labels {
+        assert!(
+            !label.contains("durable") && !label.contains("healthy"),
+            "'{label}' would assert more than was observed"
+        );
+    }
+}
+
+/// The health value is derived from the recovery the open recorded, not from
+/// anything a caller can set.
+#[test]
+fn health_is_derived_from_the_open_rather_than_configured() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("derived.log");
+    let full = three_writes(&path);
+    std::fs::write(&path, &full[..full.len() - 30]).unwrap();
+
+    let db = FeltDb::open(&path).unwrap();
+    let health: DatabaseHealth = db.health();
+
+    // The two agree because one is computed from the other.
+    match (&health.recovery, &health.storage) {
+        (
+            LogRecovery::RecoveredTornTail {
+                byte_offset: recovered_at,
+                discarded_bytes: recovered_bytes,
+            },
+            StorageHealth::RecoveredIncompleteWrite {
+                byte_offset,
+                discarded_bytes,
+            },
+        ) => {
+            assert_eq!(recovered_at, byte_offset);
+            assert_eq!(recovered_bytes, discarded_bytes);
+        }
+        other => panic!("recovery and storage disagree: {other:?}"),
+    }
+
+    // Writing more does not change what the open found.
+    db.insert("tasks:9", json!({"n": 9})).unwrap();
+    assert_eq!(db.health(), health);
+    let value: Value = db.get("tasks:9").unwrap().unwrap();
+    assert_eq!(value["n"], json!(9), "and the database is fully usable");
+}