@feltdb/core 0.8.4 → 0.8.6

package/dist/create/server-source/crates/feltdb/tests/workload_envelope_contract.rs

@@ -0,0 +1,442 @@
+//! Durability modes and the workload envelope.
+//!
+//! Two questions, in this order:
+//!
+//! > **What durability contract can FeltDB provide, and what does each one
+//! > cost under a frozen workload?**
+//!
+//! The order matters. Choosing between `flush` and `fsync` first would be
+//! picking an implementation before deciding what a successful write should
+//! mean.
+//!
+//! # What this file asserts, and what it does not
+//!
+//! Measurements on a shared machine are noisy, so the **tests assert
+//! relationships** — a barrier per write costs more than none, grouping sits
+//! between them — and never absolute numbers. The numbers themselves belong in
+//! `docs/architecture/workload-envelope.md`, recorded as *observed under this
+//! frozen workload on this environment* rather than as a capacity claim.
+//!
+//! Nothing here proves power-loss durability. Issuing `fsync` is not evidence
+//! that bytes reached a platter: that depends on the filesystem, the mount
+//! options and the drive's write cache, none of which this database can
+//! observe. D5 stays Unproven, and that is the honest end state.
+
+use feltdb::state_model::{RetentionPolicy, StateStore};
+use feltdb::{AtomicMutation, DurabilityMode, FeltDb};
+use serde_json::json;
+use std::collections::HashMap;
+use std::path::Path;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+use tempfile::TempDir;
+
+// ---------------------------------------------------------------------------
+// The frozen workload
+// ---------------------------------------------------------------------------
+
+/// A workload is fixed by its shape, not by a wall-clock budget, so two runs
+/// measure the same thing.
+#[derive(Debug, Clone, Copy)]
+struct Workload {
+    name: &'static str,
+    writes: usize,
+    resources: usize,
+    payload_bytes: usize,
+    retention: Option<usize>,
+}
+
+/// What one run of a workload cost.
+#[derive(Debug, Clone)]
+struct Measurement {
+    elapsed: Duration,
+    log_bytes: u64,
+    revisions: usize,
+}
+
+impl Measurement {
+    fn writes_per_second(&self, writes: usize) -> f64 {
+        writes as f64 / self.elapsed.as_secs_f64().max(f64::EPSILON)
+    }
+}
+
+fn payload(index: usize, bytes: usize) -> serde_json::Value {
+    json!({ "n": index, "pad": "x".repeat(bytes) })
+}
+
+/// Run one workload under one durability mode. Deterministic in shape.
+fn run(directory: &Path, workload: Workload, mode: DurabilityMode) -> Measurement {
+    let path = directory.join(format!("{}-{:?}.log", workload.name, mode));
+    let db = Arc::new(FeltDb::open(&path).unwrap());
+    db.set_durability_mode(mode);
+    let store = StateStore::with_feltdb(db.clone()).unwrap();
+
+    for resource in 0..workload.resources {
+        let key = format!("bench:{resource}");
+        if let Some(keep) = workload.retention {
+            store
+                .set_retention_policy(&key, RetentionPolicy::keep_last(keep))
+                .unwrap();
+        }
+        db.insert(&key, payload(0, workload.payload_bytes)).unwrap();
+    }
+
+    let start = Instant::now();
+    for index in 1..=workload.writes {
+        let key = format!("bench:{}", index % workload.resources);
+        db.update(&key, payload(index, workload.payload_bytes))
+            .unwrap();
+    }
+    let elapsed = start.elapsed();
+
+    let revisions: usize = (0..workload.resources)
+        .map(|resource| store.history_of(&format!("bench:{resource}")).len())
+        .sum();
+
+    Measurement {
+        elapsed,
+        log_bytes: std::fs::metadata(&path).unwrap().len(),
+        revisions,
+    }
+}
+
+/// The frozen matrix. Changing any row changes what every recorded number
+/// means, so the table lives here rather than being chosen per test.
+const WORKLOADS: &[Workload] = &[
+    Workload {
+        name: "sequential-unbounded",
+        writes: 300,
+        resources: 1,
+        payload_bytes: 100,
+        retention: None,
+    },
+    Workload {
+        name: "sequential-keep5",
+        writes: 300,
+        resources: 1,
+        payload_bytes: 100,
+        retention: Some(5),
+    },
+    Workload {
+        name: "sequential-keep50",
+        writes: 300,
+        resources: 1,
+        payload_bytes: 100,
+        retention: Some(50),
+    },
+    Workload {
+        name: "many-resources",
+        writes: 300,
+        resources: 50,
+        payload_bytes: 100,
+        retention: None,
+    },
+];
+
+// ---------------------------------------------------------------------------
+// Durability modes: the contract, then the cost
+// ---------------------------------------------------------------------------
+
+/// **Every mode states what a successful write guarantees.**
+///
+/// The point of the enum is that the contract is readable rather than inferred
+/// from an API name. `Flushed` does not claim stable storage; `Synced` claims a
+/// barrier was *issued* and explicitly not that it landed; `Grouped` names the
+/// window of acknowledged writes that may not have been through one.
+#[test]
+fn every_durability_mode_states_its_own_guarantee() {
+    let modes = [
+        DurabilityMode::Flushed,
+        DurabilityMode::Synced,
+        DurabilityMode::Grouped { every: 16 },
+    ];
+
+    for mode in modes {
+        let guarantee = mode.guarantee();
+        assert!(!guarantee.is_empty());
+        assert!(
+            !guarantee.contains("power-loss safe") && !guarantee.contains("guaranteed durable"),
+            "{mode:?} overstates: {guarantee}"
+        );
+    }
+
+    // The window each mode leaves is explicit rather than implied.
+    assert_eq!(DurabilityMode::Flushed.unbarriered_window(), None);
+    assert_eq!(DurabilityMode::Synced.unbarriered_window(), Some(0));
+    assert_eq!(
+        DurabilityMode::Grouped { every: 16 }.unbarriered_window(),
+        Some(15)
+    );
+
+    // Only one mode claims a barrier per write, and it still does not claim the
+    // barrier reached the device.
+    assert!(DurabilityMode::Synced.guarantee().contains("barrier"));
+    assert!(DurabilityMode::Synced
+        .guarantee()
+        .contains("cannot observe"));
+}
+
+/// **The default is the weakest contract**, so nothing is silently promised.
+#[test]
+fn the_default_mode_is_the_one_whose_cost_was_already_measured() {
+    let directory = TempDir::new().unwrap();
+    let db = FeltDb::open(directory.path().join("default.log")).unwrap();
+    assert_eq!(db.durability_mode(), DurabilityMode::Flushed);
+    assert_eq!(DurabilityMode::default(), DurabilityMode::Flushed);
+}
+
+/// **A stronger barrier costs more.**
+///
+/// Asserted as an ordering rather than as numbers, because the numbers depend
+/// on the machine. Per-write `fsync` runs roughly three to four times slower
+/// than flush across every row of the frozen matrix, which is large enough to
+/// be a property rather than noise.
+///
+/// **Grouping is deliberately not asserted to sit between them.** The
+/// measurements do not support it at this scale: on the recorded environment
+/// grouped/32 came out *slower* than per-write `fsync` on one workload and
+/// faster on the others. An assertion that can flip on a shared machine is
+/// worse than no assertion, so the observation is recorded in the envelope
+/// document and left unasserted here. What grouping buys — a smaller barrier
+/// count for a stated window of unbarriered writes — is a contract property,
+/// and that *is* asserted, above.
+#[test]
+fn a_barrier_per_write_costs_more_than_none() {
+    let directory = TempDir::new().unwrap();
+    let workload = WORKLOADS[0];
+
+    let flushed = run(directory.path(), workload, DurabilityMode::Flushed);
+    let synced = run(directory.path(), workload, DurabilityMode::Synced);
+
+    assert!(
+        synced.elapsed > flushed.elapsed,
+        "a barrier per write must cost something: synced {:?} vs flushed {:?}",
+        synced.elapsed,
+        flushed.elapsed
+    );
+
+    // Durability is the only thing that changed. The database is identical.
+    assert_eq!(flushed.revisions, synced.revisions);
+    assert_eq!(flushed.log_bytes, synced.log_bytes);
+}
+
+/// Changing the mode changes the contract and nothing else: the same writes
+/// produce the same database.
+///
+/// Run against **one path**, deliberately. An authority's identity is derived
+/// from its path, and a revision records the authority that committed it, so
+/// two databases in different directories never share a history digest even
+/// with identical logical history. That is correct — provenance is part of what
+/// happened — and it means a comparison like this has to reuse the path rather
+/// than compare two locations.
+#[test]
+fn the_durability_mode_does_not_change_what_is_stored() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("same.log");
+    let mut digests = Vec::new();
+
+    for mode in [
+        DurabilityMode::Flushed,
+        DurabilityMode::Synced,
+        DurabilityMode::Grouped { every: 4 },
+    ] {
+        let _ = std::fs::remove_file(&path);
+        let db = Arc::new(FeltDb::open(&path).unwrap());
+        db.set_durability_mode(mode);
+        db.insert("tasks:1", json!({"n": 0})).unwrap();
+        for n in 1..12 {
+            db.update("tasks:1", json!({ "n": n })).unwrap();
+        }
+        let store = StateStore::with_feltdb(db.clone()).unwrap();
+        digests.push((db.state_digest().unwrap(), store.history_digest()));
+    }
+
+    assert_eq!(digests[0], digests[1]);
+    assert_eq!(digests[0], digests[2]);
+}
+
+/// A grouped database still recovers cleanly, because the flush — not the
+/// barrier — is what makes a record survive a process crash.
+#[test]
+fn every_mode_recovers_to_a_clean_prefix() {
+    for mode in [
+        DurabilityMode::Flushed,
+        DurabilityMode::Synced,
+        DurabilityMode::Grouped { every: 4 },
+    ] {
+        let directory = TempDir::new().unwrap();
+        let path = directory.path().join("recover.log");
+        let db = Arc::new(FeltDb::open(&path).unwrap());
+        db.set_durability_mode(mode);
+        db.insert("tasks:1", json!({"n": 0})).unwrap();
+        db.update("tasks:1", json!({"n": 1})).unwrap();
+        db.update("tasks:1", json!({"n": 2})).unwrap();
+        drop(db);
+
+        let full = std::fs::read(&path).unwrap();
+        for cut in (0..=full.len()).step_by(29) {
+            std::fs::write(&path, &full[..cut]).unwrap();
+            let db = Arc::new(FeltDb::open(&path).unwrap());
+            let store = StateStore::with_feltdb(db.clone()).unwrap();
+            let value = db
+                .get::<serde_json::Value>("tasks:1")
+                .unwrap()
+                .map(|value| value["n"].as_i64().unwrap());
+            assert_eq!(
+                store.history_of("tasks:1").len(),
+                value.map(|n| n as usize + 1).unwrap_or(0),
+                "{mode:?} at cut {cut}"
+            );
+        }
+    }
+}
+
+/// **Transactions already carry the strongest contract**, whatever the
+/// single-record mode is.
+///
+/// `append_transaction` issues a barrier unconditionally. So a caller who needs
+/// stable-storage semantics for a group of records has had a way to ask for it
+/// all along, which is a real option in the trade-off rather than a workaround.
+#[test]
+fn a_transaction_commits_under_a_barrier_regardless_of_the_mode() {
+    let directory = TempDir::new().unwrap();
+    let path = directory.path().join("txn.log");
+    let db = Arc::new(FeltDb::open(&path).unwrap());
+    // Even at the weakest single-record contract.
+    db.set_durability_mode(DurabilityMode::Flushed);
+
+    db.apply_atomic_transaction(
+        "t1",
+        None,
+        &[],
+        &[AtomicMutation {
+            capability: "acct".into(),
+            key: "acct:a".into(),
+            value: Some(json!({"balance": 1})),
+        }],
+        None,
+    )
+    .unwrap();
+
+    let value: serde_json::Value = db.get("acct:a").unwrap().unwrap();
+    assert_eq!(value["balance"], json!(1));
+    assert_eq!(
+        db.durability_mode(),
+        DurabilityMode::Flushed,
+        "the transaction path is independent of the single-record contract"
+    );
+}
+
+// ---------------------------------------------------------------------------
+// The envelope
+// ---------------------------------------------------------------------------
+
+/// **Retention cost grows with the window, and the log grows with retention.**
+///
+/// Both directions of the trade-off the retention work exposed, now measured
+/// across the frozen matrix rather than one ad-hoc run. Asserted as ordering.
+#[test]
+fn the_retention_envelope_has_the_measured_shape() {
+    let directory = TempDir::new().unwrap();
+
+    let unbounded = run(directory.path(), WORKLOADS[0], DurabilityMode::Flushed);
+    let keep5 = run(directory.path(), WORKLOADS[1], DurabilityMode::Flushed);
+    let keep50 = run(directory.path(), WORKLOADS[2], DurabilityMode::Flushed);
+
+    assert_eq!(unbounded.revisions, 301, "unbounded keeps everything");
+    assert_eq!(keep5.revisions, 5);
+    assert_eq!(keep50.revisions, 50);
+
+    assert!(
+        keep50.elapsed > keep5.elapsed,
+        "a wider window costs more per write: keep50 {:?} vs keep5 {:?}",
+        keep50.elapsed,
+        keep5.elapsed
+    );
+    assert!(
+        keep5.log_bytes > unbounded.log_bytes,
+        "expiry tombstones inflate the log before compaction: {} vs {}",
+        keep5.log_bytes,
+        unbounded.log_bytes
+    );
+}
+
+/// Spreading writes over many resources does not change what is stored per
+/// write, and history stays per-resource.
+#[test]
+fn many_resources_keep_independent_bounded_histories() {
+    let directory = TempDir::new().unwrap();
+    let measurement = run(directory.path(), WORKLOADS[3], DurabilityMode::Flushed);
+    assert_eq!(
+        measurement.revisions, 350,
+        "50 initial writes plus 300 updates, spread evenly"
+    );
+}
+
+/// Replaying a database costs time proportional to what it holds, and recovery
+/// is part of the envelope an operator has to size for.
+#[test]
+fn restart_cost_is_measurable_and_bounded_by_retention() {
+    let directory = TempDir::new().unwrap();
+
+    // Two databases with the same writes and different retention.
+    let unbounded = run(directory.path(), WORKLOADS[0], DurabilityMode::Flushed);
+    let bounded = run(directory.path(), WORKLOADS[1], DurabilityMode::Flushed);
+
+    let reopen = |name: &str| {
+        let path = directory
+            .path()
+            .join(format!("{name}-{:?}.log", DurabilityMode::Flushed));
+        let start = Instant::now();
+        let db = FeltDb::open(&path).unwrap();
+        let elapsed = start.elapsed();
+        drop(db);
+        elapsed
+    };
+
+    let _ = reopen("sequential-unbounded");
+    let _ = reopen("sequential-keep5");
+
+    // The claim is that both reopen, and that retention bounds what has to be
+    // held afterwards — not a timing threshold, which would be noise.
+    assert!(unbounded.revisions > bounded.revisions * 10);
+}
+
+/// The whole frozen matrix runs under every mode, which is what makes the
+/// recorded table reproducible rather than a set of one-off numbers.
+#[test]
+fn the_frozen_matrix_runs_under_every_durability_mode() {
+    let directory = TempDir::new().unwrap();
+    let mut rows = 0;
+
+    for workload in WORKLOADS {
+        for mode in [
+            DurabilityMode::Flushed,
+            DurabilityMode::Grouped { every: 32 },
+            DurabilityMode::Synced,
+        ] {
+            let measurement = run(directory.path(), *workload, mode);
+            assert!(measurement.writes_per_second(workload.writes) > 0.0);
+            assert!(measurement.log_bytes > 0);
+            rows += 1;
+        }
+    }
+
+    assert_eq!(rows, WORKLOADS.len() * 3, "every cell of the matrix ran");
+}
+
+/// A measured number is not a capacity claim, and the harness records the
+/// workload alongside it so it cannot be quoted without one.
+#[test]
+fn a_measurement_carries_the_workload_that_produced_it() {
+    let directory = TempDir::new().unwrap();
+    let workload = WORKLOADS[1];
+    let measurement = run(directory.path(), workload, DurabilityMode::Flushed);
+
+    // Everything needed to reproduce or dispute the number.
+    assert_eq!(workload.writes, 300);
+    assert_eq!(workload.resources, 1);
+    assert_eq!(workload.payload_bytes, 100);
+    assert_eq!(workload.retention, Some(5));
+    assert!(measurement.writes_per_second(workload.writes).is_finite());
+}

package/dist/create/server-source/crates/feltdb-server/src/app_state.rs

@@ -31,7 +31,7 @@ use feltdb::{
     sync_contract::SyncStore,
     worker_mesh::WorkerMeshStore,
     workload::WorkloadStore,
-    FeltDb,
+    FeltDb, StateTriggerStore,
 };
 
 #[derive(Clone)]
@@ -41,6 +41,7 @@ pub struct AppState {
     pub namespace: Arc<str>,
     pub ids: Arc<AtomicU64>,
     pub auth_enabled: bool,
+    pub authentication_workers: Arc<tokio::sync::Semaphore>,
     pub keys: KeyStore,
     pub metrics: Metrics,
     pub cluster: ClusterStore,
@@ -72,7 +73,21 @@ pub struct AppState {
     pub grants: Arc<std::sync::Mutex<GrantStore>>,
     pub sync: Arc<std::sync::Mutex<SyncStore>>,
     pub workloads: Arc<std::sync::Mutex<WorkloadStore>>,
+    pub state_triggers: Arc<std::sync::Mutex<StateTriggerStore>>,
     pub mesh: Arc<std::sync::Mutex<WorkerMeshStore>>,
     pub readiness_probe: Arc<PathBuf>,
     pub bounded_query_cursors: Arc<std::sync::Mutex<HashMap<String, BoundedQueryCursor>>>,
+    /// Bounds how many requests may be in flight at once.
+    ///
+    /// Every read and write serializes on one lock inside the database, so
+    /// admitting unbounded concurrency does not increase throughput — it only
+    /// converts a queue the server could reject quickly into one it holds until
+    /// clients give up. Shedding at the door turns a pile-up into a fast 503
+    /// with `Retry-After`.
+    pub admission: Arc<tokio::sync::Semaphore>,
+    /// How long a request may take before the server abandons it.
+    ///
+    /// Without this a slow operation runs until the *client* times out, and the
+    /// work keeps running after the client has gone.
+    pub request_deadline: std::time::Duration,
 }