@feltdb/core 0.11.9 → 0.12.1

package/dist/create/server-source/crates/feltdb/tests/state_ref_authority.rs

@@ -0,0 +1,918 @@
+//! State-model refs and garbage collection under concurrency, restart and
+//! process death.
+//!
+//! These are regression tests for the defects measured by the research probe
+//! (`docs/audits/enroute/probe/`) and decided in
+//! `adrs/state-ref-authority-and-safe-reclamation.md`:
+//!
+//! - P1: concurrent ref CAS across `StateStore` instances admitted several
+//!   winners;
+//! - P2: GC with no refs deleted every revision;
+//! - P3: GC racing a promotion left a ref naming a deleted revision;
+//! - P4: a ref move was a flush-only single-row write.
+//!
+//! Everything here uses the durable backing (`FeltDb`), because that is where
+//! the defects were. Race tests repeat an interleaving many times. They detect
+//! a broken implementation with high probability; they do not prove a correct
+//! one. The argument for correctness is the transaction ordering documented in
+//! `state_model.rs`, and these tests check that the code behaves as argued.
+
+use feltdb::state_model::{RefError, StateRef, StateRevision, StateStore};
+use feltdb::{FeltDBStateSystem, FeltDb, StateId};
+use serde_json::{json, Value};
+use std::collections::BTreeMap;
+use std::io::{BufRead, BufReader};
+use std::path::{Path, PathBuf};
+use std::process::{Command, Stdio};
+use std::sync::{Arc, Barrier};
+use std::thread;
+use tempfile::TempDir;
+
+// ---------------------------------------------------------------------------
+// helpers
+// ---------------------------------------------------------------------------
+
+fn open(path: &Path) -> Arc<FeltDb> {
+    Arc::new(FeltDb::open(path).expect("open"))
+}
+
+fn store(db: &Arc<FeltDb>) -> StateStore {
+    FeltDBStateSystem::create_store(db).expect("store")
+}
+
+fn fresh() -> (TempDir, PathBuf, Arc<FeltDb>) {
+    let dir = TempDir::new().unwrap();
+    let path = dir.path().join("state.log");
+    let db = open(&path);
+    (dir, path, db)
+}
+
+fn no_metadata() -> BTreeMap<String, Value> {
+    BTreeMap::new()
+}
+
+fn log_lines(path: &Path) -> Vec<String> {
+    std::fs::read_to_string(path)
+        .unwrap_or_default()
+        .lines()
+        .map(str::to_string)
+        .collect()
+}
+
+/// A chain `base(0) → … → head(len-1)` on `resource`, committed normally.
+fn chain(store: &StateStore, resource: &str, len: usize) -> Vec<StateRevision> {
+    let mut revisions = vec![store
+        .create(resource, json!({"n": 0}).to_string(), "app".into())
+        .unwrap()];
+    for n in 1..len {
+        let parent = revisions.last().unwrap().clone();
+        revisions.push(
+            store
+                .commit(json!({"n": n}).to_string(), &parent, "app".into())
+                .unwrap(),
+        );
+    }
+    revisions
+}
+
+/// A child of `parent` written as a durable row *as an old revision*: its
+/// timestamp is two hours in the past, so it is outside any reasonable GC grace
+/// period. The identity does not cover the timestamp, so integrity still holds.
+fn old_child(db: &FeltDb, parent: &StateRevision, content: Value) -> StateRevision {
+    let mut revision = StateRevision::child(content.to_string(), parent, "app".into());
+    revision.timestamp_ms = revision.timestamp_ms.saturating_sub(2 * 60 * 60 * 1000);
+    assert!(revision.verify_integrity());
+    db.insert(&format!("state:revision:{}", revision.id.as_hex()), &revision)
+        .unwrap();
+    revision
+}
+
+/// A ref row exactly as the pre-transactional implementation wrote it.
+fn legacy_ref(db: &FeltDb, namespace: &str, name: &str, target: &StateId, version: u64) {
+    let reference = StateRef {
+        namespace: namespace.into(),
+        name: name.into(),
+        target: target.clone(),
+        version,
+        authority: "legacy".into(),
+        updated_at_ms: 1,
+        metadata: BTreeMap::new(),
+    };
+    db.insert(&format!("state:ref:{namespace}:{name}"), &reference)
+        .unwrap();
+}
+
+fn every_ref_target_exists(store: &StateStore) -> bool {
+    store
+        .list_refs(None)
+        .iter()
+        .all(|reference| store.exists(&reference.target))
+}
+
+// ---------------------------------------------------------------------------
+// 1. Concurrent CAS has exactly one winner (P1)
+// ---------------------------------------------------------------------------
+
+const RACE_ITERATIONS: usize = 1_000;
+const RACERS: usize = 8;
+
+fn cas_race(stores: Vec<StateStore>) {
+    let setup = stores[0].clone();
+    let base = setup
+        .create("doc:race", json!({"v": 0}).to_string(), "app".into())
+        .unwrap();
+    let targets: Vec<StateId> = (0..RACERS)
+        .map(|i| {
+            setup
+                .commit(json!({"v": i + 1}).to_string(), &base, "app".into())
+                .unwrap()
+                .id
+        })
+        .collect();
+
+    for iteration in 0..RACE_ITERATIONS {
+        let name = format!("r{iteration}");
+        setup
+            .create_ref("race", &name, base.id.clone(), "app".into(), no_metadata())
+            .unwrap();
+        let barrier = Arc::new(Barrier::new(RACERS));
+        let handles: Vec<_> = (0..RACERS)
+            .map(|i| {
+                let store = stores[i % stores.len()].clone();
+                let barrier = barrier.clone();
+                let target = targets[i].clone();
+                let name = name.clone();
+                thread::spawn(move || {
+                    barrier.wait();
+                    store
+                        .compare_and_swap_ref("race", &name, 1, target.clone(), format!("w{i}"), None)
+                        .map(|_| target)
+                })
+            })
+            .collect();
+        let outcomes: Vec<_> = handles.into_iter().map(|h| h.join().unwrap()).collect();
+        let winners: Vec<&StateId> = outcomes.iter().filter_map(|o| o.as_ref().ok()).collect();
+        assert_eq!(
+            winners.len(),
+            1,
+            "iteration {iteration}: exactly one CAS from version 1 may succeed, got {}",
+            winners.len()
+        );
+        for outcome in &outcomes {
+            if let Err(error) = outcome {
+                assert!(
+                    matches!(error, RefError::VersionConflict { expected: 1, current: 2 }),
+                    "iteration {iteration}: a loser must see a version conflict, got {error:?}"
+                );
+            }
+        }
+        let after = setup.get_ref("race", &name).unwrap();
+        assert_eq!(after.version, 2, "iteration {iteration}: one advance");
+        assert_eq!(&after.target, winners[0], "iteration {iteration}: the winner's target");
+    }
+}
+
+/// The server builds a `StateStore` per request; each has its own instance.
+#[test]
+fn ref_cas_has_one_winner_across_store_instances() {
+    let (_dir, _path, db) = fresh();
+    cas_race((0..RACERS).map(|_| store(&db)).collect());
+}
+
+/// Two independently opened handles on one path, each with its own stores.
+#[test]
+fn ref_cas_has_one_winner_across_open_handles() {
+    let (_dir, path, one) = fresh();
+    let two = open(&path);
+    let stores = vec![store(&one), store(&two), store(&one), store(&two)];
+    cas_race(stores);
+    // Both handles observe the same authority, not two copies.
+    assert_eq!(
+        store(&one).get_ref("race", "r0"),
+        store(&two).get_ref("race", "r0")
+    );
+}
+
+#[test]
+fn concurrent_ref_create_has_one_winner() {
+    let (_dir, _path, db) = fresh();
+    let setup = store(&db);
+    let targets: Vec<StateId> = (0..RACERS)
+        .map(|i| {
+            setup
+                .create(&format!("doc:c{i}"), json!({"i": i}).to_string(), "app".into())
+                .unwrap()
+                .id
+        })
+        .collect();
+    for iteration in 0..200 {
+        let name = format!("c{iteration}");
+        let barrier = Arc::new(Barrier::new(RACERS));
+        let handles: Vec<_> = (0..RACERS)
+            .map(|i| {
+                let store = store(&db);
+                let barrier = barrier.clone();
+                let target = targets[i].clone();
+                let name = name.clone();
+                thread::spawn(move || {
+                    barrier.wait();
+                    store.create_ref("create", &name, target, format!("w{i}"), BTreeMap::new())
+                })
+            })
+            .collect();
+        let outcomes: Vec<_> = handles.into_iter().map(|h| h.join().unwrap()).collect();
+        let created = outcomes.iter().filter(|o| o.is_ok()).count();
+        assert_eq!(created, 1, "iteration {iteration}: one creator wins");
+        assert!(outcomes
+            .iter()
+            .filter_map(|o| o.as_ref().err())
+            .all(|e| *e == RefError::AlreadyExists));
+        assert_eq!(setup.get_ref("create", &name).unwrap().version, 1);
+    }
+}
+
+// ---------------------------------------------------------------------------
+// 2. Missing targets and stale versions never become accepted state
+// ---------------------------------------------------------------------------
+
+#[test]
+fn missing_targets_and_stale_versions_write_nothing() {
+    let (_dir, path, db) = fresh();
+    let store = store(&db);
+    let revisions = chain(&store, "doc:v", 2);
+    store
+        .create_ref("heads", "main", revisions[0].id.clone(), "app".into(), no_metadata())
+        .unwrap();
+    let before = log_lines(&path).len();
+    let missing = StateId::compute("never committed");
+
+    assert_eq!(
+        store.create_ref("heads", "other", missing.clone(), "app".into(), no_metadata()),
+        Err(RefError::UnknownTarget(missing.clone()))
+    );
+    assert_eq!(
+        store
+            .compare_and_swap_ref("heads", "main", 1, missing.clone(), "app".into(), None)
+            .unwrap_err(),
+        RefError::UnknownTarget(missing.clone())
+    );
+    assert_eq!(
+        store
+            .compare_and_swap_ref("heads", "main", 7, revisions[1].id.clone(), "app".into(), None)
+            .unwrap_err(),
+        RefError::VersionConflict { expected: 7, current: 1 }
+    );
+    assert_eq!(
+        store.delete_ref("heads", "main", Some(7)),
+        Err(RefError::VersionConflict { expected: 7, current: 1 })
+    );
+
+    assert_eq!(log_lines(&path).len(), before, "a refused ref operation appends nothing");
+    let main = store.get_ref("heads", "main").unwrap();
+    assert_eq!((main.version, main.target), (1, revisions[0].id.clone()));
+    assert!(store.get_ref("heads", "other").is_none());
+}
+
+// ---------------------------------------------------------------------------
+// 3. GC policy: history is roots and their ancestors (Option A)
+// ---------------------------------------------------------------------------
+
+/// P2: with no refs at all, GC must not touch a resource's history.
+#[test]
+fn gc_without_refs_keeps_every_head_and_its_ancestors() {
+    let (_dir, _path, db) = fresh();
+    let store = store(&db);
+    let revisions = chain(&store, "orders:1", 10);
+    let report = store.collect_unreachable().unwrap();
+    assert!(report.collected_revisions.is_empty(), "{report:?}");
+    for revision in &revisions {
+        assert!(store.exists(&revision.id), "revision {} was collected", revision.sequence);
+    }
+}
+
+/// Only old revisions that are neither a ref target, nor a resource's latest
+/// revision, nor an ancestor of either, are collected.
+#[test]
+fn gc_collects_exactly_the_old_unreachable_forks() {
+    let (_dir, _path, db) = fresh();
+    let store = store(&db);
+    // main line: 0 → 1 → 2 → 3, latest is 3
+    let main = chain(&store, "doc:g", 4);
+    // an old abandoned fork off 0: f1 (seq 1) → f2 (seq 2)
+    let f1 = old_child(&db, &main[0], json!({"fork": 1}));
+    let f2 = old_child(&db, &f1, json!({"fork": 2}));
+    // an old fork that a ref still names, with an old parent of its own
+    let kept_parent = old_child(&db, &main[0], json!({"kept": 1}));
+    let kept = old_child(&db, &kept_parent, json!({"kept": 2}));
+    store
+        .create_ref("candidates", "kept", kept.id.clone(), "app".into(), no_metadata())
+        .unwrap();
+    // a young fork, committed normally just now
+    let young = store
+        .commit(json!({"young": 1}).to_string(), &main[0], "app".into())
+        .unwrap();
+    // a second resource with no refs at all
+    let other = chain(&store, "doc:h", 3);
+
+    let report = store.collect_unreachable().unwrap();
+    let mut collected = report.collected_revisions.clone();
+    collected.sort();
+    let mut expected = vec![f1.id.clone(), f2.id.clone()];
+    expected.sort();
+    assert_eq!(collected, expected, "only the old, unreferenced, non-latest fork");
+
+    for kept in main
+        .iter()
+        .chain(other.iter())
+        .chain([&kept_parent, &kept, &young])
+    {
+        assert!(store.exists(&kept.id), "{} must survive", kept.id);
+    }
+    // Idempotent: nothing more to collect.
+    assert!(store.collect_unreachable().unwrap().collected_revisions.is_empty());
+}
+
+// ---------------------------------------------------------------------------
+// 4. GC against concurrent promotion and ref changes (P3)
+// ---------------------------------------------------------------------------
+
+/// A promotion of an old, collectable candidate races GC. Either GC wins and
+/// the promotion is refused, or the promotion wins and the candidate survives.
+/// Never a ref naming a deleted revision, and never lost latest history.
+#[test]
+fn gc_racing_promotion_never_dangles() {
+    let mut promoted = 0;
+    let mut refused = 0;
+    for iteration in 0..300 {
+        let (_dir, _path, db) = fresh();
+        let setup = store(&db);
+        let main = chain(&setup, "doc:p", 3);
+        let candidate = old_child(&db, &main[0], json!({"candidate": iteration}));
+        setup
+            .create_ref("heads", "main", main[0].id.clone(), "app".into(), no_metadata())
+            .unwrap();
+
+        let barrier = Arc::new(Barrier::new(2));
+        let promoter = {
+            let store = store(&db);
+            let barrier = barrier.clone();
+            let target = candidate.id.clone();
+            thread::spawn(move || {
+                barrier.wait();
+                store.compare_and_swap_ref("heads", "main", 1, target, "agent".into(), None)
+            })
+        };
+        let collector = {
+            let store = store(&db);
+            let barrier = barrier.clone();
+            thread::spawn(move || {
+                barrier.wait();
+                store.collect_unreachable()
+            })
+        };
+        let promotion = promoter.join().unwrap();
+        let _ = collector.join().unwrap();
+
+        let check = store(&db);
+        assert!(every_ref_target_exists(&check), "iteration {iteration}: dangling ref");
+        for revision in &main {
+            assert!(check.exists(&revision.id), "iteration {iteration}: latest history lost");
+        }
+        match promotion {
+            Ok(_) => {
+                promoted += 1;
+                assert!(check.exists(&candidate.id));
+            }
+            Err(RefError::UnknownTarget(_)) => {
+                refused += 1;
+                assert_eq!(check.get_ref("heads", "main").unwrap().version, 1);
+            }
+            Err(other) => panic!("iteration {iteration}: unexpected {other:?}"),
+        }
+    }
+    assert_eq!(promoted + refused, 300);
+    // Report both sides, so a run that never exercised one of them is visible.
+    eprintln!("gc_racing_promotion_never_dangles: promoted={promoted} refused={refused}");
+}
+
+/// Refs are created, moved and deleted continuously, each round onto a freshly
+/// written old (collectable) revision, while GC runs continuously. No ref ever
+/// names a missing revision, and both sides make progress.
+#[test]
+fn gc_racing_ref_changes_never_dangles() {
+    let (_dir, _path, db) = fresh();
+    let setup = store(&db);
+    // Long enough that the old revisions (sequence 1) are not the resource's
+    // latest, so they are collectable whenever no ref names them.
+    let main = chain(&setup, "doc:r", 4);
+
+    let writer = {
+        let db = db.clone();
+        let store = store(&db);
+        let base = main[0].clone();
+        thread::spawn(move || {
+            let mut applied = 0;
+            let mut refused = 0;
+            for round in 0..600 {
+                let candidate = old_child(&db, &base, json!({"round": round}));
+                let name = format!("n{}", round % 8);
+                let outcome = match store.get_ref("live", &name) {
+                    None => store
+                        .create_ref("live", &name, candidate.id, "w".into(), BTreeMap::new())
+                        .map(|_| ()),
+                    Some(current) if round % 5 == 0 => {
+                        store.delete_ref("live", &name, Some(current.version))
+                    }
+                    Some(current) => store
+                        .compare_and_swap_ref("live", &name, current.version, candidate.id, "w".into(), None)
+                        .map(|_| ()),
+                };
+                match outcome {
+                    Ok(()) => applied += 1,
+                    Err(RefError::UnknownTarget(_)) => refused += 1,
+                    Err(other) => panic!("round {round}: unexpected {other:?}"),
+                }
+            }
+            (applied, refused)
+        })
+    };
+    let collector = {
+        let store = store(&db);
+        thread::spawn(move || {
+            let mut collected = 0;
+            for _ in 0..200 {
+                if let Ok(report) = store.collect_unreachable() {
+                    collected += report.collected_revisions.len();
+                }
+                assert!(every_ref_target_exists(&store), "dangling ref observed mid-run");
+            }
+            collected
+        })
+    };
+    let (applied, refused) = writer.join().unwrap();
+    let collected = collector.join().unwrap();
+    eprintln!(
+        "gc_racing_ref_changes_never_dangles: ref writes applied={applied} \
+         refused_unknown_target={refused} revisions collected={collected}"
+    );
+    assert!(applied > 0, "the writer must have made progress");
+    assert!(collected > 0, "the collector must actually have collected");
+    let check = store(&db);
+    assert!(every_ref_target_exists(&check));
+    assert!(main.iter().all(|r| check.exists(&r.id)));
+}
+
+// ---------------------------------------------------------------------------
+// 5. Retry, restart, compaction, recreation
+// ---------------------------------------------------------------------------
+
+/// A CAS whose response was lost is retried with the same request after the
+/// process restarted and the log was compacted: the retry succeeds and does
+/// not advance the ref a second time.
+#[test]
+fn cas_retry_after_success_is_idempotent_across_restart_and_compaction() {
+    let (_dir, path, db) = fresh();
+    let (a, b) = {
+        let store = store(&db);
+        let revisions = chain(&store, "doc:i", 2);
+        store
+            .create_ref("heads", "main", revisions[0].id.clone(), "app".into(), no_metadata())
+            .unwrap();
+        let moved = store
+            .compare_and_swap_ref("heads", "main", 1, revisions[1].id.clone(), "app".into(), None)
+            .unwrap();
+        assert_eq!(moved.version, 2);
+        (revisions[0].id.clone(), revisions[1].id.clone())
+    };
+    drop(db);
+
+    let db = open(&path);
+    let retried = store(&db)
+        .compare_and_swap_ref("heads", "main", 1, b.clone(), "app".into(), None)
+        .expect("an identical retry after restart is the same request");
+    assert_eq!((retried.version, retried.target.clone()), (2, b.clone()));
+    assert_eq!(store(&db).get_ref("heads", "main").unwrap().version, 2);
+
+    let versions = db.operation_versions().unwrap();
+    db.acknowledge_peer_versions("peer".into(), versions).unwrap();
+    db.compact_operation_log(&["peer".into()]).unwrap();
+    drop(db);
+
+    let db = open(&path);
+    let store = store(&db);
+    let again = store
+        .compare_and_swap_ref("heads", "main", 1, b.clone(), "app".into(), None)
+        .expect("still the same request after compaction");
+    assert_eq!(again.version, 2);
+    // A different request for the same version slot is a conflict, not a replay.
+    assert_eq!(
+        store
+            .compare_and_swap_ref("heads", "main", 1, a, "app".into(), None)
+            .unwrap_err(),
+        RefError::VersionConflict { expected: 1, current: 2 }
+    );
+    assert_eq!(store.get_ref("heads", "main").unwrap().target, b);
+}
+
+#[test]
+fn create_and_delete_retries_are_idempotent_across_restart() {
+    let (_dir, path, db) = fresh();
+    let (a, b) = {
+        let store = store(&db);
+        let revisions = chain(&store, "doc:cd", 2);
+        store
+            .create_ref("heads", "main", revisions[0].id.clone(), "app".into(), no_metadata())
+            .unwrap();
+        (revisions[0].id.clone(), revisions[1].id.clone())
+    };
+    drop(db);
+
+    let db = open(&path);
+    let store_ = store(&db);
+    let created = store_
+        .create_ref("heads", "main", a.clone(), "app".into(), no_metadata())
+        .expect("identical create retried after restart");
+    assert_eq!(created.version, 1);
+    assert_eq!(
+        store_.create_ref("heads", "main", b.clone(), "app".into(), no_metadata()),
+        Err(RefError::AlreadyExists),
+        "a different create of an existing ref"
+    );
+    store_.delete_ref("heads", "main", Some(1)).unwrap();
+    drop(store_);
+    drop(db);
+
+    let db = open(&path);
+    let store = store(&db);
+    store
+        .delete_ref("heads", "main", Some(1))
+        .expect("an identical delete retried after restart");
+    assert!(store.get_ref("heads", "main").is_none());
+}
+
+/// After delete and recreate, the ref is back at version 1. A CAS from 1 on the
+/// new ref is a new request, never a replay of the first ref's CAS from 1.
+#[test]
+fn a_recreated_ref_does_not_replay_an_old_transaction() {
+    let (_dir, path, db) = fresh();
+    let (a, b) = {
+        let store = store(&db);
+        let revisions = chain(&store, "doc:rc", 2);
+        let (a, b) = (revisions[0].id.clone(), revisions[1].id.clone());
+        store.create_ref("heads", "main", a.clone(), "app".into(), no_metadata()).unwrap();
+        store
+            .compare_and_swap_ref("heads", "main", 1, b.clone(), "app".into(), None)
+            .unwrap();
+        store.delete_ref("heads", "main", Some(2)).unwrap();
+        (a, b)
+    };
+    drop(db);
+
+    let db = open(&path);
+    let store_ = store(&db);
+    let recreated = store_
+        .create_ref("heads", "main", a.clone(), "app".into(), no_metadata())
+        .unwrap();
+    assert_eq!((recreated.version, recreated.target), (1, a.clone()));
+    let moved = store_
+        .compare_and_swap_ref("heads", "main", 1, b.clone(), "app".into(), None)
+        .unwrap();
+    assert_eq!(moved.version, 2);
+    let current = store_.get_ref("heads", "main").unwrap();
+    assert_eq!((current.version, current.target), (2, b), "applied, not replayed");
+    drop(store_);
+    drop(db);
+
+    let db = open(&path);
+    assert_eq!(store(&db).get_ref("heads", "main").unwrap().version, 2);
+}
+
+/// Refs written by the pre-transactional implementation remain usable.
+#[test]
+fn legacy_ref_rows_remain_casable_across_restart() {
+    let (_dir, path, db) = fresh();
+    let (a, b) = {
+        let store = store(&db);
+        let revisions = chain(&store, "doc:l", 2);
+        (revisions[0].id.clone(), revisions[1].id.clone())
+    };
+    legacy_ref(&db, "heads", "one", &a, 1);
+    legacy_ref(&db, "heads", "three", &a, 3);
+    drop(db);
+
+    let db = open(&path);
+    let store_ = store(&db);
+    assert_eq!(store_.get_ref("heads", "three").unwrap().version, 3);
+    assert_eq!(
+        store_
+            .compare_and_swap_ref("heads", "three", 1, b.clone(), "app".into(), None)
+            .unwrap_err(),
+        RefError::VersionConflict { expected: 1, current: 3 }
+    );
+    assert_eq!(
+        store_
+            .compare_and_swap_ref("heads", "one", 1, b.clone(), "app".into(), None)
+            .unwrap()
+            .version,
+        2
+    );
+    assert_eq!(
+        store_
+            .compare_and_swap_ref("heads", "three", 3, b.clone(), "app".into(), None)
+            .unwrap()
+            .version,
+        4
+    );
+    store_.delete_ref("heads", "one", Some(2)).unwrap();
+    drop(store_);
+    drop(db);
+
+    let db = open(&path);
+    let store = store(&db);
+    assert!(store.get_ref("heads", "one").is_none());
+    let three = store.get_ref("heads", "three").unwrap();
+    assert_eq!((three.version, three.target), (4, b));
+}
+
+// ---------------------------------------------------------------------------
+// 6. Durability and atomicity of the write itself (P4)
+// ---------------------------------------------------------------------------
+
+fn record_type(line: &str) -> Option<String> {
+    serde_json::from_str::<Value>(line)
+        .ok()?
+        .get("record_type")?
+        .as_str()
+        .map(str::to_string)
+}
+
+/// A ref move is one durable transaction record, which `FeltDb` fsyncs before
+/// returning (`append_transaction` → `sync_data`).
+#[test]
+fn a_ref_move_is_one_transaction_record() {
+    let (_dir, path, db) = fresh();
+    let store = store(&db);
+    let revisions = chain(&store, "doc:d", 2);
+    store
+        .create_ref("heads", "main", revisions[0].id.clone(), "app".into(), no_metadata())
+        .unwrap();
+    let before = log_lines(&path);
+    store
+        .compare_and_swap_ref("heads", "main", 1, revisions[1].id.clone(), "app".into(), None)
+        .unwrap();
+    let after = log_lines(&path);
+    assert_eq!(after.len(), before.len() + 1, "one record per ref move");
+    assert_eq!(
+        record_type(after.last().unwrap()).as_deref(),
+        Some("feltdb.transaction.v1"),
+        "the ref move is a transaction record"
+    );
+}
+
+/// A committed revision and its content object are one transaction record.
+#[test]
+fn a_revision_commit_is_one_transaction_record() {
+    let (_dir, path, db) = fresh();
+    let store = store(&db);
+    let base = store
+        .create("doc:o", json!({"v": 0}).to_string(), "app".into())
+        .unwrap();
+    let before = log_lines(&path);
+    store
+        .commit(json!({"v": 1}).to_string(), &base, "app".into())
+        .unwrap();
+    let after = log_lines(&path);
+    assert_eq!(after.len(), before.len() + 1, "object and revision in one record");
+    assert_eq!(
+        record_type(after.last().unwrap()).as_deref(),
+        Some("feltdb.transaction.v1")
+    );
+}
+
+/// Truncate the log at every byte inside the ref move's record: the database
+/// always opens, and the ref is either entirely before or entirely after.
+#[test]
+fn a_ref_move_is_all_or_nothing_at_every_crash_point() {
+    let (_dir, path, db) = fresh();
+    let (a, b) = {
+        let store = store(&db);
+        let revisions = chain(&store, "doc:t", 2);
+        store
+            .create_ref("heads", "main", revisions[0].id.clone(), "app".into(), no_metadata())
+            .unwrap();
+        (revisions[0].id.clone(), revisions[1].id.clone())
+    };
+    let prefix = std::fs::read(&path).unwrap().len();
+    store(&db)
+        .compare_and_swap_ref("heads", "main", 1, b.clone(), "app".into(), None)
+        .unwrap();
+    drop(db);
+    let full = std::fs::read(&path).unwrap();
+
+    let scratch = TempDir::new().unwrap();
+    for cut in prefix..=full.len() {
+        let copy = scratch.path().join(format!("cut-{cut}.log"));
+        std::fs::write(&copy, &full[..cut]).unwrap();
+        let db = Arc::new(FeltDb::open(&copy).unwrap_or_else(|e| panic!("cut {cut}: {e:?}")));
+        let store = store(&db);
+        let main = store.get_ref("heads", "main").expect("the ref survives");
+        let state = (main.version, main.target.clone());
+        assert!(
+            state == (1, a.clone()) || state == (2, b.clone()),
+            "cut {cut}: torn ref {state:?}"
+        );
+        assert!(store.exists(&main.target), "cut {cut}: dangling ref");
+        drop(store);
+        drop(db);
+        std::fs::remove_file(&copy).ok();
+    }
+}
+
+// ---------------------------------------------------------------------------
+// 7. Real process death
+// ---------------------------------------------------------------------------
+
+const CHILD: &str = "FELTDB_REF_AUTHORITY_CHILD";
+const CHILD_PATH: &str = "FELTDB_REF_AUTHORITY_PATH";
+
+/// The value after `marker` on any line. The test harness prints
+/// `test <name> ... ` without a newline before the child's own output, so the
+/// first marker can land mid-line.
+fn marked(line: &str, marker: &str) -> Option<String> {
+    line.find(marker)
+        .map(|at| line[at + marker.len()..].trim().to_string())
+}
+
+fn child_command(test: &str, path: &Path, mode: &str) -> Command {
+    let mut command = Command::new(std::env::current_exe().unwrap());
+    command
+        .args([test, "--exact", "--nocapture", "--test-threads=1"])
+        .env(CHILD, mode)
+        .env(CHILD_PATH, path);
+    command
+}
+
+/// The child acknowledges a CAS and dies by `abort()` before its caller could
+/// have seen the result. The restarted caller retries the same request: it is
+/// accepted as the request it already was, not applied twice.
+#[test]
+fn a_cas_acknowledged_before_abort_is_retried_idempotently() {
+    if std::env::var(CHILD).as_deref() == Ok("ack-then-abort") {
+        let path = PathBuf::from(std::env::var(CHILD_PATH).unwrap());
+        let db = open(&path);
+        let store = store(&db);
+        let revisions = chain(&store, "doc:k", 2);
+        store
+            .create_ref("heads", "main", revisions[0].id.clone(), "app".into(), no_metadata())
+            .unwrap();
+        store
+            .compare_and_swap_ref("heads", "main", 1, revisions[1].id.clone(), "app".into(), None)
+            .unwrap();
+        println!("ACK {}", revisions[1].id.as_hex());
+        std::process::abort();
+    }
+
+    let dir = TempDir::new().unwrap();
+    let path = dir.path().join("abort.log");
+    let output = child_command(
+        "a_cas_acknowledged_before_abort_is_retried_idempotently",
+        &path,
+        "ack-then-abort",
+    )
+    .output()
+    .unwrap();
+    assert!(!output.status.success(), "the child must have died");
+    let stdout = String::from_utf8_lossy(&output.stdout);
+    let acked = stdout
+        .lines()
+        .find_map(|line| marked(line, "ACK "))
+        .expect("the child acknowledged before dying");
+    let target = StateId::from_hex(acked);
+
+    let db = open(&path);
+    let store = store(&db);
+    let current = store.get_ref("heads", "main").unwrap();
+    assert_eq!((current.version, current.target.clone()), (2, target.clone()));
+    let retried = store
+        .compare_and_swap_ref("heads", "main", 1, target.clone(), "app".into(), None)
+        .expect("the retry of an acknowledged CAS is accepted");
+    assert_eq!(retried.version, 2);
+    assert_eq!(store.get_ref("heads", "main").unwrap().version, 2, "not applied twice");
+}
+
+/// The child commits a revision and dies before promoting it. After restart the
+/// revision is durable, the ref is unchanged, and the promotion can complete.
+#[test]
+fn death_between_commit_and_promotion_leaves_a_durable_unpromoted_revision() {
+    if std::env::var(CHILD).as_deref() == Ok("commit-then-abort") {
+        let path = PathBuf::from(std::env::var(CHILD_PATH).unwrap());
+        let db = open(&path);
+        let store = store(&db);
+        let base = store
+            .create("doc:m", json!({"v": 0}).to_string(), "app".into())
+            .unwrap();
+        store
+            .create_ref("heads", "main", base.id.clone(), "app".into(), no_metadata())
+            .unwrap();
+        let next = store
+            .commit(json!({"v": 1}).to_string(), &base, "app".into())
+            .unwrap();
+        println!("COMMITTED {}", next.id.as_hex());
+        std::process::abort();
+    }
+
+    let dir = TempDir::new().unwrap();
+    let path = dir.path().join("abort.log");
+    let output = child_command(
+        "death_between_commit_and_promotion_leaves_a_durable_unpromoted_revision",
+        &path,
+        "commit-then-abort",
+    )
+    .output()
+    .unwrap();
+    assert!(!output.status.success());
+    let stdout = String::from_utf8_lossy(&output.stdout);
+    let committed = StateId::from_hex(
+        stdout
+            .lines()
+            .find_map(|line| marked(line, "COMMITTED "))
+            .expect("the child committed before dying"),
+    );
+
+    let db = open(&path);
+    let store = store(&db);
+    assert!(store.exists(&committed), "the committed revision is durable");
+    assert_eq!(store.get_ref("heads", "main").unwrap().version, 1, "not promoted");
+    assert!(store.collect_unreachable().unwrap().collected_revisions.is_empty());
+    store
+        .compare_and_swap_ref("heads", "main", 1, committed.clone(), "app".into(), None)
+        .unwrap();
+    assert_eq!(store.get_ref("heads", "main").unwrap().target, committed);
+}
+
+/// The child moves refs continuously and reports each acknowledged move; the
+/// parent SIGKILLs it at an arbitrary point. Every acknowledged move survives,
+/// and the ref never names a missing revision.
+#[test]
+fn sigkill_during_ref_moves_keeps_every_acknowledged_move() {
+    if std::env::var(CHILD).as_deref() == Ok("move-forever") {
+        let path = PathBuf::from(std::env::var(CHILD_PATH).unwrap());
+        let db = open(&path);
+        let store = store(&db);
+        let base = store
+            .create("doc:s", json!({"v": 0}).to_string(), "app".into())
+            .unwrap();
+        store
+            .create_ref("heads", "main", base.id.clone(), "app".into(), no_metadata())
+            .unwrap();
+        let mut head = base;
+        let mut version = 1;
+        loop {
+            let next = store
+                .commit(json!({"v": version}).to_string(), &head, "app".into())
+                .unwrap();
+            let moved = store
+                .compare_and_swap_ref("heads", "main", version, next.id.clone(), "app".into(), None)
+                .unwrap();
+            version = moved.version;
+            println!("ACK {version}");
+            head = next;
+        }
+    }
+
+    let dir = TempDir::new().unwrap();
+    let path = dir.path().join("kill.log");
+    let mut child = child_command(
+        "sigkill_during_ref_moves_keeps_every_acknowledged_move",
+        &path,
+        "move-forever",
+    )
+    .stdout(Stdio::piped())
+    .stderr(Stdio::null())
+    .spawn()
+    .unwrap();
+    let mut last_acked = 0;
+    for line in BufReader::new(child.stdout.take().unwrap()).lines() {
+        let line = line.unwrap();
+        if let Some(version) = marked(&line, "ACK ") {
+            last_acked = version.parse::<u64>().unwrap();
+            if last_acked >= 40 {
+                child.kill().unwrap(); // SIGKILL
+                break;
+            }
+        }
+    }
+    child.wait().unwrap();
+    assert!(last_acked >= 40);
+
+    let db = open(&path);
+    let store = store(&db);
+    let main = store.get_ref("heads", "main").unwrap();
+    assert!(
+        main.version >= last_acked,
+        "acknowledged version {last_acked} lost; recovered {}",
+        main.version
+    );
+    assert!(store.exists(&main.target), "dangling ref after SIGKILL");
+    assert!(every_ref_target_exists(&store));
+}