@feltdb/core 0.13.0 → 0.16.0

package/dist/create/server-source/crates/feltdb/tests/state_ref_authority.rs

@@ -1,918 +0,0 @@
-//! State-model refs and garbage collection under concurrency, restart and
-//! process death.
-//!
-//! These are regression tests for the defects measured by the research probe
-//! (`docs/audits/enroute/probe/`) and decided in
-//! `adrs/state-ref-authority-and-safe-reclamation.md`:
-//!
-//! - P1: concurrent ref CAS across `StateStore` instances admitted several
-//!   winners;
-//! - P2: GC with no refs deleted every revision;
-//! - P3: GC racing a promotion left a ref naming a deleted revision;
-//! - P4: a ref move was a flush-only single-row write.
-//!
-//! Everything here uses the durable backing (`FeltDb`), because that is where
-//! the defects were. Race tests repeat an interleaving many times. They detect
-//! a broken implementation with high probability; they do not prove a correct
-//! one. The argument for correctness is the transaction ordering documented in
-//! `state_model.rs`, and these tests check that the code behaves as argued.
-
-use feltdb::state_model::{RefError, StateRef, StateRevision, StateStore};
-use feltdb::{FeltDBStateSystem, FeltDb, StateId};
-use serde_json::{json, Value};
-use std::collections::BTreeMap;
-use std::io::{BufRead, BufReader};
-use std::path::{Path, PathBuf};
-use std::process::{Command, Stdio};
-use std::sync::{Arc, Barrier};
-use std::thread;
-use tempfile::TempDir;
-
-// ---------------------------------------------------------------------------
-// helpers
-// ---------------------------------------------------------------------------
-
-fn open(path: &Path) -> Arc<FeltDb> {
-    Arc::new(FeltDb::open(path).expect("open"))
-}
-
-fn store(db: &Arc<FeltDb>) -> StateStore {
-    FeltDBStateSystem::create_store(db).expect("store")
-}
-
-fn fresh() -> (TempDir, PathBuf, Arc<FeltDb>) {
-    let dir = TempDir::new().unwrap();
-    let path = dir.path().join("state.log");
-    let db = open(&path);
-    (dir, path, db)
-}
-
-fn no_metadata() -> BTreeMap<String, Value> {
-    BTreeMap::new()
-}
-
-fn log_lines(path: &Path) -> Vec<String> {
-    std::fs::read_to_string(path)
-        .unwrap_or_default()
-        .lines()
-        .map(str::to_string)
-        .collect()
-}
-
-/// A chain `base(0) → … → head(len-1)` on `resource`, committed normally.
-fn chain(store: &StateStore, resource: &str, len: usize) -> Vec<StateRevision> {
-    let mut revisions = vec![store
-        .create(resource, json!({"n": 0}).to_string(), "app".into())
-        .unwrap()];
-    for n in 1..len {
-        let parent = revisions.last().unwrap().clone();
-        revisions.push(
-            store
-                .commit(json!({"n": n}).to_string(), &parent, "app".into())
-                .unwrap(),
-        );
-    }
-    revisions
-}
-
-/// A child of `parent` written as a durable row *as an old revision*: its
-/// timestamp is two hours in the past, so it is outside any reasonable GC grace
-/// period. The identity does not cover the timestamp, so integrity still holds.
-fn old_child(db: &FeltDb, parent: &StateRevision, content: Value) -> StateRevision {
-    let mut revision = StateRevision::child(content.to_string(), parent, "app".into());
-    revision.timestamp_ms = revision.timestamp_ms.saturating_sub(2 * 60 * 60 * 1000);
-    assert!(revision.verify_integrity());
-    db.insert(&format!("state:revision:{}", revision.id.as_hex()), &revision)
-        .unwrap();
-    revision
-}
-
-/// A ref row exactly as the pre-transactional implementation wrote it.
-fn legacy_ref(db: &FeltDb, namespace: &str, name: &str, target: &StateId, version: u64) {
-    let reference = StateRef {
-        namespace: namespace.into(),
-        name: name.into(),
-        target: target.clone(),
-        version,
-        authority: "legacy".into(),
-        updated_at_ms: 1,
-        metadata: BTreeMap::new(),
-    };
-    db.insert(&format!("state:ref:{namespace}:{name}"), &reference)
-        .unwrap();
-}
-
-fn every_ref_target_exists(store: &StateStore) -> bool {
-    store
-        .list_refs(None)
-        .iter()
-        .all(|reference| store.exists(&reference.target))
-}
-
-// ---------------------------------------------------------------------------
-// 1. Concurrent CAS has exactly one winner (P1)
-// ---------------------------------------------------------------------------
-
-const RACE_ITERATIONS: usize = 1_000;
-const RACERS: usize = 8;
-
-fn cas_race(stores: Vec<StateStore>) {
-    let setup = stores[0].clone();
-    let base = setup
-        .create("doc:race", json!({"v": 0}).to_string(), "app".into())
-        .unwrap();
-    let targets: Vec<StateId> = (0..RACERS)
-        .map(|i| {
-            setup
-                .commit(json!({"v": i + 1}).to_string(), &base, "app".into())
-                .unwrap()
-                .id
-        })
-        .collect();
-
-    for iteration in 0..RACE_ITERATIONS {
-        let name = format!("r{iteration}");
-        setup
-            .create_ref("race", &name, base.id.clone(), "app".into(), no_metadata())
-            .unwrap();
-        let barrier = Arc::new(Barrier::new(RACERS));
-        let handles: Vec<_> = (0..RACERS)
-            .map(|i| {
-                let store = stores[i % stores.len()].clone();
-                let barrier = barrier.clone();
-                let target = targets[i].clone();
-                let name = name.clone();
-                thread::spawn(move || {
-                    barrier.wait();
-                    store
-                        .compare_and_swap_ref("race", &name, 1, target.clone(), format!("w{i}"), None)
-                        .map(|_| target)
-                })
-            })
-            .collect();
-        let outcomes: Vec<_> = handles.into_iter().map(|h| h.join().unwrap()).collect();
-        let winners: Vec<&StateId> = outcomes.iter().filter_map(|o| o.as_ref().ok()).collect();
-        assert_eq!(
-            winners.len(),
-            1,
-            "iteration {iteration}: exactly one CAS from version 1 may succeed, got {}",
-            winners.len()
-        );
-        for outcome in &outcomes {
-            if let Err(error) = outcome {
-                assert!(
-                    matches!(error, RefError::VersionConflict { expected: 1, current: 2 }),
-                    "iteration {iteration}: a loser must see a version conflict, got {error:?}"
-                );
-            }
-        }
-        let after = setup.get_ref("race", &name).unwrap();
-        assert_eq!(after.version, 2, "iteration {iteration}: one advance");
-        assert_eq!(&after.target, winners[0], "iteration {iteration}: the winner's target");
-    }
-}
-
-/// The server builds a `StateStore` per request; each has its own instance.
-#[test]
-fn ref_cas_has_one_winner_across_store_instances() {
-    let (_dir, _path, db) = fresh();
-    cas_race((0..RACERS).map(|_| store(&db)).collect());
-}
-
-/// Two independently opened handles on one path, each with its own stores.
-#[test]
-fn ref_cas_has_one_winner_across_open_handles() {
-    let (_dir, path, one) = fresh();
-    let two = open(&path);
-    let stores = vec![store(&one), store(&two), store(&one), store(&two)];
-    cas_race(stores);
-    // Both handles observe the same authority, not two copies.
-    assert_eq!(
-        store(&one).get_ref("race", "r0"),
-        store(&two).get_ref("race", "r0")
-    );
-}
-
-#[test]
-fn concurrent_ref_create_has_one_winner() {
-    let (_dir, _path, db) = fresh();
-    let setup = store(&db);
-    let targets: Vec<StateId> = (0..RACERS)
-        .map(|i| {
-            setup
-                .create(&format!("doc:c{i}"), json!({"i": i}).to_string(), "app".into())
-                .unwrap()
-                .id
-        })
-        .collect();
-    for iteration in 0..200 {
-        let name = format!("c{iteration}");
-        let barrier = Arc::new(Barrier::new(RACERS));
-        let handles: Vec<_> = (0..RACERS)
-            .map(|i| {
-                let store = store(&db);
-                let barrier = barrier.clone();
-                let target = targets[i].clone();
-                let name = name.clone();
-                thread::spawn(move || {
-                    barrier.wait();
-                    store.create_ref("create", &name, target, format!("w{i}"), BTreeMap::new())
-                })
-            })
-            .collect();
-        let outcomes: Vec<_> = handles.into_iter().map(|h| h.join().unwrap()).collect();
-        let created = outcomes.iter().filter(|o| o.is_ok()).count();
-        assert_eq!(created, 1, "iteration {iteration}: one creator wins");
-        assert!(outcomes
-            .iter()
-            .filter_map(|o| o.as_ref().err())
-            .all(|e| *e == RefError::AlreadyExists));
-        assert_eq!(setup.get_ref("create", &name).unwrap().version, 1);
-    }
-}
-
-// ---------------------------------------------------------------------------
-// 2. Missing targets and stale versions never become accepted state
-// ---------------------------------------------------------------------------
-
-#[test]
-fn missing_targets_and_stale_versions_write_nothing() {
-    let (_dir, path, db) = fresh();
-    let store = store(&db);
-    let revisions = chain(&store, "doc:v", 2);
-    store
-        .create_ref("heads", "main", revisions[0].id.clone(), "app".into(), no_metadata())
-        .unwrap();
-    let before = log_lines(&path).len();
-    let missing = StateId::compute("never committed");
-
-    assert_eq!(
-        store.create_ref("heads", "other", missing.clone(), "app".into(), no_metadata()),
-        Err(RefError::UnknownTarget(missing.clone()))
-    );
-    assert_eq!(
-        store
-            .compare_and_swap_ref("heads", "main", 1, missing.clone(), "app".into(), None)
-            .unwrap_err(),
-        RefError::UnknownTarget(missing.clone())
-    );
-    assert_eq!(
-        store
-            .compare_and_swap_ref("heads", "main", 7, revisions[1].id.clone(), "app".into(), None)
-            .unwrap_err(),
-        RefError::VersionConflict { expected: 7, current: 1 }
-    );
-    assert_eq!(
-        store.delete_ref("heads", "main", Some(7)),
-        Err(RefError::VersionConflict { expected: 7, current: 1 })
-    );
-
-    assert_eq!(log_lines(&path).len(), before, "a refused ref operation appends nothing");
-    let main = store.get_ref("heads", "main").unwrap();
-    assert_eq!((main.version, main.target), (1, revisions[0].id.clone()));
-    assert!(store.get_ref("heads", "other").is_none());
-}
-
-// ---------------------------------------------------------------------------
-// 3. GC policy: history is roots and their ancestors (Option A)
-// ---------------------------------------------------------------------------
-
-/// P2: with no refs at all, GC must not touch a resource's history.
-#[test]
-fn gc_without_refs_keeps_every_head_and_its_ancestors() {
-    let (_dir, _path, db) = fresh();
-    let store = store(&db);
-    let revisions = chain(&store, "orders:1", 10);
-    let report = store.collect_unreachable().unwrap();
-    assert!(report.collected_revisions.is_empty(), "{report:?}");
-    for revision in &revisions {
-        assert!(store.exists(&revision.id), "revision {} was collected", revision.sequence);
-    }
-}
-
-/// Only old revisions that are neither a ref target, nor a resource's latest
-/// revision, nor an ancestor of either, are collected.
-#[test]
-fn gc_collects_exactly_the_old_unreachable_forks() {
-    let (_dir, _path, db) = fresh();
-    let store = store(&db);
-    // main line: 0 → 1 → 2 → 3, latest is 3
-    let main = chain(&store, "doc:g", 4);
-    // an old abandoned fork off 0: f1 (seq 1) → f2 (seq 2)
-    let f1 = old_child(&db, &main[0], json!({"fork": 1}));
-    let f2 = old_child(&db, &f1, json!({"fork": 2}));
-    // an old fork that a ref still names, with an old parent of its own
-    let kept_parent = old_child(&db, &main[0], json!({"kept": 1}));
-    let kept = old_child(&db, &kept_parent, json!({"kept": 2}));
-    store
-        .create_ref("candidates", "kept", kept.id.clone(), "app".into(), no_metadata())
-        .unwrap();
-    // a young fork, committed normally just now
-    let young = store
-        .commit(json!({"young": 1}).to_string(), &main[0], "app".into())
-        .unwrap();
-    // a second resource with no refs at all
-    let other = chain(&store, "doc:h", 3);
-
-    let report = store.collect_unreachable().unwrap();
-    let mut collected = report.collected_revisions.clone();
-    collected.sort();
-    let mut expected = vec![f1.id.clone(), f2.id.clone()];
-    expected.sort();
-    assert_eq!(collected, expected, "only the old, unreferenced, non-latest fork");
-
-    for kept in main
-        .iter()
-        .chain(other.iter())
-        .chain([&kept_parent, &kept, &young])
-    {
-        assert!(store.exists(&kept.id), "{} must survive", kept.id);
-    }
-    // Idempotent: nothing more to collect.
-    assert!(store.collect_unreachable().unwrap().collected_revisions.is_empty());
-}
-
-// ---------------------------------------------------------------------------
-// 4. GC against concurrent promotion and ref changes (P3)
-// ---------------------------------------------------------------------------
-
-/// A promotion of an old, collectable candidate races GC. Either GC wins and
-/// the promotion is refused, or the promotion wins and the candidate survives.
-/// Never a ref naming a deleted revision, and never lost latest history.
-#[test]
-fn gc_racing_promotion_never_dangles() {
-    let mut promoted = 0;
-    let mut refused = 0;
-    for iteration in 0..300 {
-        let (_dir, _path, db) = fresh();
-        let setup = store(&db);
-        let main = chain(&setup, "doc:p", 3);
-        let candidate = old_child(&db, &main[0], json!({"candidate": iteration}));
-        setup
-            .create_ref("heads", "main", main[0].id.clone(), "app".into(), no_metadata())
-            .unwrap();
-
-        let barrier = Arc::new(Barrier::new(2));
-        let promoter = {
-            let store = store(&db);
-            let barrier = barrier.clone();
-            let target = candidate.id.clone();
-            thread::spawn(move || {
-                barrier.wait();
-                store.compare_and_swap_ref("heads", "main", 1, target, "agent".into(), None)
-            })
-        };
-        let collector = {
-            let store = store(&db);
-            let barrier = barrier.clone();
-            thread::spawn(move || {
-                barrier.wait();
-                store.collect_unreachable()
-            })
-        };
-        let promotion = promoter.join().unwrap();
-        let _ = collector.join().unwrap();
-
-        let check = store(&db);
-        assert!(every_ref_target_exists(&check), "iteration {iteration}: dangling ref");
-        for revision in &main {
-            assert!(check.exists(&revision.id), "iteration {iteration}: latest history lost");
-        }
-        match promotion {
-            Ok(_) => {
-                promoted += 1;
-                assert!(check.exists(&candidate.id));
-            }
-            Err(RefError::UnknownTarget(_)) => {
-                refused += 1;
-                assert_eq!(check.get_ref("heads", "main").unwrap().version, 1);
-            }
-            Err(other) => panic!("iteration {iteration}: unexpected {other:?}"),
-        }
-    }
-    assert_eq!(promoted + refused, 300);
-    // Report both sides, so a run that never exercised one of them is visible.
-    eprintln!("gc_racing_promotion_never_dangles: promoted={promoted} refused={refused}");
-}
-
-/// Refs are created, moved and deleted continuously, each round onto a freshly
-/// written old (collectable) revision, while GC runs continuously. No ref ever
-/// names a missing revision, and both sides make progress.
-#[test]
-fn gc_racing_ref_changes_never_dangles() {
-    let (_dir, _path, db) = fresh();
-    let setup = store(&db);
-    // Long enough that the old revisions (sequence 1) are not the resource's
-    // latest, so they are collectable whenever no ref names them.
-    let main = chain(&setup, "doc:r", 4);
-
-    let writer = {
-        let db = db.clone();
-        let store = store(&db);
-        let base = main[0].clone();
-        thread::spawn(move || {
-            let mut applied = 0;
-            let mut refused = 0;
-            for round in 0..600 {
-                let candidate = old_child(&db, &base, json!({"round": round}));
-                let name = format!("n{}", round % 8);
-                let outcome = match store.get_ref("live", &name) {
-                    None => store
-                        .create_ref("live", &name, candidate.id, "w".into(), BTreeMap::new())
-                        .map(|_| ()),
-                    Some(current) if round % 5 == 0 => {
-                        store.delete_ref("live", &name, Some(current.version))
-                    }
-                    Some(current) => store
-                        .compare_and_swap_ref("live", &name, current.version, candidate.id, "w".into(), None)
-                        .map(|_| ()),
-                };
-                match outcome {
-                    Ok(()) => applied += 1,
-                    Err(RefError::UnknownTarget(_)) => refused += 1,
-                    Err(other) => panic!("round {round}: unexpected {other:?}"),
-                }
-            }
-            (applied, refused)
-        })
-    };
-    let collector = {
-        let store = store(&db);
-        thread::spawn(move || {
-            let mut collected = 0;
-            for _ in 0..200 {
-                if let Ok(report) = store.collect_unreachable() {
-                    collected += report.collected_revisions.len();
-                }
-                assert!(every_ref_target_exists(&store), "dangling ref observed mid-run");
-            }
-            collected
-        })
-    };
-    let (applied, refused) = writer.join().unwrap();
-    let collected = collector.join().unwrap();
-    eprintln!(
-        "gc_racing_ref_changes_never_dangles: ref writes applied={applied} \
-         refused_unknown_target={refused} revisions collected={collected}"
-    );
-    assert!(applied > 0, "the writer must have made progress");
-    assert!(collected > 0, "the collector must actually have collected");
-    let check = store(&db);
-    assert!(every_ref_target_exists(&check));
-    assert!(main.iter().all(|r| check.exists(&r.id)));
-}
-
-// ---------------------------------------------------------------------------
-// 5. Retry, restart, compaction, recreation
-// ---------------------------------------------------------------------------
-
-/// A CAS whose response was lost is retried with the same request after the
-/// process restarted and the log was compacted: the retry succeeds and does
-/// not advance the ref a second time.
-#[test]
-fn cas_retry_after_success_is_idempotent_across_restart_and_compaction() {
-    let (_dir, path, db) = fresh();
-    let (a, b) = {
-        let store = store(&db);
-        let revisions = chain(&store, "doc:i", 2);
-        store
-            .create_ref("heads", "main", revisions[0].id.clone(), "app".into(), no_metadata())
-            .unwrap();
-        let moved = store
-            .compare_and_swap_ref("heads", "main", 1, revisions[1].id.clone(), "app".into(), None)
-            .unwrap();
-        assert_eq!(moved.version, 2);
-        (revisions[0].id.clone(), revisions[1].id.clone())
-    };
-    drop(db);
-
-    let db = open(&path);
-    let retried = store(&db)
-        .compare_and_swap_ref("heads", "main", 1, b.clone(), "app".into(), None)
-        .expect("an identical retry after restart is the same request");
-    assert_eq!((retried.version, retried.target.clone()), (2, b.clone()));
-    assert_eq!(store(&db).get_ref("heads", "main").unwrap().version, 2);
-
-    let versions = db.operation_versions().unwrap();
-    db.acknowledge_peer_versions("peer".into(), versions).unwrap();
-    db.compact_operation_log(&["peer".into()]).unwrap();
-    drop(db);
-
-    let db = open(&path);
-    let store = store(&db);
-    let again = store
-        .compare_and_swap_ref("heads", "main", 1, b.clone(), "app".into(), None)
-        .expect("still the same request after compaction");
-    assert_eq!(again.version, 2);
-    // A different request for the same version slot is a conflict, not a replay.
-    assert_eq!(
-        store
-            .compare_and_swap_ref("heads", "main", 1, a, "app".into(), None)
-            .unwrap_err(),
-        RefError::VersionConflict { expected: 1, current: 2 }
-    );
-    assert_eq!(store.get_ref("heads", "main").unwrap().target, b);
-}
-
-#[test]
-fn create_and_delete_retries_are_idempotent_across_restart() {
-    let (_dir, path, db) = fresh();
-    let (a, b) = {
-        let store = store(&db);
-        let revisions = chain(&store, "doc:cd", 2);
-        store
-            .create_ref("heads", "main", revisions[0].id.clone(), "app".into(), no_metadata())
-            .unwrap();
-        (revisions[0].id.clone(), revisions[1].id.clone())
-    };
-    drop(db);
-
-    let db = open(&path);
-    let store_ = store(&db);
-    let created = store_
-        .create_ref("heads", "main", a.clone(), "app".into(), no_metadata())
-        .expect("identical create retried after restart");
-    assert_eq!(created.version, 1);
-    assert_eq!(
-        store_.create_ref("heads", "main", b.clone(), "app".into(), no_metadata()),
-        Err(RefError::AlreadyExists),
-        "a different create of an existing ref"
-    );
-    store_.delete_ref("heads", "main", Some(1)).unwrap();
-    drop(store_);
-    drop(db);
-
-    let db = open(&path);
-    let store = store(&db);
-    store
-        .delete_ref("heads", "main", Some(1))
-        .expect("an identical delete retried after restart");
-    assert!(store.get_ref("heads", "main").is_none());
-}
-
-/// After delete and recreate, the ref is back at version 1. A CAS from 1 on the
-/// new ref is a new request, never a replay of the first ref's CAS from 1.
-#[test]
-fn a_recreated_ref_does_not_replay_an_old_transaction() {
-    let (_dir, path, db) = fresh();
-    let (a, b) = {
-        let store = store(&db);
-        let revisions = chain(&store, "doc:rc", 2);
-        let (a, b) = (revisions[0].id.clone(), revisions[1].id.clone());
-        store.create_ref("heads", "main", a.clone(), "app".into(), no_metadata()).unwrap();
-        store
-            .compare_and_swap_ref("heads", "main", 1, b.clone(), "app".into(), None)
-            .unwrap();
-        store.delete_ref("heads", "main", Some(2)).unwrap();
-        (a, b)
-    };
-    drop(db);
-
-    let db = open(&path);
-    let store_ = store(&db);
-    let recreated = store_
-        .create_ref("heads", "main", a.clone(), "app".into(), no_metadata())
-        .unwrap();
-    assert_eq!((recreated.version, recreated.target), (1, a.clone()));
-    let moved = store_
-        .compare_and_swap_ref("heads", "main", 1, b.clone(), "app".into(), None)
-        .unwrap();
-    assert_eq!(moved.version, 2);
-    let current = store_.get_ref("heads", "main").unwrap();
-    assert_eq!((current.version, current.target), (2, b), "applied, not replayed");
-    drop(store_);
-    drop(db);
-
-    let db = open(&path);
-    assert_eq!(store(&db).get_ref("heads", "main").unwrap().version, 2);
-}
-
-/// Refs written by the pre-transactional implementation remain usable.
-#[test]
-fn legacy_ref_rows_remain_casable_across_restart() {
-    let (_dir, path, db) = fresh();
-    let (a, b) = {
-        let store = store(&db);
-        let revisions = chain(&store, "doc:l", 2);
-        (revisions[0].id.clone(), revisions[1].id.clone())
-    };
-    legacy_ref(&db, "heads", "one", &a, 1);
-    legacy_ref(&db, "heads", "three", &a, 3);
-    drop(db);
-
-    let db = open(&path);
-    let store_ = store(&db);
-    assert_eq!(store_.get_ref("heads", "three").unwrap().version, 3);
-    assert_eq!(
-        store_
-            .compare_and_swap_ref("heads", "three", 1, b.clone(), "app".into(), None)
-            .unwrap_err(),
-        RefError::VersionConflict { expected: 1, current: 3 }
-    );
-    assert_eq!(
-        store_
-            .compare_and_swap_ref("heads", "one", 1, b.clone(), "app".into(), None)
-            .unwrap()
-            .version,
-        2
-    );
-    assert_eq!(
-        store_
-            .compare_and_swap_ref("heads", "three", 3, b.clone(), "app".into(), None)
-            .unwrap()
-            .version,
-        4
-    );
-    store_.delete_ref("heads", "one", Some(2)).unwrap();
-    drop(store_);
-    drop(db);
-
-    let db = open(&path);
-    let store = store(&db);
-    assert!(store.get_ref("heads", "one").is_none());
-    let three = store.get_ref("heads", "three").unwrap();
-    assert_eq!((three.version, three.target), (4, b));
-}
-
-// ---------------------------------------------------------------------------
-// 6. Durability and atomicity of the write itself (P4)
-// ---------------------------------------------------------------------------
-
-fn record_type(line: &str) -> Option<String> {
-    serde_json::from_str::<Value>(line)
-        .ok()?
-        .get("record_type")?
-        .as_str()
-        .map(str::to_string)
-}
-
-/// A ref move is one durable transaction record, which `FeltDb` fsyncs before
-/// returning (`append_transaction` → `sync_data`).
-#[test]
-fn a_ref_move_is_one_transaction_record() {
-    let (_dir, path, db) = fresh();
-    let store = store(&db);
-    let revisions = chain(&store, "doc:d", 2);
-    store
-        .create_ref("heads", "main", revisions[0].id.clone(), "app".into(), no_metadata())
-        .unwrap();
-    let before = log_lines(&path);
-    store
-        .compare_and_swap_ref("heads", "main", 1, revisions[1].id.clone(), "app".into(), None)
-        .unwrap();
-    let after = log_lines(&path);
-    assert_eq!(after.len(), before.len() + 1, "one record per ref move");
-    assert_eq!(
-        record_type(after.last().unwrap()).as_deref(),
-        Some("feltdb.transaction.v1"),
-        "the ref move is a transaction record"
-    );
-}
-
-/// A committed revision and its content object are one transaction record.
-#[test]
-fn a_revision_commit_is_one_transaction_record() {
-    let (_dir, path, db) = fresh();
-    let store = store(&db);
-    let base = store
-        .create("doc:o", json!({"v": 0}).to_string(), "app".into())
-        .unwrap();
-    let before = log_lines(&path);
-    store
-        .commit(json!({"v": 1}).to_string(), &base, "app".into())
-        .unwrap();
-    let after = log_lines(&path);
-    assert_eq!(after.len(), before.len() + 1, "object and revision in one record");
-    assert_eq!(
-        record_type(after.last().unwrap()).as_deref(),
-        Some("feltdb.transaction.v1")
-    );
-}
-
-/// Truncate the log at every byte inside the ref move's record: the database
-/// always opens, and the ref is either entirely before or entirely after.
-#[test]
-fn a_ref_move_is_all_or_nothing_at_every_crash_point() {
-    let (_dir, path, db) = fresh();
-    let (a, b) = {
-        let store = store(&db);
-        let revisions = chain(&store, "doc:t", 2);
-        store
-            .create_ref("heads", "main", revisions[0].id.clone(), "app".into(), no_metadata())
-            .unwrap();
-        (revisions[0].id.clone(), revisions[1].id.clone())
-    };
-    let prefix = std::fs::read(&path).unwrap().len();
-    store(&db)
-        .compare_and_swap_ref("heads", "main", 1, b.clone(), "app".into(), None)
-        .unwrap();
-    drop(db);
-    let full = std::fs::read(&path).unwrap();
-
-    let scratch = TempDir::new().unwrap();
-    for cut in prefix..=full.len() {
-        let copy = scratch.path().join(format!("cut-{cut}.log"));
-        std::fs::write(&copy, &full[..cut]).unwrap();
-        let db = Arc::new(FeltDb::open(&copy).unwrap_or_else(|e| panic!("cut {cut}: {e:?}")));
-        let store = store(&db);
-        let main = store.get_ref("heads", "main").expect("the ref survives");
-        let state = (main.version, main.target.clone());
-        assert!(
-            state == (1, a.clone()) || state == (2, b.clone()),
-            "cut {cut}: torn ref {state:?}"
-        );
-        assert!(store.exists(&main.target), "cut {cut}: dangling ref");
-        drop(store);
-        drop(db);
-        std::fs::remove_file(&copy).ok();
-    }
-}
-
-// ---------------------------------------------------------------------------
-// 7. Real process death
-// ---------------------------------------------------------------------------
-
-const CHILD: &str = "FELTDB_REF_AUTHORITY_CHILD";
-const CHILD_PATH: &str = "FELTDB_REF_AUTHORITY_PATH";
-
-/// The value after `marker` on any line. The test harness prints
-/// `test <name> ... ` without a newline before the child's own output, so the
-/// first marker can land mid-line.
-fn marked(line: &str, marker: &str) -> Option<String> {
-    line.find(marker)
-        .map(|at| line[at + marker.len()..].trim().to_string())
-}
-
-fn child_command(test: &str, path: &Path, mode: &str) -> Command {
-    let mut command = Command::new(std::env::current_exe().unwrap());
-    command
-        .args([test, "--exact", "--nocapture", "--test-threads=1"])
-        .env(CHILD, mode)
-        .env(CHILD_PATH, path);
-    command
-}
-
-/// The child acknowledges a CAS and dies by `abort()` before its caller could
-/// have seen the result. The restarted caller retries the same request: it is
-/// accepted as the request it already was, not applied twice.
-#[test]
-fn a_cas_acknowledged_before_abort_is_retried_idempotently() {
-    if std::env::var(CHILD).as_deref() == Ok("ack-then-abort") {
-        let path = PathBuf::from(std::env::var(CHILD_PATH).unwrap());
-        let db = open(&path);
-        let store = store(&db);
-        let revisions = chain(&store, "doc:k", 2);
-        store
-            .create_ref("heads", "main", revisions[0].id.clone(), "app".into(), no_metadata())
-            .unwrap();
-        store
-            .compare_and_swap_ref("heads", "main", 1, revisions[1].id.clone(), "app".into(), None)
-            .unwrap();
-        println!("ACK {}", revisions[1].id.as_hex());
-        std::process::abort();
-    }
-
-    let dir = TempDir::new().unwrap();
-    let path = dir.path().join("abort.log");
-    let output = child_command(
-        "a_cas_acknowledged_before_abort_is_retried_idempotently",
-        &path,
-        "ack-then-abort",
-    )
-    .output()
-    .unwrap();
-    assert!(!output.status.success(), "the child must have died");
-    let stdout = String::from_utf8_lossy(&output.stdout);
-    let acked = stdout
-        .lines()
-        .find_map(|line| marked(line, "ACK "))
-        .expect("the child acknowledged before dying");
-    let target = StateId::from_hex(acked);
-
-    let db = open(&path);
-    let store = store(&db);
-    let current = store.get_ref("heads", "main").unwrap();
-    assert_eq!((current.version, current.target.clone()), (2, target.clone()));
-    let retried = store
-        .compare_and_swap_ref("heads", "main", 1, target.clone(), "app".into(), None)
-        .expect("the retry of an acknowledged CAS is accepted");
-    assert_eq!(retried.version, 2);
-    assert_eq!(store.get_ref("heads", "main").unwrap().version, 2, "not applied twice");
-}
-
-/// The child commits a revision and dies before promoting it. After restart the
-/// revision is durable, the ref is unchanged, and the promotion can complete.
-#[test]
-fn death_between_commit_and_promotion_leaves_a_durable_unpromoted_revision() {
-    if std::env::var(CHILD).as_deref() == Ok("commit-then-abort") {
-        let path = PathBuf::from(std::env::var(CHILD_PATH).unwrap());
-        let db = open(&path);
-        let store = store(&db);
-        let base = store
-            .create("doc:m", json!({"v": 0}).to_string(), "app".into())
-            .unwrap();
-        store
-            .create_ref("heads", "main", base.id.clone(), "app".into(), no_metadata())
-            .unwrap();
-        let next = store
-            .commit(json!({"v": 1}).to_string(), &base, "app".into())
-            .unwrap();
-        println!("COMMITTED {}", next.id.as_hex());
-        std::process::abort();
-    }
-
-    let dir = TempDir::new().unwrap();
-    let path = dir.path().join("abort.log");
-    let output = child_command(
-        "death_between_commit_and_promotion_leaves_a_durable_unpromoted_revision",
-        &path,
-        "commit-then-abort",
-    )
-    .output()
-    .unwrap();
-    assert!(!output.status.success());
-    let stdout = String::from_utf8_lossy(&output.stdout);
-    let committed = StateId::from_hex(
-        stdout
-            .lines()
-            .find_map(|line| marked(line, "COMMITTED "))
-            .expect("the child committed before dying"),
-    );
-
-    let db = open(&path);
-    let store = store(&db);
-    assert!(store.exists(&committed), "the committed revision is durable");
-    assert_eq!(store.get_ref("heads", "main").unwrap().version, 1, "not promoted");
-    assert!(store.collect_unreachable().unwrap().collected_revisions.is_empty());
-    store
-        .compare_and_swap_ref("heads", "main", 1, committed.clone(), "app".into(), None)
-        .unwrap();
-    assert_eq!(store.get_ref("heads", "main").unwrap().target, committed);
-}
-
-/// The child moves refs continuously and reports each acknowledged move; the
-/// parent SIGKILLs it at an arbitrary point. Every acknowledged move survives,
-/// and the ref never names a missing revision.
-#[test]
-fn sigkill_during_ref_moves_keeps_every_acknowledged_move() {
-    if std::env::var(CHILD).as_deref() == Ok("move-forever") {
-        let path = PathBuf::from(std::env::var(CHILD_PATH).unwrap());
-        let db = open(&path);
-        let store = store(&db);
-        let base = store
-            .create("doc:s", json!({"v": 0}).to_string(), "app".into())
-            .unwrap();
-        store
-            .create_ref("heads", "main", base.id.clone(), "app".into(), no_metadata())
-            .unwrap();
-        let mut head = base;
-        let mut version = 1;
-        loop {
-            let next = store
-                .commit(json!({"v": version}).to_string(), &head, "app".into())
-                .unwrap();
-            let moved = store
-                .compare_and_swap_ref("heads", "main", version, next.id.clone(), "app".into(), None)
-                .unwrap();
-            version = moved.version;
-            println!("ACK {version}");
-            head = next;
-        }
-    }
-
-    let dir = TempDir::new().unwrap();
-    let path = dir.path().join("kill.log");
-    let mut child = child_command(
-        "sigkill_during_ref_moves_keeps_every_acknowledged_move",
-        &path,
-        "move-forever",
-    )
-    .stdout(Stdio::piped())
-    .stderr(Stdio::null())
-    .spawn()
-    .unwrap();
-    let mut last_acked = 0;
-    for line in BufReader::new(child.stdout.take().unwrap()).lines() {
-        let line = line.unwrap();
-        if let Some(version) = marked(&line, "ACK ") {
-            last_acked = version.parse::<u64>().unwrap();
-            if last_acked >= 40 {
-                child.kill().unwrap(); // SIGKILL
-                break;
-            }
-        }
-    }
-    child.wait().unwrap();
-    assert!(last_acked >= 40);
-
-    let db = open(&path);
-    let store = store(&db);
-    let main = store.get_ref("heads", "main").unwrap();
-    assert!(
-        main.version >= last_acked,
-        "acknowledged version {last_acked} lost; recovered {}",
-        main.version
-    );
-    assert!(store.exists(&main.target), "dangling ref after SIGKILL");
-    assert!(every_ref_target_exists(&store));
-}