@feltdb/core 0.12.0 → 0.12.1

package/dist/create/server-source/crates/feltdb/src/state_model.rs

@@ -1371,6 +1371,553 @@ fn current_time_ms() -> u64 {
         .as_millis() as u64
 }
 
+/// How long a revision is protected from garbage collection after it is
+/// minted, measured on its own `timestamp_ms`. A candidate an agent has just
+/// committed but not yet promoted is a root for this long, so a collection that
+/// races the promotion cannot reclaim it out from under the caller.
+pub(crate) const GC_GRACE_MS: u64 = 15 * 60 * 1000;
+
+/// Durable state-model writes as FeltDB transactions.
+///
+/// # Ordering
+///
+/// Every operation here follows the same three steps:
+///
+/// 1. **Read** the `state` rows the decision depends on.
+/// 2. **Decide** in Rust: versions, presence, integrity.
+/// 3. **Commit** with `FeltDb::apply_atomic_transaction_guarded`, carrying a
+///    record precondition for *every row read in step 1*. FeltDB evaluates the
+///    preconditions inside its commit lock, before staging anything, and
+///    applies the mutations — one log record, `sync_data` before returning —
+///    only if every row is still as it was read. Otherwise nothing is applied
+///    and the operation starts again from step 1.
+///
+/// So a decision is only ever applied to the state it was made against. The
+/// mutual exclusion is FeltDB's `Inner` mutex, shared by every `StateStore`,
+/// every `Arc<FeltDb>` clone and every `FeltDb::open` of the same path in one
+/// process; the per-instance `mutation_lock` plays no part on this backing.
+///
+/// # How a row is pinned
+///
+/// FeltDB's record precondition compares `__version`, reading a row without one
+/// as version 1, and stamps `N + 1` on a row written under `expected_version =
+/// N` (`require_absent` writes 1). That gives three kinds of pin:
+///
+/// - **Versioned rows** (ref, ref incarnation) are only ever written under a
+///   precondition, so their `__version` advances on every write and
+///   `expected_version = <as read>` pins them; an absent one is pinned by
+///   `require_absent`.
+/// - **Immutable rows** (revision, content object) are never versioned, so
+///   `expected_version = 1` means exactly "still present" and writes nothing.
+/// - **The generation row** (`state:_gcgen`) is rewritten by every ref change
+///   and every committed revision *without* a precondition, carrying a
+///   `__version` unique to that transaction (an unconditional write is stored
+///   verbatim). Writers therefore never conflict with each other through it,
+///   while a collection that pins the value it read is refused if any of them
+///   committed in between.
+///
+/// | operation | pinned | writes |
+/// | --- | --- | --- |
+/// | create ref | ref (absent), incarnation, target | ref, incarnation, generation |
+/// | CAS ref | ref, incarnation, target | ref, generation |
+/// | delete ref | ref, incarnation | (ref removed), incarnation, generation |
+/// | commit revision | revision (absent), content object, parent | object (if new), revision, generation |
+/// | collect | generation | (unreachable revisions and objects removed) |
+///
+/// The incarnation row is pinned alongside the ref because a delete followed by
+/// a create returns the ref row to `__version` 1; the incarnation row is never
+/// deleted, so its version only grows and the pair cannot repeat.
+///
+/// # The races this closes
+///
+/// - *Concurrent CAS.* Both read the ref at `__version` N; the first commit
+///   stamps N + 1; the second's precondition fails, it re-reads, and reports
+///   `VersionConflict`.
+/// - *GC vs promotion.* If the promotion commits first, the generation moved
+///   and GC re-plans with the target as a ref root. If GC commits first, the
+///   promotion's target pin fails, and on re-read it reports `UnknownTarget`.
+/// - *GC vs commit of a child.* If the commit lands first, the generation
+///   moved. If GC lands first, the commit's parent pin fails and it reports
+///   `UnknownParent`.
+///
+/// Revisions minted by the record write path are not committed here; each is a
+/// child of that path's current head, which collection always treats as a
+/// root, so they cannot gain a collected parent.
+///
+/// # Idempotent ref requests
+///
+/// Ref transactions carry deterministic ids derived from the ref, its
+/// *incarnation* and the version they act on, and a payload hash of the rest
+/// of the request. FeltDB binds an id to its payload durably, across restart
+/// and compaction. A repeat of a committed request finds its own payload under
+/// its id and returns `Ok` without writing; any other request for the same slot
+/// finds a different payload and is a conflict. The incarnation
+/// (`state:refinc:…`, advanced by every create) keeps a deleted-and-recreated
+/// ref, back at version 1, from matching a transaction of its predecessor. Rows
+/// written before this scheme have no incarnation and are incarnation 0.
+pub(crate) mod durable {
+    use super::*;
+    use crate::{AtomicMutation, FeltDb, FlowError, PreconditionFailure, RecordPrecondition};
+    use serde_json::json;
+    use std::sync::atomic::{AtomicU64, Ordering};
+
+    /// Fresh reads and a new attempt after a refused precondition. Each refusal
+    /// means a competing write committed, so the system as a whole makes
+    /// progress; the bound only turns pathological contention into an error.
+    const ATTEMPTS: usize = 64;
+
+    pub(super) const GENERATION_KEY: &str = "state:_gcgen";
+
+    fn incarnation_key(namespace: &str, name: &str) -> String {
+        format!(
+            "state:refinc:{}:{}",
+            encode_ref_part(namespace),
+            encode_ref_part(name)
+        )
+    }
+
+    fn put(key: &str, value: Value) -> AtomicMutation {
+        AtomicMutation {
+            capability: "state".into(),
+            key: key.into(),
+            value: Some(value),
+        }
+    }
+
+    fn remove(key: &str) -> AtomicMutation {
+        AtomicMutation {
+            capability: "state".into(),
+            key: key.into(),
+            value: None,
+        }
+    }
+
+    /// The version FeltDB's record precondition reads for a row.
+    fn row_version(row: &Value) -> u64 {
+        row.get("__version").and_then(Value::as_u64).unwrap_or(1)
+    }
+
+    /// Pin a row to exactly what was read: absent, or this version.
+    fn unchanged(key: &str, read: Option<&Value>) -> RecordPrecondition {
+        RecordPrecondition {
+            capability: "state".into(),
+            key: key.into(),
+            require_absent: read.is_none(),
+            expected_version: read.map(row_version),
+            ..RecordPrecondition::default()
+        }
+    }
+
+    /// Pin an immutable (never versioned) row as present.
+    fn present(key: &str) -> RecordPrecondition {
+        RecordPrecondition {
+            capability: "state".into(),
+            key: key.into(),
+            expected_version: Some(1),
+            ..RecordPrecondition::default()
+        }
+    }
+
+    /// The generation row's new value: a `__version` unique to this transaction.
+    /// Transaction ids are never applied twice, so every rewrite changes it.
+    fn advance_generation(transaction_id: &str) -> AtomicMutation {
+        let digest = Sha256::digest(transaction_id.as_bytes());
+        let mut bytes = [0u8; 8];
+        bytes.copy_from_slice(&digest[..8]);
+        put(
+            GENERATION_KEY,
+            json!({ "__version": u64::from_be_bytes(bytes), "transaction": transaction_id }),
+        )
+    }
+
+    fn unique_suffix() -> String {
+        static COUNTER: AtomicU64 = AtomicU64::new(0);
+        let nanos = std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap_or_default()
+            .as_nanos();
+        format!(
+            "{}-{nanos}-{}",
+            std::process::id(),
+            COUNTER.fetch_add(1, Ordering::Relaxed)
+        )
+    }
+
+    fn ref_transaction_id(
+        kind: &str,
+        namespace: &str,
+        name: &str,
+        incarnation: u64,
+        version: Option<u64>,
+    ) -> String {
+        let mut hasher = Sha256::new();
+        for part in [
+            "feltdb.state.ref.v1",
+            kind,
+            namespace,
+            name,
+            &incarnation.to_string(),
+            &version.map(|v| v.to_string()).unwrap_or_default(),
+        ] {
+            hasher.update(part.as_bytes());
+            hasher.update([0u8]);
+        }
+        format!("state-ref:{:x}", hasher.finalize())
+    }
+
+    /// serde_json maps are ordered (no `preserve_order`), so this is canonical.
+    fn payload_hash(payload: &Value) -> String {
+        format!("sha256:{:x}", Sha256::digest(payload.to_string().as_bytes()))
+    }
+
+    fn incarnation_of(row: &Value) -> u64 {
+        row.get("__incarnation").and_then(Value::as_u64).unwrap_or(0)
+    }
+
+    fn incarnation_in(row: Option<&Value>) -> u64 {
+        row.and_then(|row| row.get("incarnation").and_then(Value::as_u64))
+            .unwrap_or(0)
+    }
+
+    fn ref_row(reference: &StateRef, incarnation: u64) -> Value {
+        let mut row = serde_json::to_value(reference).unwrap_or(Value::Null);
+        if let Some(object) = row.as_object_mut() {
+            object.insert("__incarnation".into(), Value::from(incarnation));
+        }
+        row
+    }
+
+    enum Refusal {
+        /// A pinned row changed: re-read and decide again.
+        Changed,
+        /// The id is already bound to another payload: re-read and decide again.
+        OtherRequest,
+        Failed(String),
+    }
+
+    fn refusal(error: FlowError) -> Refusal {
+        match error {
+            FlowError::PreconditionFailed(failure) => match *failure {
+                PreconditionFailure::Present { .. }
+                | PreconditionFailure::Missing { .. }
+                | PreconditionFailure::Version { .. } => Refusal::Changed,
+                other => Refusal::Failed(format!("{other:?}")),
+            },
+            FlowError::CapabilityError(message)
+                if message.starts_with("TRANSACTION_ID_PAYLOAD_MISMATCH") =>
+            {
+                Refusal::OtherRequest
+            }
+            other => Refusal::Failed(other.to_string()),
+        }
+    }
+
+    fn storage(error: impl std::fmt::Display) -> RefError {
+        RefError::Storage(error.to_string())
+    }
+
+    fn contended(what: &str) -> String {
+        format!(
+            "{what} was refused by concurrent changes on {ATTEMPTS} consecutive attempts; nothing was applied"
+        )
+    }
+
+    fn row(db: &FeltDb, key: &str) -> Result<Option<Value>, RefError> {
+        db.get::<Value>(key).map_err(storage)
+    }
+
+    fn committed_payload(db: &FeltDb, id: &str) -> Result<Option<String>, RefError> {
+        db.applied_transaction_payload_hash(id).map_err(storage)
+    }
+
+    fn decode(row: Value) -> Result<StateRef, RefError> {
+        serde_json::from_value(row).map_err(storage)
+    }
+
+    fn commit(
+        db: &FeltDb,
+        id: &str,
+        payload: Option<&str>,
+        pins: &[RecordPrecondition],
+        mutations: &[AtomicMutation],
+    ) -> Result<(), Refusal> {
+        db.apply_atomic_transaction_guarded(id, payload, None, &[], pins, mutations, None)
+            .map(|_| ())
+            .map_err(refusal)
+    }
+
+    pub(super) fn create_ref(
+        db: &FeltDb,
+        namespace: &str,
+        name: &str,
+        target: StateId,
+        authority: String,
+        metadata: BTreeMap<String, Value>,
+    ) -> Result<StateRef, RefError> {
+        let ref_k = ref_key(namespace, name);
+        let inc_k = incarnation_key(namespace, name);
+        let target_k = revision_key(&target);
+        let payload = payload_hash(&json!({
+            "target": target.as_hex(),
+            "authority": authority,
+            "metadata": metadata,
+        }));
+        for _ in 0..ATTEMPTS {
+            let incarnation_row = row(db, &inc_k)?;
+            if let Some(existing) = row(db, &ref_k)? {
+                // The create that made this incarnation, repeated, is that create.
+                let id = ref_transaction_id("create", namespace, name, incarnation_of(&existing), None);
+                if committed_payload(db, &id)?.as_deref() == Some(payload.as_str()) {
+                    return decode(existing);
+                }
+                return Err(RefError::AlreadyExists);
+            }
+            if row(db, &target_k)?.is_none() {
+                return Err(RefError::UnknownTarget(target));
+            }
+            let incarnation = incarnation_in(incarnation_row.as_ref()) + 1;
+            let id = ref_transaction_id("create", namespace, name, incarnation, None);
+            let reference = StateRef {
+                namespace: namespace.to_string(),
+                name: name.to_string(),
+                target: target.clone(),
+                version: 1,
+                authority: authority.clone(),
+                updated_at_ms: current_time_ms(),
+                metadata: metadata.clone(),
+            };
+            let pins = [
+                unchanged(&ref_k, None),
+                unchanged(&inc_k, incarnation_row.as_ref()),
+                present(&target_k),
+            ];
+            let mutations = [
+                put(&ref_k, ref_row(&reference, incarnation)),
+                put(&inc_k, json!({ "incarnation": incarnation })),
+                advance_generation(&id),
+            ];
+            match commit(db, &id, Some(&payload), &pins, &mutations) {
+                Ok(()) => return Ok(reference),
+                Err(Refusal::Changed | Refusal::OtherRequest) => continue,
+                Err(Refusal::Failed(message)) => return Err(RefError::Storage(message)),
+            }
+        }
+        Err(RefError::Storage(contended("ref create")))
+    }
+
+    pub(super) fn compare_and_swap_ref(
+        db: &FeltDb,
+        namespace: &str,
+        name: &str,
+        expected_version: u64,
+        target: StateId,
+        authority: String,
+        metadata: Option<BTreeMap<String, Value>>,
+    ) -> Result<StateRef, RefError> {
+        let ref_k = ref_key(namespace, name);
+        let inc_k = incarnation_key(namespace, name);
+        let target_k = revision_key(&target);
+        let payload = payload_hash(&json!({
+            "target": target.as_hex(),
+            "authority": authority,
+            "metadata": metadata,
+        }));
+        for _ in 0..ATTEMPTS {
+            let incarnation_row = row(db, &inc_k)?;
+            let Some(ref_read) = row(db, &ref_k)? else {
+                return Err(RefError::NotFound);
+            };
+            let incarnation = incarnation_of(&ref_read);
+            let current = decode(ref_read.clone())?;
+            let id = ref_transaction_id("cas", namespace, name, incarnation, Some(expected_version));
+            if current.version != expected_version {
+                // Either this exact request already committed (a retry), or
+                // another one did: the payload bound to the id says which.
+                if committed_payload(db, &id)?.as_deref() == Some(payload.as_str()) {
+                    if current.version == expected_version + 1 && current.target == target {
+                        return Ok(current);
+                    }
+                    // Committed, and the ref has moved on since. Describe the
+                    // transition this request made; `updated_at_ms` is now.
+                    return Ok(StateRef {
+                        version: expected_version + 1,
+                        target,
+                        authority,
+                        updated_at_ms: current_time_ms(),
+                        metadata: metadata.unwrap_or(current.metadata),
+                        ..current
+                    });
+                }
+                return Err(RefError::VersionConflict {
+                    expected: expected_version,
+                    current: current.version,
+                });
+            }
+            if row(db, &target_k)?.is_none() {
+                return Err(RefError::UnknownTarget(target));
+            }
+            let next = StateRef {
+                version: expected_version + 1,
+                target: target.clone(),
+                authority: authority.clone(),
+                updated_at_ms: current_time_ms(),
+                metadata: metadata.clone().unwrap_or_else(|| current.metadata.clone()),
+                ..current
+            };
+            let pins = [
+                unchanged(&ref_k, Some(&ref_read)),
+                unchanged(&inc_k, incarnation_row.as_ref()),
+                present(&target_k),
+            ];
+            let mutations = [put(&ref_k, ref_row(&next, incarnation)), advance_generation(&id)];
+            match commit(db, &id, Some(&payload), &pins, &mutations) {
+                Ok(()) => return Ok(next),
+                Err(Refusal::Changed | Refusal::OtherRequest) => continue,
+                Err(Refusal::Failed(message)) => return Err(RefError::Storage(message)),
+            }
+        }
+        Err(RefError::Storage(contended("ref compare-and-swap")))
+    }
+
+    pub(super) fn delete_ref(
+        db: &FeltDb,
+        namespace: &str,
+        name: &str,
+        expected_version: Option<u64>,
+    ) -> Result<(), RefError> {
+        let ref_k = ref_key(namespace, name);
+        let inc_k = incarnation_key(namespace, name);
+        let payload = payload_hash(&json!({ "expected_version": expected_version }));
+        for _ in 0..ATTEMPTS {
+            let incarnation_row = row(db, &inc_k)?;
+            let Some(ref_read) = row(db, &ref_k)? else {
+                // A repeat of a delete that already removed this incarnation.
+                let id = ref_transaction_id(
+                    "delete",
+                    namespace,
+                    name,
+                    incarnation_in(incarnation_row.as_ref()),
+                    None,
+                );
+                if committed_payload(db, &id)?.as_deref() == Some(payload.as_str()) {
+                    return Ok(());
+                }
+                return Err(RefError::NotFound);
+            };
+            let incarnation = incarnation_of(&ref_read);
+            let current = decode(ref_read.clone())?;
+            if let Some(expected) = expected_version {
+                if current.version != expected {
+                    return Err(RefError::VersionConflict {
+                        expected,
+                        current: current.version,
+                    });
+                }
+            }
+            let id = ref_transaction_id("delete", namespace, name, incarnation, None);
+            let pins = [
+                unchanged(&ref_k, Some(&ref_read)),
+                unchanged(&inc_k, incarnation_row.as_ref()),
+            ];
+            let mutations = [
+                remove(&ref_k),
+                put(&inc_k, json!({ "incarnation": incarnation })),
+                advance_generation(&id),
+            ];
+            match commit(db, &id, Some(&payload), &pins, &mutations) {
+                Ok(()) => return Ok(()),
+                Err(Refusal::Changed | Refusal::OtherRequest) => continue,
+                Err(Refusal::Failed(message)) => return Err(RefError::Storage(message)),
+            }
+        }
+        Err(RefError::Storage(contended("ref delete")))
+    }
+
+    pub(super) fn commit_revision(
+        db: &FeltDb,
+        revision: StateRevision,
+    ) -> Result<StateRevision, RevisionError> {
+        let fail = |error: &dyn std::fmt::Display| RevisionError::Storage(error.to_string());
+        let object = StateObject::new(revision.content.clone());
+        let rev_k = revision_key(&revision.id);
+        let obj_k = object_key(&object.id);
+        let parent_k = revision.parent_id.as_ref().map(revision_key);
+        for _ in 0..ATTEMPTS {
+            if let Some(existing) = db.get::<StateRevision>(&rev_k).map_err(|e| fail(&e))? {
+                let same = existing.resource == revision.resource
+                    && existing.content_id == revision.content_id
+                    && existing.parent_id == revision.parent_id
+                    && existing.sequence == revision.sequence;
+                return if same {
+                    Ok(existing)
+                } else {
+                    Err(RevisionError::AlreadyCommitted(revision.id.clone()))
+                };
+            }
+            let mut pins = vec![unchanged(&rev_k, None)];
+            if let (Some(parent), Some(parent_k)) = (&revision.parent_id, &parent_k) {
+                if db.get::<Value>(parent_k).map_err(|e| fail(&e))?.is_none() {
+                    return Err(RevisionError::UnknownParent(parent.clone()));
+                }
+                pins.push(present(parent_k));
+            }
+            let id = format!("state-revision:{}:{}", revision.id.as_hex(), unique_suffix());
+            let mut mutations = Vec::with_capacity(3);
+            match db.get::<StateObject>(&obj_k).map_err(|e| fail(&e))? {
+                Some(existing) if !existing.verify_integrity() => {
+                    return Err(RevisionError::Storage(
+                        "Stored state object failed integrity verification".to_string(),
+                    ));
+                }
+                Some(_) => pins.push(present(&obj_k)),
+                None => {
+                    pins.push(unchanged(&obj_k, None));
+                    mutations.push(put(&obj_k, serde_json::to_value(&object).map_err(|e| fail(&e))?));
+                }
+            }
+            mutations.push(put(&rev_k, serde_json::to_value(&revision).map_err(|e| fail(&e))?));
+            mutations.push(advance_generation(&id));
+            match commit(db, &id, None, &pins, &mutations) {
+                Ok(()) => return Ok(revision),
+                Err(Refusal::Changed | Refusal::OtherRequest) => continue,
+                Err(Refusal::Failed(message)) => return Err(RevisionError::Storage(message)),
+            }
+        }
+        Err(RevisionError::Storage(contended("revision commit")))
+    }
+
+    pub(super) fn collect(
+        store: &StateStore,
+        db: &FeltDb,
+        now_ms: u64,
+    ) -> Result<GarbageCollectionReport, GarbageCollectionError> {
+        let fail = |error: &dyn std::fmt::Display| GarbageCollectionError::Storage(error.to_string());
+        for _ in 0..ATTEMPTS {
+            // The generation first: everything planned below was read after it,
+            // so a write that lands after this read moves it.
+            let generation = db.get::<Value>(GENERATION_KEY).map_err(|e| fail(&e))?;
+            let report = store.plan_collection(now_ms, &db.revision_head_ids());
+            if report.collected_revisions.is_empty() && report.collected_objects.is_empty() {
+                return Ok(report);
+            }
+            let pins = [unchanged(GENERATION_KEY, generation.as_ref())];
+            let mutations: Vec<AtomicMutation> = report
+                .collected_revisions
+                .iter()
+                .map(|id| remove(&revision_key(id)))
+                .chain(report.collected_objects.iter().map(|id| remove(&object_key(id))))
+                .collect();
+            let id = format!("state-gc:{}", unique_suffix());
+            match commit(db, &id, None, &pins, &mutations) {
+                Ok(()) => return Ok(report),
+                Err(Refusal::Changed | Refusal::OtherRequest) => continue,
+                Err(Refusal::Failed(message)) => return Err(GarbageCollectionError::Storage(message)),
+            }
+        }
+        Err(GarbageCollectionError::Storage(contended("garbage collection")))
+    }
+}
+
 fn collect_referenced_objects(revision: &StateRevision) -> BTreeSet<StateId> {
     let mut ids = BTreeSet::from([revision.content_id.clone()]);
     if let Some(values) = revision
@@ -1692,34 +2239,7 @@ impl StateStore {
                 state.revisions.insert(revision.id.clone(), revision.clone());
             }
             Backing::Durable(feltdb) => {
-                let object = StateObject::new(revision.content.clone());
-                if let Some(existing) = feltdb
-                    .get::<StateObject>(&object_key(&object.id))
-                    .map_err(|error| {
-                        RevisionError::Storage(format!("Failed to read state object: {error}"))
-                    })?
-                {
-                    if !existing.verify_integrity() {
-                        return Err(RevisionError::Storage(
-                            "Stored state object failed integrity verification".to_string(),
-                        ));
-                    }
-                } else {
-                    feltdb
-                        .insert(&object_key(&object.id), object)
-                        .map_err(|error| {
-                            RevisionError::Storage(format!(
-                                "Failed to persist state object to FeltDB: {error}"
-                            ))
-                        })?;
-                }
-                feltdb
-                    .insert(&revision_key(&revision.id), revision.clone())
-                    .map_err(|error| {
-                        RevisionError::Storage(format!(
-                            "Failed to persist revision to FeltDB: {error}"
-                        ))
-                    })?;
+                return durable::commit_revision(feltdb, revision);
             }
         }
         Ok(revision)
@@ -1816,6 +2336,12 @@ impl StateStore {
     }
 
     /// Create a mutable reference in a namespace.
+    ///
+    /// On the durable backing this is one fenced FeltDB transaction (see
+    /// [`durable`]): the ref is created at version 1 only if, at the moment of
+    /// commit, it is still absent and its target revision is still present.
+    /// Repeating an identical create that already succeeded returns the ref it
+    /// created; a different create of an existing ref is `AlreadyExists`.
     pub fn create_ref(
         &self,
         namespace: &str,
@@ -1827,24 +2353,31 @@ impl StateStore {
         if !valid_ref_component(namespace) || !valid_ref_component(name) {
             return Err(RefError::InvalidIdentifier);
         }
-        if !self.exists(&target) {
-            return Err(RefError::UnknownTarget(target));
-        }
-        let _guard = self.mutation_lock.lock().unwrap();
-        if self.get_ref(namespace, name).is_some() {
-            return Err(RefError::AlreadyExists);
+        match &self.backing {
+            Backing::Durable(feltdb) => {
+                durable::create_ref(feltdb, namespace, name, target, authority, metadata)
+            }
+            Backing::Volatile(_) => {
+                let _guard = self.mutation_lock.lock().unwrap();
+                if !self.exists(&target) {
+                    return Err(RefError::UnknownTarget(target));
+                }
+                if self.get_ref(namespace, name).is_some() {
+                    return Err(RefError::AlreadyExists);
+                }
+                let reference = StateRef {
+                    namespace: namespace.to_string(),
+                    name: name.to_string(),
+                    target,
+                    version: 1,
+                    authority,
+                    updated_at_ms: current_time_ms(),
+                    metadata,
+                };
+                self.write_volatile_ref(&reference);
+                Ok(reference)
+            }
         }
-        let reference = StateRef {
-            namespace: namespace.to_string(),
-            name: name.to_string(),
-            target,
-            version: 1,
-            authority,
-            updated_at_ms: current_time_ms(),
-            metadata,
-        };
-        self.write_ref(&reference)?;
-        Ok(reference)
     }
 
     /// Read a reference.
@@ -1879,6 +2412,17 @@ impl StateStore {
     }
 
     /// Compare-and-swap move for a reference.
+    ///
+    /// On the durable backing this is one fenced FeltDB transaction: it commits
+    /// only if, at the moment of commit, the ref is still at
+    /// `expected_version` and the target revision is still present, whichever
+    /// `StateStore` or `FeltDb` handle the competing writes came through.
+    ///
+    /// The request `(ref incarnation, expected_version, target, authority,
+    /// metadata)` is idempotent: repeating one that already committed — after a
+    /// lost response, a restart or a compaction — returns `Ok` and does not
+    /// advance the ref again. Any other request for the same expected version
+    /// is a `VersionConflict`.
     pub fn compare_and_swap_ref(
         &self,
         namespace: &str,
@@ -1891,29 +2435,45 @@ impl StateStore {
         if !valid_ref_component(namespace) || !valid_ref_component(name) {
             return Err(RefError::InvalidIdentifier);
         }
-        if !self.exists(&target) {
-            return Err(RefError::UnknownTarget(target));
-        }
-        let _guard = self.mutation_lock.lock().unwrap();
-        let mut current = self.get_ref(namespace, name).ok_or(RefError::NotFound)?;
-        if current.version != expected_version {
-            return Err(RefError::VersionConflict {
-                expected: expected_version,
-                current: current.version,
-            });
-        }
-        current.target = target;
-        current.version += 1;
-        current.authority = authority;
-        current.updated_at_ms = current_time_ms();
-        if let Some(metadata) = metadata {
-            current.metadata = metadata;
+        match &self.backing {
+            Backing::Durable(feltdb) => durable::compare_and_swap_ref(
+                feltdb,
+                namespace,
+                name,
+                expected_version,
+                target,
+                authority,
+                metadata,
+            ),
+            Backing::Volatile(_) => {
+                let _guard = self.mutation_lock.lock().unwrap();
+                let mut current = self.get_ref(namespace, name).ok_or(RefError::NotFound)?;
+                if current.version != expected_version {
+                    return Err(RefError::VersionConflict {
+                        expected: expected_version,
+                        current: current.version,
+                    });
+                }
+                if !self.exists(&target) {
+                    return Err(RefError::UnknownTarget(target));
+                }
+                current.target = target;
+                current.version += 1;
+                current.authority = authority;
+                current.updated_at_ms = current_time_ms();
+                if let Some(metadata) = metadata {
+                    current.metadata = metadata;
+                }
+                self.write_volatile_ref(&current);
+                Ok(current)
+            }
         }
-        self.write_ref(&current)?;
-        Ok(current)
     }
 
     /// Delete a reference, optionally enforcing the current version.
+    ///
+    /// On the durable backing this is one fenced FeltDB transaction, and a
+    /// repeat of an identical delete that already committed returns `Ok`.
     pub fn delete_ref(
         &self,
         namespace: &str,
@@ -1923,17 +2483,29 @@ impl StateStore {
         if !valid_ref_component(namespace) || !valid_ref_component(name) {
             return Err(RefError::InvalidIdentifier);
         }
-        let _guard = self.mutation_lock.lock().unwrap();
-        let current = self.get_ref(namespace, name).ok_or(RefError::NotFound)?;
-        if let Some(expected) = expected_version {
-            if current.version != expected {
-                return Err(RefError::VersionConflict {
-                    expected,
-                    current: current.version,
-                });
+        match &self.backing {
+            Backing::Durable(feltdb) => {
+                durable::delete_ref(feltdb, namespace, name, expected_version)
+            }
+            Backing::Volatile(state) => {
+                let _guard = self.mutation_lock.lock().unwrap();
+                let current = self.get_ref(namespace, name).ok_or(RefError::NotFound)?;
+                if let Some(expected) = expected_version {
+                    if current.version != expected {
+                        return Err(RefError::VersionConflict {
+                            expected,
+                            current: current.version,
+                        });
+                    }
+                }
+                state
+                    .lock()
+                    .unwrap()
+                    .refs
+                    .remove(&(namespace.to_string(), name.to_string()));
+                Ok(())
             }
         }
-        self.remove_ref(namespace, name)
     }
 
     /// Enumerate refs, optionally scoped to one namespace.
@@ -1991,95 +2563,122 @@ impl StateStore {
         Ok(projected)
     }
 
-    /// Collect revisions and objects unreachable from current refs.
+    /// Collect revisions and objects that no root can reach.
+    ///
+    /// **Roots** are: every ref target; every resource's latest revision (each
+    /// revision at the resource's highest sequence, and the head the record
+    /// write path will mint its next revision from); and every revision
+    /// younger than 15 minutes (`GC_GRACE_MS`). **Reachable** is the roots and all their
+    /// ancestors by `parent_id`, because history, diff, rollback and replicated
+    /// reconstruction all walk that chain. Only a revision that is old, named
+    /// by no ref, not latest, and an ancestor of none of those is collected —
+    /// in practice, an abandoned fork. Retention (`keep_last`) is unaffected:
+    /// it is positional and runs at the mint, not here.
+    ///
+    /// On the durable backing every deletion commits in one FeltDB transaction
+    /// fenced on the state generation, which every ref change and every
+    /// committed revision advances. A concurrent change aborts the collection
+    /// and it is re-planned from fresh reads; see [`durable`].
     pub fn collect_unreachable(&self) -> Result<GarbageCollectionReport, GarbageCollectionError> {
-        let _guard = self.mutation_lock.lock().unwrap();
+        self.collect_unreachable_at(current_time_ms())
+    }
+
+    /// [`collect_unreachable`](Self::collect_unreachable) with an explicit
+    /// clock, so the grace period can be tested without waiting for it.
+    pub(crate) fn collect_unreachable_at(
+        &self,
+        now_ms: u64,
+    ) -> Result<GarbageCollectionReport, GarbageCollectionError> {
+        match &self.backing {
+            Backing::Volatile(state) => {
+                let _guard = self.mutation_lock.lock().unwrap();
+                let report = self.plan_collection(now_ms, &[]);
+                let mut state = state.lock().unwrap();
+                for id in &report.collected_revisions {
+                    state.revisions.remove(id);
+                }
+                for id in &report.collected_objects {
+                    state.objects.remove(id);
+                }
+                Ok(report)
+            }
+            Backing::Durable(feltdb) => durable::collect(self, feltdb, now_ms),
+        }
+    }
+
+    /// What a collection at `now_ms` would keep and delete, from one read of
+    /// refs and revisions. `extra_roots` are heads known outside `StateStore`.
+    fn plan_collection(&self, now_ms: u64, extra_roots: &[StateId]) -> GarbageCollectionReport {
         let roots = self.list_refs(None);
+        let revisions = self.all_revisions();
+        let by_id: HashMap<StateId, &StateRevision> =
+            revisions.iter().map(|revision| (revision.id.clone(), revision)).collect();
+
+        let mut latest: HashMap<&str, u64> = HashMap::new();
+        for revision in &revisions {
+            let entry = latest.entry(revision.resource.as_str()).or_insert(revision.sequence);
+            if revision.sequence > *entry {
+                *entry = revision.sequence;
+            }
+        }
+
+        let mut pending: Vec<StateId> = roots.iter().map(|reference| reference.target.clone()).collect();
+        pending.extend(extra_roots.iter().cloned());
+        for revision in &revisions {
+            let is_latest = latest.get(revision.resource.as_str()) == Some(&revision.sequence);
+            let is_young = now_ms.saturating_sub(revision.timestamp_ms) < GC_GRACE_MS;
+            if is_latest || is_young {
+                pending.push(revision.id.clone());
+            }
+        }
+
         let mut reachable_revisions = BTreeSet::new();
         let mut reachable_objects = BTreeSet::new();
-        let mut pending: Vec<StateId> = roots.iter().map(|reference| reference.target.clone()).collect();
         while let Some(id) = pending.pop() {
+            let Some(revision) = by_id.get(&id) else {
+                // A root or parent that is not held (expired by retention, or
+                // never committed) ends the walk; it is not an error.
+                continue;
+            };
             if !reachable_revisions.insert(id.clone()) {
                 continue;
             }
-            let Some(revision) = self.get(&id) else {
-                continue;
-            };
             if let Some(parent_id) = revision.parent_id.clone() {
                 pending.push(parent_id);
             }
-            for object_id in collect_referenced_objects(&revision) {
-                reachable_objects.insert(object_id);
-            }
-        }
-
-        let mut all_revisions: Vec<StateId> = self.all_revisions().into_iter().map(|r| r.id).collect();
-        all_revisions.sort();
-        let mut collected_revisions = Vec::new();
-        for revision_id in all_revisions {
-            if !reachable_revisions.contains(&revision_id) {
-                self.remove(&revision_id)
-                    .map_err(|error| GarbageCollectionError::Storage(error.to_string()))?;
-                collected_revisions.push(revision_id);
-            }
+            reachable_objects.extend(collect_referenced_objects(revision));
         }
 
-        let mut all_objects: Vec<StateId> = self.all_objects().into_iter().map(|o| o.id).collect();
-        all_objects.sort();
-        let mut collected_objects = Vec::new();
-        for object_id in all_objects {
-            if !reachable_objects.contains(&object_id) {
-                self.remove_object(&object_id)?;
-                collected_objects.push(object_id);
-            }
-        }
+        let mut collected_revisions: Vec<StateId> = revisions
+            .iter()
+            .map(|revision| revision.id.clone())
+            .filter(|id| !reachable_revisions.contains(id))
+            .collect();
+        collected_revisions.sort();
+        let mut collected_objects: Vec<StateId> = self
+            .all_objects()
+            .into_iter()
+            .map(|object| object.id)
+            .filter(|id| !reachable_objects.contains(id))
+            .collect();
+        collected_objects.sort();
 
-        let mut reachable_revisions_out: Vec<_> = reachable_revisions.into_iter().collect();
-        reachable_revisions_out.sort();
-        let mut reachable_objects_out: Vec<_> = reachable_objects.into_iter().collect();
-        reachable_objects_out.sort();
-        Ok(GarbageCollectionReport {
+        GarbageCollectionReport {
             roots,
-            reachable_revisions: reachable_revisions_out,
-            reachable_objects: reachable_objects_out,
+            reachable_revisions: reachable_revisions.into_iter().collect(),
+            reachable_objects: reachable_objects.into_iter().collect(),
             collected_revisions,
             collected_objects,
-        })
-    }
-
-    fn write_ref(&self, reference: &StateRef) -> Result<(), RefError> {
-        match &self.backing {
-            Backing::Volatile(state) => {
-                state.lock().unwrap().refs.insert(
-                    (reference.namespace.clone(), reference.name.clone()),
-                    reference.clone(),
-                );
-            }
-            Backing::Durable(feltdb) => {
-                feltdb
-                    .insert(&ref_key(&reference.namespace, &reference.name), reference.clone())
-                    .map_err(|error| RefError::Storage(error.to_string()))?;
-            }
         }
-        Ok(())
     }
 
-    fn remove_ref(&self, namespace: &str, name: &str) -> Result<(), RefError> {
-        match &self.backing {
-            Backing::Volatile(state) => {
-                state
-                    .lock()
-                    .unwrap()
-                    .refs
-                    .remove(&(namespace.to_string(), name.to_string()));
-            }
-            Backing::Durable(feltdb) => {
-                feltdb
-                    .delete(&ref_key(namespace, name))
-                    .map_err(|error| RefError::Storage(error.to_string()))?;
-            }
+    fn write_volatile_ref(&self, reference: &StateRef) {
+        if let Backing::Volatile(state) = &self.backing {
+            state.lock().unwrap().refs.insert(
+                (reference.namespace.clone(), reference.name.clone()),
+                reference.clone(),
+            );
         }
-        Ok(())
     }
 
     fn all_revisions(&self) -> Vec<StateRevision> {
@@ -2234,20 +2833,6 @@ impl StateStore {
         Ok(())
     }
 
-    fn remove_object(&self, id: &StateId) -> Result<(), GarbageCollectionError> {
-        match &self.backing {
-            Backing::Volatile(state) => {
-                state.lock().unwrap().objects.remove(id);
-            }
-            Backing::Durable(feltdb) => {
-                feltdb
-                    .delete(&object_key(id))
-                    .map_err(|error| GarbageCollectionError::Storage(error.to_string()))?;
-            }
-        }
-        Ok(())
-    }
-
     fn retention_state(&self, resource: &str) -> RetentionState {
         match &self.backing {
             Backing::Volatile(state) => state
@@ -2316,6 +2901,120 @@ mod tests {
     use super::*;
     use serde_json::json;
 
+    // -- garbage collection eligibility, with an explicit clock -------------
+
+    /// `A(0) → B(1) → C(2)` on one resource, plus a fork `F(1)` off `A`.
+    fn forked(store: &StateStore) -> (StateRevision, StateRevision, StateRevision, StateRevision) {
+        let a = store.create("doc:gc", json!({"n": "a"}).to_string(), "t".into()).unwrap();
+        let b = store.commit(json!({"n": "b"}).to_string(), &a, "t".into()).unwrap();
+        let c = store.commit(json!({"n": "c"}).to_string(), &b, "t".into()).unwrap();
+        let f = store.commit(json!({"n": "f"}).to_string(), &a, "t".into()).unwrap();
+        (a, b, c, f)
+    }
+
+    #[test]
+    fn gc_grace_boundary_is_exclusive() {
+        let store = StateStore::new_volatile();
+        let (_, _, _, f) = forked(&store);
+        let inside = store.collect_unreachable_at(f.timestamp_ms + GC_GRACE_MS - 1).unwrap();
+        assert!(inside.collected_revisions.is_empty(), "younger than the grace: a root");
+        let outside = store.collect_unreachable_at(f.timestamp_ms + GC_GRACE_MS).unwrap();
+        assert_eq!(outside.collected_revisions, vec![f.id.clone()]);
+        assert!(!store.exists(&f.id));
+    }
+
+    #[test]
+    fn gc_keeps_latest_revisions_refs_and_all_their_ancestors() {
+        for store in [StateStore::new_volatile(), durable_store().1] {
+            let (a, b, c, f) = forked(&store);
+            let g = store.commit(json!({"n": "g"}).to_string(), &f, "t".into()).unwrap();
+            // g ties c at the highest sequence (2): both are "latest".
+            let far = c.timestamp_ms + 365 * 24 * 60 * 60 * 1000;
+            let report = store.collect_unreachable_at(far).unwrap();
+            assert!(report.collected_revisions.is_empty(), "every revision is a root or an ancestor");
+
+            // A later revision on the main line makes f and g a stale fork …
+            let d = store.commit(json!({"n": "d"}).to_string(), &c, "t".into()).unwrap();
+            // … but a ref on g keeps g and its ancestor f.
+            store
+                .create_ref("candidates", "g", g.id.clone(), "t".into(), BTreeMap::new())
+                .unwrap();
+            assert!(store.collect_unreachable_at(far).unwrap().collected_revisions.is_empty());
+
+            store.delete_ref("candidates", "g", None).unwrap();
+            let mut collected = store.collect_unreachable_at(far).unwrap().collected_revisions;
+            collected.sort();
+            let mut expected = vec![f.id.clone(), g.id.clone()];
+            expected.sort();
+            assert_eq!(collected, expected, "only the unreferenced stale fork");
+            for kept in [&a, &b, &c, &d] {
+                assert!(store.exists(&kept.id));
+            }
+            // The fork's content objects went with it; the kept ones stayed.
+            let objects: BTreeSet<StateId> = store.all_objects().into_iter().map(|o| o.id).collect();
+            for kept in [&a, &b, &c, &d] {
+                assert!(objects.contains(&kept.content_id), "object of a kept revision");
+            }
+            for gone in [&f, &g] {
+                assert!(!objects.contains(&gone.content_id), "object of a collected revision");
+            }
+        }
+    }
+
+    fn durable_store() -> (tempfile::TempDir, StateStore, Arc<crate::FeltDb>) {
+        let dir = tempfile::TempDir::new().unwrap();
+        let db = Arc::new(crate::FeltDb::open(dir.path().join("gc.log")).unwrap());
+        let store = StateStore::with_feltdb(db.clone()).unwrap();
+        (dir, store, db)
+    }
+
+    /// The record write path mints from its own head, which an explicit
+    /// `StateStore::commit` fork can overtake by sequence. That head must stay
+    /// a root, or the next record write would mint a child of a deleted parent.
+    #[test]
+    fn gc_keeps_the_write_path_head_even_when_a_fork_is_longer() {
+        let (_dir, store, db) = durable_store();
+        db.insert("tasks:1", json!({"n": 0})).unwrap();
+        db.update("tasks:1", json!({"n": 1})).unwrap();
+        let resource = store.resources().into_iter().find(|r| r.contains("tasks")).unwrap();
+        let history = store.history_of(&resource);
+        assert_eq!(history.len(), 2, "the write path minted two revisions");
+        let (w0, w1) = (history[0].clone(), history[1].clone());
+
+        // A longer explicit fork off w0: x(1) → x2(2) overtakes w1(1).
+        let x = store.commit(json!({"x": 1}).to_string(), &w0, "t".into()).unwrap();
+        let x2 = store.commit(json!({"x": 2}).to_string(), &x, "t".into()).unwrap();
+        assert_eq!(store.head_of(&resource).unwrap().id, x2.id);
+        assert!(db.revision_head_ids().contains(&w1.id), "the write path still mints from w1");
+
+        let far = x2.timestamp_ms + 365 * 24 * 60 * 60 * 1000;
+        let report = store.collect_unreachable_at(far).unwrap();
+        assert!(!report.collected_revisions.contains(&w1.id), "w1 is the write path's head");
+        assert!(store.exists(&w1.id));
+
+        db.update("tasks:1", json!({"n": 2})).unwrap();
+        let w2 = store
+            .history_of(&resource)
+            .into_iter()
+            .find(|r| r.parent_id.as_ref() == Some(&w1.id))
+            .expect("the next write minted a child of w1");
+        assert_eq!(store.parent_of(&w2.id), Some(ParentLookup::Revision(Box::new(w1))));
+    }
+
+    /// The generation row, the incarnation rows and the ref rows the durable
+    /// path writes are invisible to the history and ref listings.
+    #[test]
+    fn internal_rows_do_not_appear_as_history_or_refs() {
+        let (_dir, store, _db) = durable_store();
+        let (a, _, _, _) = forked(&store);
+        store.create_ref("heads", "main", a.id.clone(), "t".into(), BTreeMap::new()).unwrap();
+        store.delete_ref("heads", "main", Some(1)).unwrap();
+        store.create_ref("heads", "main", a.id.clone(), "t".into(), BTreeMap::new()).unwrap();
+        assert_eq!(store.list_refs(None).len(), 1);
+        assert_eq!(store.resources(), vec!["doc:gc".to_string()]);
+        assert_eq!(store.history_of("doc:gc").len(), 4);
+    }
+
     #[test]
     fn test_state_id_deterministic() {
         let json = r#"{"name":"Alice","age":30}"#;