@feltdb/core 0.6.12 → 0.6.14

package/dist/create/server-source/crates/feltdb/src/causal_dependency_barrier.rs

@@ -88,6 +88,12 @@ pub struct PendingOperation {
     pub operation_id: String,
     pub required_dependencies: VectorClock,
     pub state: OperationState,
+    /// Size on the durable log of the envelope this entry stands for.
+    ///
+    /// The entry does not hold the payload -- that stays in the operation log
+    /// -- but the backlog's byte bound is about payload volume, so the size has
+    /// to be carried even though the bytes are not.
+    pub payload_bytes: usize,
 }
 
 impl PendingOperation {
@@ -96,6 +102,7 @@ impl PendingOperation {
             operation_id,
             required_dependencies,
             state: OperationState::Received,
+            payload_bytes: 0,
         }
     }
 }
@@ -114,6 +121,10 @@ pub struct CausalDependencyBarrier {
     metrics: BarrierMetrics,
     /// Phase 7.1-opt: Dependency index for selective candidate discovery
     dependency_index: DependencyIndex,
+    /// High-water mark of pending entries, for occupancy metrics.
+    max_pending_entries_seen: usize,
+    /// High-water mark of pending payload bytes, for occupancy metrics.
+    max_pending_payload_bytes_seen: usize,
 }
 
 impl CausalDependencyBarrier {
@@ -125,6 +136,8 @@ impl CausalDependencyBarrier {
             applied: Vec::new(),
             metrics: BarrierMetrics::default(),
             dependency_index: DependencyIndex::default(),
+            max_pending_entries_seen: 0,
+            max_pending_payload_bytes_seen: 0,
         }
     }
 
@@ -181,6 +194,7 @@ impl CausalDependencyBarrier {
                     required_clock: required_dependencies.clone(),
                     missing_dependencies: missing,
                 },
+                payload_bytes: 0,
             };
             self.pending.insert(operation_id.clone(), pending_op);
 
@@ -387,6 +401,200 @@ impl CausalDependencyBarrier {
         &self.local_frontier
     }
 
+    // ---------------------------------------------------------------------
+    // Production causal-delivery API.
+    //
+    // The methods above were built for, and are exercised by, in-process tests
+    // that drive the barrier directly. The methods below are what the
+    // replicated receive path uses. They differ in three ways that matter:
+    //
+    //  1. Eligibility is dominance, not `happens_before`. `receive_operation`
+    //     buffers when `local_frontier.happens_before(deps)`, which returns
+    //     false for concurrent clocks and would therefore admit an operation
+    //     whose dependencies are unmet whenever the frontier is ahead in some
+    //     unrelated dimension.
+    //  2. Release scanning does not consult the dependency index, because
+    //     `index_operation` is `#[cfg(test)]` and so the index is empty in a
+    //     shipping build. A full scan is correct at any build setting.
+    //  3. Nothing here holds a payload. Pending entries carry an identifier and
+    //     the dependency clock; the envelope itself is replayed from the
+    //     durable operation log when it becomes eligible.
+    // ---------------------------------------------------------------------
+
+    /// True when every dependency this clock names has already been applied.
+    pub fn is_eligible(&self, dependencies: &VectorClock) -> bool {
+        self.local_frontier.dominates(dependencies)
+    }
+
+    /// Retain an operation that cannot be applied yet.
+    ///
+    /// Returns false when the key is already pending, so a duplicate delivery
+    /// of a blocked envelope stays one logical operation.
+    pub fn retain_pending(&mut self, key: String, dependencies: VectorClock) -> bool {
+        self.retain_pending_sized(key, dependencies, 0)
+    }
+
+    /// Retain an operation that cannot be applied yet, recording the size of
+    /// the envelope it stands for.
+    ///
+    /// The entry still holds no payload. The size is carried because the
+    /// backlog's byte bound is about how much durable payload is outstanding,
+    /// and that number is not recoverable from metadata alone.
+    ///
+    /// Returns false when the key is already pending, so a duplicate delivery
+    /// of a blocked envelope stays one logical operation.
+    pub fn retain_pending_sized(
+        &mut self,
+        key: String,
+        dependencies: VectorClock,
+        payload_bytes: usize,
+    ) -> bool {
+        if self.pending.contains_key(&key) {
+            return false;
+        }
+        let missing = self.compute_missing_dependencies(&dependencies);
+        self.pending.insert(
+            key.clone(),
+            PendingOperation {
+                operation_id: key,
+                required_dependencies: dependencies.clone(),
+                state: OperationState::WaitingForDependencies {
+                    required_clock: dependencies,
+                    missing_dependencies: missing,
+                },
+                payload_bytes,
+            },
+        );
+        self.observe_occupancy();
+        true
+    }
+
+    /// Record the high-water marks after the pending set grows.
+    ///
+    /// Occupancy metrics have to report maximum as well as current, because a
+    /// backlog that filled and drained between two reads is invisible to a
+    /// current-value gauge, and "did this ever approach its bound" is the
+    /// question an operator actually has.
+    fn observe_occupancy(&mut self) {
+        let entries = self.pending.len();
+        if entries > self.max_pending_entries_seen {
+            self.max_pending_entries_seen = entries;
+        }
+        let bytes = self.pending_payload_bytes();
+        if bytes > self.max_pending_payload_bytes_seen {
+            self.max_pending_payload_bytes_seen = bytes;
+        }
+    }
+
+    /// Durable payload bytes the backlog is currently waiting on.
+    ///
+    /// This is the volume of log the pending entries stand for, not a heap
+    /// measurement: the payloads are on disk. It is what the byte bound is
+    /// expressed in.
+    pub fn pending_payload_bytes(&self) -> usize {
+        self.pending.values().map(|op| op.payload_bytes).sum()
+    }
+
+    /// The most pending entries ever held at once.
+    pub fn max_pending_entries_seen(&self) -> usize {
+        self.max_pending_entries_seen
+    }
+
+    /// The most pending payload bytes ever held at once.
+    pub fn max_pending_payload_bytes_seen(&self) -> usize {
+        self.max_pending_payload_bytes_seen
+    }
+
+    /// Record that an operation was applied, advancing the frontier.
+    ///
+    /// The frontier advances here and nowhere else on the production path, so
+    /// it always describes what has been *successfully applied* rather than
+    /// what has been received or claimed.
+    pub fn record_applied(&mut self, key: &str, applied_clock: &VectorClock) {
+        self.pending.remove(key);
+        self.local_frontier.merge(applied_clock);
+        self.applied.push(key.to_string());
+    }
+
+    /// Every pending operation whose dependencies the frontier now satisfies.
+    ///
+    /// Returns them in a stable order so that two replicas releasing the same
+    /// set release it identically.
+    pub fn eligible_pending(&self) -> Vec<String> {
+        let mut ready: Vec<String> = self
+            .pending
+            .iter()
+            .filter(|(_, op)| self.local_frontier.dominates(&op.required_dependencies))
+            .map(|(key, _)| key.clone())
+            .collect();
+        ready.sort();
+        ready
+    }
+
+    /// Whether this key is currently held as pending.
+    pub fn is_pending(&self, key: &str) -> bool {
+        self.pending.contains_key(key)
+    }
+
+    /// Keys of applied operations, in the order they were applied.
+    pub fn applied_keys(&self) -> &[String] {
+        &self.applied
+    }
+
+    /// Keys currently held as pending, sorted.
+    pub fn pending_keys(&self) -> Vec<String> {
+        let mut keys: Vec<String> = self.pending.keys().cloned().collect();
+        keys.sort();
+        keys
+    }
+
+    /// Occupancy of the pending set: how many operations are waiting.
+    ///
+    /// Instrumentation only. What may be retained, and what happens when that
+    /// limit is reached, is deliberately not decided here -- see
+    /// docs/architecture/causal-backlog-bound.md.
+    pub fn pending_count(&self) -> usize {
+        self.pending.len()
+    }
+
+    /// Approximate heap footprint of the pending metadata, in bytes.
+    ///
+    /// Counts the key, both clock copies an entry carries, and the missing
+    /// dependency set, because the cost of a pending entry scales with cluster
+    /// size rather than with the size of the operation it stands for.
+    pub fn pending_metadata_bytes(&self) -> usize {
+        self.pending
+            .iter()
+            .map(|(key, op)| {
+                let clock_bytes = |clock: &VectorClock| {
+                    clock
+                        .clocks
+                        .keys()
+                        .map(|node| node.len() + std::mem::size_of::<u64>())
+                        .sum::<usize>()
+                };
+                let state_bytes = match &op.state {
+                    OperationState::WaitingForDependencies {
+                        required_clock,
+                        missing_dependencies,
+                    } => {
+                        clock_bytes(required_clock)
+                            + missing_dependencies
+                                .iter()
+                                .map(|dep| dep.len() + std::mem::size_of::<u64>())
+                                .sum::<usize>()
+                    }
+                    _ => 0,
+                };
+                key.len()
+                    + op.operation_id.len()
+                    + clock_bytes(&op.required_dependencies)
+                    + state_bytes
+                    + std::mem::size_of::<PendingOperation>()
+            })
+            .sum()
+    }
+
     /// Manually advance frontier (for testing or external coordination)
     pub fn advance_frontier(&mut self, clock: &VectorClock) {
         #[cfg(test)]

package/dist/create/server-source/crates/feltdb/src/convergence.rs

@@ -74,6 +74,22 @@ impl VectorClock {
         !self.happens_before(other) && !other.happens_before(self)
     }
 
+    /// True when this clock has seen at least everything `other` has, in every
+    /// dimension.
+    ///
+    /// This is the test a causal receiver needs, and it is not `happens_before`.
+    /// `happens_before` answers a question about ordering and returns false for
+    /// concurrent clocks, so a receiver using it would admit an operation whose
+    /// dependencies are genuinely unmet whenever the frontier happened to be
+    /// ahead in some unrelated dimension. Dominance asks only "have I seen
+    /// everything this operation needs", which is what eligibility means.
+    pub fn dominates(&self, other: &VectorClock) -> bool {
+        other
+            .clocks
+            .iter()
+            .all(|(instance_id, required)| self.get(instance_id) >= *required)
+    }
+
     /// Compare two vector clocks for causal ordering
     pub fn causal_cmp(&self, other: &VectorClock) -> Ordering {
         if self == other {

package/dist/create/server-source/crates/feltdb/src/dedup_bound_investigation.rs

@@ -0,0 +1,402 @@
+//! Is `envelopes_seen` bounded by the causal backlog, or is it independent?
+//!
+//! An investigation, not a change. Nothing here modifies the implementation;
+//! every test asserts what the code does today, including where that is wrong.
+//!
+//! `adrs/replication-retention-and-recovery-lag.md` §8 raised this as the
+//! cheapest unresolved question under P5.3, and recorded it as a question
+//! rather than a decision because it argues against a frozen contract. The
+//! hypothesis was:
+//!
+//! > since causal enforcement landed, `frontier[origin] = n` may imply that
+//! > every clock position `1..n` from that origin has been applied, in which
+//! > case the frontier is a complete deduplication test for everything it
+//! > covers and `envelopes_seen` need only hold what is above it -- which the
+//! > backlog bound already bounds.
+//!
+//! The tests below establish the four steps that hypothesis needs, find that it
+//! holds, and then find that the deduplication *key* did not -- it was unsound
+//! across a restart of the originating node, and lost writes silently.
+//!
+//! That second finding has since been fixed: identity is now allocated by the
+//! executor from the durable log, in `durable_operation_identity.rs`. The two
+//! tests at the bottom are kept and still pass, because what they assert is the
+//! *receiver's* behaviour when it is handed a colliding identity, and that has
+//! deliberately not changed -- a receiver cannot tell a re-used identity from a
+//! retransmission, which is precisely why the origin must never produce one.
+//! They are the reason durable identity allocation is load-bearing rather than
+//! tidy.
+//!
+//! Two counters are involved and the whole question turns on their difference:
+//!
+//! ```text
+//!   envelope_id.sequence   caller-supplied; the node binary resets it to 1
+//!                          on restart and never restores it from the log
+//!   vector_clock[origin]   frontier.increment(origin) at stamp time; the
+//!                          frontier IS rebuilt from the log on restart
+//! ```
+
+#[cfg(test)]
+mod tests {
+    use crate::convergence::VectorClock;
+    use crate::distributed_transactions::{
+        DistributedTransactionExecutor, ReceiveOutcome, ReplicationMessage, TransactionEnvelope,
+    };
+    use crate::state_hash::StateHash;
+    use crate::transactions::{
+        ConsistencyContract, Operation, OperationCommand, OperationId, StateVersion,
+    };
+    use std::collections::HashMap;
+
+    fn zero_version() -> StateVersion {
+        StateVersion::new(VectorClock::new(), "0".repeat(64))
+    }
+
+    /// Build an envelope with an explicitly chosen envelope sequence and an
+    /// explicitly chosen causal clock, so the two counters can be moved
+    /// independently -- which is exactly what a restart of the origin does.
+    fn envelope(origin: &str, sequence: u64, clock: &[(&str, u64)], payload: &str) -> TransactionEnvelope {
+        let mut fields = HashMap::new();
+        fields.insert("record".to_string(), serde_json::json!(payload));
+        let operation = Operation::new(
+            OperationId::new(origin.to_string(), sequence),
+            zero_version(),
+            format!("tx_{payload}"),
+            OperationCommand {
+                op_type: "set".to_string(),
+                collection: "items".to_string(),
+                record_id: payload.to_string(),
+                fields,
+            },
+            origin.to_string(),
+        );
+        let mut built = TransactionEnvelope::new(
+            origin.to_string(),
+            sequence,
+            format!("tx_{payload}"),
+            vec![operation],
+            ConsistencyContract::local(),
+        );
+        let mut vector_clock = VectorClock::new();
+        for (node, value) in clock {
+            vector_clock.clocks.insert((*node).to_string(), *value);
+        }
+        built.vector_clock = vector_clock;
+        built
+    }
+
+    fn message(envelope: TransactionEnvelope) -> ReplicationMessage {
+        let origin = envelope.originating_node.clone();
+        let sequence = envelope.envelope_id.sequence;
+        ReplicationMessage::new(envelope, origin, "receiver".to_string(), sequence)
+    }
+
+    fn receiver() -> (DistributedTransactionExecutor, tempfile::TempDir) {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let executor = DistributedTransactionExecutor::with_log(
+            "receiver".to_string(),
+            StateHash::from_hex("0".repeat(64)),
+            Some(dir.path().join("operations.jsonl")),
+        )
+        .expect("executor");
+        (executor, dir)
+    }
+
+    // ---------------------------------------------------------------------
+    // Step 1: the frontier only advances one position at a time, per origin.
+    // ---------------------------------------------------------------------
+
+    #[test]
+    fn dedup_frontier_advances_exactly_one_position_per_applied_envelope() {
+        let (mut executor, _dir) = receiver();
+        for position in 1..=5u64 {
+            let outcome = executor
+                .receive_replicated(
+                    message(envelope("A", position, &[("A", position)], &format!("r{position}"))),
+                    zero_version(),
+                )
+                .expect("receive");
+            assert!(matches!(outcome, ReceiveOutcome::Applied { .. }));
+            assert_eq!(
+                executor.causal_frontier().get("A"),
+                position,
+                "the frontier tracks applied positions exactly, with no jumps"
+            );
+        }
+    }
+
+    // ---------------------------------------------------------------------
+    // Step 2: a foreign envelope cannot raise the frontier for another origin.
+    //
+    // This is the step the hypothesis actually rests on. `record_applied` calls
+    // `VectorClock::merge`, which takes the maximum per component, so an
+    // envelope from B carrying {A: 5} would raise this replica's frontier for A
+    // to 5 -- inventing knowledge of five A operations it never applied. Causal
+    // enforcement is what prevents it: that envelope's dependencies include
+    // A@5, so it cannot be applied until A@5 already has been, at which point
+    // the merge is a no-op for A.
+    // ---------------------------------------------------------------------
+
+    #[test]
+    fn dedup_a_foreign_envelope_cannot_invent_frontier_for_another_origin() {
+        let (mut executor, _dir) = receiver();
+
+        // B's envelope claims to have seen A through position 5.
+        let outcome = executor
+            .receive_replicated(message(envelope("B", 1, &[("A", 5), ("B", 1)], "b1")), zero_version())
+            .expect("receive");
+
+        assert!(
+            matches!(outcome, ReceiveOutcome::PendingDependencies { .. }),
+            "an envelope that depends on five unseen A operations must be held, got {outcome:?}"
+        );
+        assert_eq!(
+            executor.causal_frontier().get("A"),
+            0,
+            "and it must not have advanced the frontier for A: merge takes a maximum, \
+             so applying this would have invented knowledge of A@1..5"
+        );
+    }
+
+    #[test]
+    fn dedup_a_foreign_envelope_merge_is_a_noop_for_an_origin_already_covered() {
+        let (mut executor, _dir) = receiver();
+        for position in 1..=3u64 {
+            executor
+                .receive_replicated(
+                    message(envelope("A", position, &[("A", position)], &format!("a{position}"))),
+                    zero_version(),
+                )
+                .expect("receive");
+        }
+        assert_eq!(executor.causal_frontier().get("A"), 3);
+
+        // B has seen A through 3, which this replica has too.
+        executor
+            .receive_replicated(message(envelope("B", 1, &[("A", 3), ("B", 1)], "b1")), zero_version())
+            .expect("receive");
+
+        assert_eq!(
+            executor.causal_frontier().get("A"),
+            3,
+            "applying a foreign envelope leaves the covered origin where it was"
+        );
+    }
+
+    // ---------------------------------------------------------------------
+    // Step 3: therefore the frontier covers exactly what was applied.
+    // ---------------------------------------------------------------------
+
+    #[test]
+    fn dedup_every_applied_envelope_is_covered_by_the_frontier() {
+        let (mut executor, _dir) = receiver();
+
+        // Two origins, interleaved, plus an out-of-order arrival that is held
+        // and later released -- so the applied set is not simply arrival order.
+        executor.receive_replicated(message(envelope("A", 1, &[("A", 1)], "a1")), zero_version()).unwrap();
+        executor.receive_replicated(message(envelope("A", 3, &[("A", 3)], "a3")), zero_version()).unwrap();
+        executor.receive_replicated(message(envelope("A", 2, &[("A", 2)], "a2")), zero_version()).unwrap();
+        executor.receive_replicated(message(envelope("B", 1, &[("A", 3), ("B", 1)], "b1")), zero_version()).unwrap();
+
+        assert_eq!(executor.pending_causal_count(), 0, "everything was released");
+        assert_eq!(executor.causal_frontier().get("A"), 3);
+        assert_eq!(executor.causal_frontier().get("B"), 1);
+
+        // The claim: for every envelope this replica considers seen, the
+        // frontier already covers its clock position. So a frontier test would
+        // answer "already applied" for all of them.
+        let frontier = executor.causal_frontier().clone();
+        let log = executor.operation_log.as_ref().expect("log").load_all().expect("load");
+        for envelope in &log {
+            let origin = &envelope.originating_node;
+            let position = envelope.vector_clock.get(origin);
+            assert!(
+                frontier.get(origin) >= position,
+                "{origin}@{position} was applied but the frontier reports {}",
+                frontier.get(origin)
+            );
+        }
+        assert_eq!(log.len(), 4);
+    }
+
+    // ---------------------------------------------------------------------
+    // Step 4: and it survives a restart, because the frontier is rebuilt from
+    // the log while `envelopes_seen` is rebuilt from the same place.
+    // ---------------------------------------------------------------------
+
+    #[test]
+    fn dedup_frontier_coverage_survives_restart_and_still_refuses_duplicates() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let path = dir.path().join("operations.jsonl");
+        {
+            let mut executor = DistributedTransactionExecutor::with_log(
+                "receiver".to_string(),
+                StateHash::from_hex("0".repeat(64)),
+                Some(path.clone()),
+            )
+            .expect("executor");
+            for position in 1..=3u64 {
+                executor
+                    .receive_replicated(
+                        message(envelope("A", position, &[("A", position)], &format!("a{position}"))),
+                        zero_version(),
+                    )
+                    .expect("receive");
+            }
+        }
+
+        let mut recovered = DistributedTransactionExecutor::with_log(
+            "receiver".to_string(),
+            StateHash::from_hex("0".repeat(64)),
+            Some(path.clone()),
+        )
+        .expect("executor");
+        recovered.load_from_disk(path).expect("recover");
+
+        assert_eq!(recovered.causal_frontier().get("A"), 3, "the frontier is rebuilt from the log");
+
+        // A retransmission of an already-applied operation after the restart.
+        let again = recovered
+            .receive_replicated(message(envelope("A", 2, &[("A", 2)], "a2")), zero_version())
+            .expect("receive");
+        assert_eq!(again, ReceiveOutcome::AlreadyKnown, "a duplicate after restart is refused");
+    }
+
+    // ---------------------------------------------------------------------
+    // Growth: does `envelopes_seen` grow independently of the bounded
+    // pending/deferred population? It does, and that is the whole question.
+    // ---------------------------------------------------------------------
+
+    #[test]
+    fn dedup_seen_set_grows_with_history_while_the_backlog_stays_empty() {
+        let (mut executor, _dir) = receiver();
+        for position in 1..=200u64 {
+            executor
+                .receive_replicated(
+                    message(envelope("A", position, &[("A", position)], &format!("a{position}"))),
+                    zero_version(),
+                )
+                .expect("receive");
+        }
+
+        assert_eq!(executor.pending_causal_count(), 0, "the bounded population is empty");
+        assert_eq!(executor.deferred_causal_count(), 0);
+        assert_eq!(
+            executor.envelopes_seen.len(),
+            200,
+            "yet the deduplication set holds one entry per operation ever applied: \
+             it grows with history, not with the backlog, so the backlog bound does not bound it"
+        );
+
+        // And the frontier that could replace it is one integer per origin.
+        assert_eq!(
+            executor.causal_frontier().clocks.len(),
+            1,
+            "the frontier covering the same 200 operations is O(origins), not O(operations)"
+        );
+    }
+
+    // ---------------------------------------------------------------------
+    // Why identity must be durable: what a receiver does with a colliding one.
+    //
+    // These assert the receive path's behaviour when it is *handed* a re-used
+    // identity. That behaviour is unchanged and correct in isolation -- a
+    // receiver has no way to distinguish a re-used identity from a
+    // retransmission. The defect was that origins produced such identities
+    // after a restart, and that is fixed in durable_operation_identity.rs.
+    // These remain as the demonstration of what that fix prevents.
+    // ---------------------------------------------------------------------
+
+    #[test]
+    fn dedup_a_colliding_identity_is_dropped_silently_which_is_why_identity_must_be_durable() {
+        // Deduplication keys on `(origin, sequence)`. Hand the receiver a
+        // second operation carrying an identity it has already applied -- which
+        // is what a restarted origin used to produce, before identity was
+        // recovered from the durable log -- and it is discarded as a duplicate.
+        //
+        // The receiver is not wrong to do this. It cannot distinguish a re-used
+        // identity from a retransmission, and treating retransmissions as
+        // duplicates is a property the rest of the system depends on. The fix
+        // belongs on the allocating side, and this is what it prevents.
+        let (mut executor, _dir) = receiver();
+
+        // Before the origin restarts: A's first operation, sequence 1, clock 1.
+        let first = executor
+            .receive_replicated(message(envelope("A", 1, &[("A", 1)], "before")), zero_version())
+            .expect("receive");
+        assert!(matches!(first, ReceiveOutcome::Applied { .. }));
+
+        // A restarts. Its sequence counter resets to 1; its frontier is rebuilt
+        // from its own log, so the next operation is clock position 2. This is
+        // a genuinely new write, carrying different data.
+        let after_restart = envelope("A", 1, &[("A", 2)], "after");
+        let outcome = executor
+            .receive_replicated(message(after_restart.clone()), zero_version())
+            .expect("receive");
+
+        assert_eq!(
+            outcome,
+            ReceiveOutcome::AlreadyKnown,
+            "a write carrying an already-applied identity is discarded, silently"
+        );
+        assert_eq!(
+            executor.causal_frontier().get("A"),
+            1,
+            "the receiver's frontier stalls, so every later A operation is blocked behind \
+             a dependency that will never arrive"
+        );
+        let state = executor.canonical_state().expect("state");
+        assert!(
+            !state.records.contains_key(&("items".to_string(), "after".to_string())),
+            "and the write is simply absent: no error, no divergence signal, no retry"
+        );
+
+        // Note that the frontier test the ADR proposed would not have made this
+        // mistake -- the arriving clock position is 2 and the frontier is 1 --
+        // which is a second, independent reason to prefer it. Identity
+        // allocation was fixed first because it is the cheaper change and does
+        // not alter what deduplication means on the receive path.
+        let position = after_restart.vector_clock.get("A");
+        assert!(
+            position > executor.causal_frontier().get("A"),
+            "a deduplication test keyed on the causal position would have admitted this write"
+        );
+    }
+
+    #[test]
+    fn dedup_a_partial_identity_collision_strands_the_replica_permanently() {
+        // The compounding case, and the reason this was a correctness problem
+        // rather than a lost write. Of two operations from a restarted origin,
+        // the first collides and is dropped while the second does not collide
+        // and is admitted -- so the replica ends up holding an operation whose
+        // dependency was silently discarded, and waits for it forever. Which
+        // operations collide depends only on how many the origin committed
+        // before it died.
+        let (mut executor, _dir) = receiver();
+        executor
+            .receive_replicated(message(envelope("A", 1, &[("A", 1)], "one")), zero_version())
+            .expect("receive");
+
+        // Origin restarts after one operation and commits two more. Sequences
+        // 1 and 2; clock positions 2 and 3.
+        let dropped = executor
+            .receive_replicated(message(envelope("A", 1, &[("A", 2)], "two")), zero_version())
+            .expect("receive");
+        let admitted = executor
+            .receive_replicated(message(envelope("A", 2, &[("A", 3)], "three")), zero_version())
+            .expect("receive");
+
+        assert_eq!(dropped, ReceiveOutcome::AlreadyKnown, "the first collides and is lost");
+        assert!(
+            matches!(admitted, ReceiveOutcome::PendingDependencies { .. }),
+            "the second does not collide, is admitted, and is then held forever waiting on \
+             the operation that was silently dropped: {admitted:?}"
+        );
+        assert_eq!(executor.pending_causal_count(), 1);
+        assert_eq!(
+            executor.causal_frontier().get("A"),
+            1,
+            "the replica is permanently stuck one operation behind an origin that is still writing"
+        );
+    }
+}