@feltdb/core 0.15.0 → 0.16.0

package/dist/create/server-source/crates/feltdb/src/durable_operation_identity.rs

@@ -1,482 +0,0 @@
-//! Durable operation identity.
-//!
-//! `(origin, sequence)` is an identity, not a counter. Receivers deduplicate on
-//! it, so two operations must never be assigned the same one -- and that has to
-//! hold across a process restart, which is where it did not.
-//!
-//! `dedup_bound_investigation.rs` established the failure: `feltdb_node` kept
-//! the counter in memory and started it at 1 on every launch, while the causal
-//! clock was rebuilt from the log. A restarted origin came back knowing its
-//! causal position and not its identity position, re-issued identities its own
-//! earlier operations already held, and its peers discarded the new writes as
-//! duplicates -- with no error and no divergence signal.
-//!
-//! The authority is now the durable log, and specifically the log rather than
-//! the causal clock. Those are **different identity domains**: the clock
-//! component advances only on a successful apply, the sequence is burned by an
-//! attempt, and they disagree by exactly the number of refused transactions.
-//! `replication-convergence.md` fixes that the sequence is sparse and that
-//! making it dense is a separate deferred decision, so nothing here derives
-//! identity from the clock and nothing here makes the sequence dense.
-//!
-//! The invariant, stated precisely, because the weaker version is not enough:
-//!
-//! > no two **persisted** envelopes from one origin ever share a sequence.
-//!
-//! Not "restart usually finds the highest sequence". An identity is reserved
-//! forever exactly when an envelope bearing it becomes durable, which is why a
-//! number burned and then lost to a crash is safe to re-issue -- nothing
-//! carrying it was ever written.
-
-#[cfg(test)]
-mod tests {
-    use crate::convergence::VectorClock;
-    use crate::distributed_transactions::{
-        DistributedTransactionExecutor, ReceiveOutcome, ReplicationMessage, SubmitOutcome,
-        TransactionEnvelope,
-    };
-    use crate::state_hash::StateHash;
-    use crate::transactions::{
-        ConsistencyContract, Operation, OperationCommand, OperationId, StateVersion,
-    };
-    use std::collections::HashMap;
-    use std::path::PathBuf;
-
-    fn zero_version() -> StateVersion {
-        StateVersion::new(VectorClock::new(), "0".repeat(64))
-    }
-
-    fn operation_at(node: &str, sequence: u64, payload: &str) -> Operation {
-        let mut fields = HashMap::new();
-        fields.insert("record".to_string(), serde_json::json!(payload));
-        Operation::new(
-            OperationId::new(node.to_string(), sequence),
-            zero_version(),
-            format!("tx_{payload}"),
-            OperationCommand {
-                op_type: "set".to_string(),
-                collection: "items".to_string(),
-                record_id: payload.to_string(),
-                fields,
-            },
-            node.to_string(),
-        )
-    }
-
-    /// Open a node over a log path, replaying it when it already exists --
-    /// which is what the node binary does on start.
-    fn open(node: &str, path: &PathBuf) -> DistributedTransactionExecutor {
-        let mut executor = DistributedTransactionExecutor::with_log(
-            node.to_string(),
-            StateHash::from_hex("0".repeat(64)),
-            Some(path.clone()),
-        )
-        .expect("executor");
-        if path.exists() {
-            executor.load_from_disk(path.clone()).expect("recover");
-        }
-        executor
-    }
-
-    /// Commit as the node binary does: chain the parent version forward on
-    /// success, and give each operation a distinct id.
-    ///
-    /// The operation id is taken from the executor's next identity, which is
-    /// exactly what the binary does now that the executor owns allocation.
-    fn commit(
-        executor: &mut DistributedTransactionExecutor,
-        node: &str,
-        payload: &str,
-    ) -> TransactionEnvelope {
-        let sequence = executor.next_origin_sequence();
-        let parent = chained_version(executor, node);
-        match executor
-            .submit_local_transaction(
-                format!("tx_{payload}"),
-                parent,
-                vec![operation_at(node, sequence, payload)],
-                ConsistencyContract::local(),
-            )
-            .expect("submit")
-        {
-            SubmitOutcome::Admitted(envelope) => envelope,
-            SubmitOutcome::CapacityExceeded { .. } => panic!("unexpected capacity refusal"),
-        }
-    }
-
-    /// The parent version this node would commit on top of.
-    fn chained_version(executor: &DistributedTransactionExecutor, node: &str) -> StateVersion {
-        match executor.get_replica_state(node) {
-            Some(replica) => {
-                StateVersion::new(replica.vector_clock.clone(), replica.state_hash.to_string())
-            }
-            None => zero_version(),
-        }
-    }
-
-    fn message(envelope: TransactionEnvelope) -> ReplicationMessage {
-        let origin = envelope.originating_node.clone();
-        let sequence = envelope.envelope_id.sequence;
-        ReplicationMessage::new(envelope, origin, "receiver".to_string(), sequence)
-    }
-
-    // -- allocation --------------------------------------------------------
-
-    #[test]
-    fn identity_a_fresh_origin_starts_at_one() {
-        let dir = tempfile::tempdir().unwrap();
-        let mut node = open("A", &dir.path().join("operations.jsonl"));
-        assert_eq!(
-            node.next_origin_sequence(),
-            1,
-            "a brand-new origin legitimately starts at 1"
-        );
-        assert_eq!(commit(&mut node, "A", "first").envelope_id.sequence, 1);
-    }
-
-    #[test]
-    fn identity_restart_after_three_operations_continues_at_four() {
-        let dir = tempfile::tempdir().unwrap();
-        let path = dir.path().join("operations.jsonl");
-        {
-            let mut node = open("A", &path);
-            for payload in ["one", "two", "three"] {
-                commit(&mut node, "A", payload);
-            }
-            assert_eq!(node.next_origin_sequence(), 4);
-        }
-        let restarted = open("A", &path);
-        assert_eq!(
-            restarted.next_origin_sequence(),
-            4,
-            "identity is recovered from the durable log, not reset"
-        );
-    }
-
-    #[test]
-    fn identity_is_recovered_even_without_replaying_the_log() {
-        // `with_log` recovers identity too, so a caller that opens an existing
-        // log and issues an operation without calling load_from_disk still
-        // cannot re-issue an identity that log contains.
-        let dir = tempfile::tempdir().unwrap();
-        let path = dir.path().join("operations.jsonl");
-        {
-            let mut node = open("A", &path);
-            commit(&mut node, "A", "one");
-            commit(&mut node, "A", "two");
-        }
-        let reopened = DistributedTransactionExecutor::with_log(
-            "A".to_string(),
-            StateHash::from_hex("0".repeat(64)),
-            Some(path),
-        )
-        .expect("executor");
-        assert_eq!(reopened.next_origin_sequence(), 3);
-    }
-
-    #[test]
-    fn identity_spaces_are_independent_per_origin() {
-        // A receiver that has applied another node's high sequences must not
-        // inflate its own identity space: the log holds replicated envelopes
-        // too, and another node's sequence says nothing about this one's.
-        let dir = tempfile::tempdir().unwrap();
-        let path = dir.path().join("operations.jsonl");
-        let mut receiver = open("receiver", &path);
-
-        for position in 1..=9u64 {
-            let mut envelope = TransactionEnvelope::new(
-                "A".to_string(),
-                position,
-                format!("tx_a{position}"),
-                vec![operation_at("A", position, &format!("a{position}"))],
-                ConsistencyContract::local(),
-            );
-            let mut clock = VectorClock::new();
-            clock.clocks.insert("A".to_string(), position);
-            envelope.vector_clock = clock;
-            receiver
-                .receive_replicated(message(envelope), zero_version())
-                .expect("receive");
-        }
-
-        assert_eq!(
-            receiver.next_origin_sequence(),
-            1,
-            "nine applied envelopes from A leave this node's own identity space untouched"
-        );
-        assert_eq!(
-            commit(&mut receiver, "receiver", "mine")
-                .envelope_id
-                .sequence,
-            1
-        );
-
-        let restarted = open("receiver", &path);
-        assert_eq!(
-            restarted.next_origin_sequence(),
-            2,
-            "and recovery filters by origin too"
-        );
-    }
-
-    // -- crash boundaries --------------------------------------------------
-
-    #[test]
-    fn identity_survives_a_crash_after_the_operation_became_durable() {
-        let dir = tempfile::tempdir().unwrap();
-        let path = dir.path().join("operations.jsonl");
-        {
-            let mut node = open("A", &path);
-            commit(&mut node, "A", "durable");
-            // The process dies here: no clean shutdown, no flush of any
-            // in-memory counter. The envelope is already fsynced.
-        }
-        let restarted = open("A", &path);
-        assert_eq!(
-            restarted.next_origin_sequence(),
-            2,
-            "the identity of a durable operation is never re-issued"
-        );
-    }
-
-    #[test]
-    fn identity_burned_without_persistence_is_safely_reissued_after_a_crash() {
-        // allocate -> crash before persistence -> restart.
-        //
-        // The number is re-issued, and that is correct rather than a lapse: no
-        // envelope bearing it was ever written, so nothing can collide with it.
-        // Reserving it forever would leak identity on every refused
-        // transaction and would make the durable sequence advance past
-        // operations that never existed.
-        let dir = tempfile::tempdir().unwrap();
-        let path = dir.path().join("operations.jsonl");
-        {
-            let mut node = open("A", &path);
-            commit(&mut node, "A", "one");
-            let burned = node.allocate_origin_sequence();
-            assert_eq!(burned, 2, "claimed but never persisted");
-        }
-        let mut restarted = open("A", &path);
-        assert_eq!(
-            restarted.next_origin_sequence(),
-            2,
-            "the log holds only sequence 1, so 2 is free and is re-issued"
-        );
-        assert_eq!(commit(&mut restarted, "A", "two").envelope_id.sequence, 2);
-    }
-
-    #[test]
-    fn identity_allocation_stays_sparse_because_the_frozen_contract_says_so() {
-        // A burned number leaves a gap within a single process lifetime.
-        // Making the sequence dense is a separate deferred decision
-        // (replication-convergence.md) and is deliberately not made here.
-        let dir = tempfile::tempdir().unwrap();
-        let mut node = open("A", &dir.path().join("operations.jsonl"));
-        commit(&mut node, "A", "one");
-        let _burned = node.allocate_origin_sequence();
-        assert_eq!(
-            commit(&mut node, "A", "three").envelope_id.sequence,
-            3,
-            "the attempt burned 2; the sequence is sparse, as the contract fixes it"
-        );
-    }
-
-    #[test]
-    fn identity_is_not_burned_by_a_capacity_refusal() {
-        // A refusal admits nothing, so it must not consume an identity either:
-        // there is no operation for that number to account for.
-        use crate::distributed_transactions::CausalCapacity;
-        let dir = tempfile::tempdir().unwrap();
-        let mut node = open("A", &dir.path().join("operations.jsonl"));
-        node.set_causal_capacity(CausalCapacity::new(0, usize::MAX));
-
-        let before = node.next_origin_sequence();
-        let outcome = node
-            .submit_local_transaction(
-                "tx_refused".to_string(),
-                zero_version(),
-                vec![operation_at("A", 1, "refused")],
-                ConsistencyContract::local(),
-            )
-            .expect("submit");
-        assert!(matches!(outcome, SubmitOutcome::CapacityExceeded { .. }));
-        assert_eq!(
-            node.next_origin_sequence(),
-            before,
-            "a refusal consumes no identity"
-        );
-    }
-
-    #[test]
-    fn identity_holds_across_many_restart_cycles() {
-        let dir = tempfile::tempdir().unwrap();
-        let path = dir.path().join("operations.jsonl");
-        let mut issued = Vec::new();
-        for cycle in 0..25 {
-            let mut node = open("A", &path);
-            for index in 0..4 {
-                issued.push(
-                    commit(&mut node, "A", &format!("c{cycle}_{index}"))
-                        .envelope_id
-                        .sequence,
-                );
-            }
-        }
-        let mut unique = issued.clone();
-        unique.sort_unstable();
-        unique.dedup();
-        assert_eq!(
-            unique.len(),
-            issued.len(),
-            "100 operations across 25 restart cycles produced no repeated identity"
-        );
-        assert_eq!(issued.len(), 100);
-    }
-
-    // -- the regression, at executor level ---------------------------------
-
-    #[test]
-    fn identity_a_receiver_accepts_writes_from_an_origin_that_restarted() {
-        // The exact failure #210 found, now at the executor boundary.
-        let origin_dir = tempfile::tempdir().unwrap();
-        let origin_path = origin_dir.path().join("operations.jsonl");
-        let receiver_dir = tempfile::tempdir().unwrap();
-        let mut receiver = open("receiver", &receiver_dir.path().join("operations.jsonl"));
-
-        let before = {
-            let mut origin = open("A", &origin_path);
-            commit(&mut origin, "A", "before")
-        };
-        assert!(matches!(
-            receiver
-                .receive_replicated(message(before), zero_version())
-                .expect("receive"),
-            ReceiveOutcome::Applied { .. }
-        ));
-
-        // The origin restarts and commits again.
-        let after = {
-            let mut origin = open("A", &origin_path);
-            commit(&mut origin, "A", "after")
-        };
-        assert_eq!(
-            after.envelope_id.sequence, 2,
-            "the identity did not restart"
-        );
-
-        let outcome = receiver
-            .receive_replicated(message(after), zero_version())
-            .expect("receive");
-        assert!(
-            matches!(outcome, ReceiveOutcome::Applied { .. }),
-            "a write from a restarted origin must be applied, not discarded: {outcome:?}"
-        );
-        assert_eq!(
-            receiver.causal_frontier().get("A"),
-            2,
-            "and the frontier advances"
-        );
-
-        let state = receiver.canonical_state().expect("state");
-        assert!(state
-            .records
-            .contains_key(&("items".to_string(), "after".to_string())));
-        assert!(state
-            .records
-            .contains_key(&("items".to_string(), "before".to_string())));
-    }
-
-    #[test]
-    fn identity_a_genuine_retransmission_is_still_refused() {
-        // The fix must not weaken deduplication: the same envelope twice is
-        // still one operation, and a *new* operation after it is still new.
-        let origin_dir = tempfile::tempdir().unwrap();
-        let receiver_dir = tempfile::tempdir().unwrap();
-        let mut origin = open("A", &origin_dir.path().join("operations.jsonl"));
-        let mut receiver = open("receiver", &receiver_dir.path().join("operations.jsonl"));
-
-        let first = commit(&mut origin, "A", "one");
-        receiver
-            .receive_replicated(message(first.clone()), zero_version())
-            .expect("receive");
-        assert_eq!(
-            receiver
-                .receive_replicated(message(first), zero_version())
-                .expect("receive"),
-            ReceiveOutcome::AlreadyKnown,
-            "a retransmission is refused"
-        );
-
-        let second = commit(&mut origin, "A", "two");
-        assert!(
-            matches!(
-                receiver
-                    .receive_replicated(message(second), zero_version())
-                    .expect("receive"),
-                ReceiveOutcome::Applied { .. }
-            ),
-            "and a new operation after a duplicate is still accepted"
-        );
-        assert_eq!(receiver.causal_frontier().get("A"), 2);
-    }
-
-    #[test]
-    fn identity_no_collision_when_the_origin_restarts_while_the_receiver_holds_a_backlog() {
-        // The two mechanisms have to compose: the receiver is holding causal
-        // backlog from this origin when the origin restarts, so a colliding
-        // identity would land on a pending entry rather than an applied one.
-        let origin_dir = tempfile::tempdir().unwrap();
-        let origin_path = origin_dir.path().join("operations.jsonl");
-        let receiver_dir = tempfile::tempdir().unwrap();
-        let mut receiver = open("receiver", &receiver_dir.path().join("operations.jsonl"));
-
-        let (first, second) = {
-            let mut origin = open("A", &origin_path);
-            (
-                commit(&mut origin, "A", "one"),
-                commit(&mut origin, "A", "two"),
-            )
-        };
-
-        // Only the second is delivered, so it is held waiting for the first.
-        assert!(matches!(
-            receiver
-                .receive_replicated(message(second), zero_version())
-                .expect("receive"),
-            ReceiveOutcome::PendingDependencies { .. }
-        ));
-        assert_eq!(receiver.pending_causal_count(), 1);
-
-        // The origin restarts, with the receiver still holding that backlog.
-        let third = {
-            let mut origin = open("A", &origin_path);
-            commit(&mut origin, "A", "three")
-        };
-        assert_eq!(
-            third.envelope_id.sequence, 3,
-            "no collision with the held entry"
-        );
-
-        let held = receiver
-            .receive_replicated(message(third), zero_version())
-            .expect("receive");
-        assert!(
-            matches!(held, ReceiveOutcome::PendingDependencies { .. }),
-            "it is held on its own merits, not mistaken for the entry already pending: {held:?}"
-        );
-        assert_eq!(receiver.pending_causal_count(), 2);
-
-        // The gap closes and everything applies exactly once.
-        receiver
-            .receive_replicated(message(first), zero_version())
-            .expect("receive");
-        assert_eq!(receiver.pending_causal_count(), 0);
-        assert_eq!(receiver.causal_frontier().get("A"), 3);
-        let state = receiver.canonical_state().expect("state");
-        for record in ["one", "two", "three"] {
-            assert!(state
-                .records
-                .contains_key(&("items".to_string(), record.to_string())));
-        }
-    }
-}

package/dist/create/server-source/crates/feltdb/src/durable_operation_log.rs

@@ -1,203 +0,0 @@
-/// Durable Operation Log
-///
-/// Provides persistent storage for operation envelopes with fsync durability.
-/// The log is the source of truth for recovery; replica state and dedup sets are derived from it.
-///
-/// Invariant: Every successfully persisted operation can be recovered after process restart.
-use crate::distributed_transactions::TransactionEnvelope;
-use std::fs::{File, OpenOptions};
-use std::io::{BufRead, BufReader, Write};
-use std::path::{Path, PathBuf};
-
-/// Durable operation log with fsync semantics
-pub struct OperationLog {
-    path: PathBuf,
-    file: Option<File>,
-}
-
-impl OperationLog {
-    /// Create or open a durable log at the given path
-    pub fn new<P: AsRef<Path>>(path: P) -> Result<Self, std::io::Error> {
-        let path = path.as_ref().to_path_buf();
-
-        // Create parent directory if needed
-        if let Some(parent) = path.parent() {
-            std::fs::create_dir_all(parent)?;
-        }
-
-        // Open log file in append mode
-        let file = OpenOptions::new().create(true).append(true).open(&path)?;
-
-        Ok(Self {
-            path,
-            file: Some(file),
-        })
-    }
-
-    /// Append and durably persist an operation envelope
-    /// Returns error if write or fsync fails
-    pub fn append(&mut self, envelope: &TransactionEnvelope) -> Result<(), std::io::Error> {
-        if let Some(ref mut file) = self.file {
-            // Serialize envelope to JSON line
-            let json = serde_json::to_string(envelope)
-                .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
-
-            // Write JSON line (newline-delimited)
-            writeln!(file, "{}", json)?;
-
-            // Force durable persistence to disk
-            file.sync_all()?;
-
-            Ok(())
-        } else {
-            Err(std::io::Error::new(
-                std::io::ErrorKind::NotConnected,
-                "Operation log file not open",
-            ))
-        }
-    }
-
-    /// Load all persisted envelopes from log
-    /// Returns envelopes in order they were persisted
-    pub fn load_all(&self) -> Result<Vec<TransactionEnvelope>, std::io::Error> {
-        if !self.path.exists() {
-            return Ok(Vec::new());
-        }
-
-        let file = File::open(&self.path)?;
-        let reader = BufReader::new(file);
-        let mut envelopes = Vec::new();
-
-        for line in reader.lines() {
-            let line = line?;
-            if line.trim().is_empty() {
-                continue;
-            }
-
-            match serde_json::from_str::<TransactionEnvelope>(&line) {
-                Ok(envelope) => envelopes.push(envelope),
-                Err(e) => {
-                    return Err(std::io::Error::new(
-                        std::io::ErrorKind::InvalidData,
-                        format!("Failed to deserialize envelope: {}", e),
-                    ))
-                }
-            }
-        }
-
-        Ok(envelopes)
-    }
-
-    /// Get the number of persisted operations
-    pub fn count(&self) -> Result<usize, std::io::Error> {
-        let envelopes = self.load_all()?;
-        Ok(envelopes.len())
-    }
-
-    /// Clear the log (used for testing)
-    pub fn clear(&mut self) -> Result<(), std::io::Error> {
-        if let Some(file) = self.file.take() {
-            drop(file);
-        }
-
-        std::fs::remove_file(&self.path)?;
-
-        let file = OpenOptions::new()
-            .create(true)
-            .append(true)
-            .open(&self.path)?;
-
-        self.file = Some(file);
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::transactions::{ConsistencyContract, ConsistencyLevel};
-    use tempfile::TempDir;
-
-    fn create_test_envelope(sequence: u64) -> TransactionEnvelope {
-        TransactionEnvelope::new(
-            "node1".to_string(),
-            sequence,
-            format!("tx{}", sequence),
-            Vec::new(), // Empty operations for test
-            ConsistencyContract {
-                level: ConsistencyLevel::Causal,
-                timeout_ms: Some(1000),
-                required_acks: None,
-            },
-        )
-    }
-
-    #[test]
-    fn test_operation_log_append() -> Result<(), Box<dyn std::error::Error>> {
-        let temp_dir = TempDir::new()?;
-        let log_path = temp_dir.path().join("test.log");
-
-        let mut log = OperationLog::new(&log_path)?;
-        let envelope = create_test_envelope(1);
-
-        log.append(&envelope)?;
-
-        assert!(log_path.exists());
-        Ok(())
-    }
-
-    #[test]
-    fn test_operation_log_load() -> Result<(), Box<dyn std::error::Error>> {
-        let temp_dir = TempDir::new()?;
-        let log_path = temp_dir.path().join("test.log");
-
-        let mut log = OperationLog::new(&log_path)?;
-        let envelope1 = create_test_envelope(1);
-        let envelope2 = create_test_envelope(2);
-
-        log.append(&envelope1)?;
-        log.append(&envelope2)?;
-
-        let loaded = log.load_all()?;
-        assert_eq!(loaded.len(), 2);
-        assert_eq!(loaded[0].envelope_id.sequence, 1);
-        assert_eq!(loaded[1].envelope_id.sequence, 2);
-
-        Ok(())
-    }
-
-    #[test]
-    fn test_operation_log_persistence() -> Result<(), Box<dyn std::error::Error>> {
-        let temp_dir = TempDir::new()?;
-        let log_path = temp_dir.path().join("test.log");
-
-        // Write to log
-        {
-            let mut log = OperationLog::new(&log_path)?;
-            let envelope = create_test_envelope(1);
-            log.append(&envelope)?;
-        }
-
-        // Reload from disk
-        {
-            let log = OperationLog::new(&log_path)?;
-            let loaded = log.load_all()?;
-            assert_eq!(loaded.len(), 1);
-            assert_eq!(loaded[0].envelope_id.sequence, 1);
-        }
-
-        Ok(())
-    }
-
-    #[test]
-    fn test_operation_log_empty() -> Result<(), Box<dyn std::error::Error>> {
-        let temp_dir = TempDir::new()?;
-        let log_path = temp_dir.path().join("nonexistent.log");
-
-        let log = OperationLog::new(&log_path)?;
-        let loaded = log.load_all()?;
-        assert_eq!(loaded.len(), 0);
-
-        Ok(())
-    }
-}