@feltdb/core 0.6.9 → 0.6.11

package/dist/create/server-source/crates/feltdb/src/multi_operation_transaction.rs

@@ -0,0 +1,659 @@
+//! Multi-Operation Atomicity (Production Gate P1)
+//!
+//! A transaction groups several operations and guarantees that a reader sees
+//! all of them or none of them — never a subset — across commit, crash,
+//! restart, replay, concurrent writers, and replication.
+//!
+//! # How the guarantee is obtained
+//!
+//! The durable log is the *only* source of truth for committed state. A
+//! transaction becomes durable when, and only when, its complete record has
+//! been written and fsynced as a single log entry. Observable state is derived
+//! by replaying committed records.
+//!
+//! ```text
+//!   transaction { A, B, C }
+//!            |
+//!            v
+//!   preflight every precondition        <- no state has been touched yet
+//!            |
+//!            v
+//!   one record, one write, one fsync    <- the atomic commit point
+//!            |
+//!            v
+//!   apply A, B, C to derived state      <- cannot fail; preflight proved it
+//! ```
+//!
+//! A crash before the fsync leaves no complete record, so recovery replays
+//! nothing. A crash after the fsync leaves a complete record, so recovery
+//! replays all of A, B, and C. A crash *during* the write leaves a torn tail,
+//! which is not a complete record and is discarded. There is no interleaving
+//! that produces a partial transaction, because partial application is never
+//! durable in the first place.
+//!
+//! Two properties do the real work:
+//!
+//! 1. **Preflight before commit.** Every precondition is evaluated against the
+//!    current state before anything is written. A transaction that would fail
+//!    part-way through is rejected before it can be committed, so the apply
+//!    step after commit cannot fail.
+//! 2. **Integrity per record.** Each record carries a checksum over its body.
+//!    A record that does not parse, or whose checksum does not match, was not
+//!    completely written and therefore never committed.
+//!
+//! # Boundary
+//!
+//! This guarantee holds because derived state is rebuilt from the log. If a
+//! checkpointing layer is added later, it must checkpoint only at transaction
+//! boundaries; a checkpoint taken mid-apply would reintroduce exactly the
+//! partial-visibility problem this module removes.
+
+use crate::state_hash::CanonicalState;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use sha2::{Digest, Sha256};
+use std::collections::HashSet;
+use std::fs::{File, OpenOptions};
+use std::io::{BufRead, BufReader, Seek, SeekFrom, Write};
+use std::path::{Path, PathBuf};
+
+/// What a single operation does to one record.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum TransactionOp {
+    /// Create or replace the record with this value.
+    Set(Value),
+    /// Remove the record.
+    Delete,
+}
+
+/// A condition that must hold before the transaction is allowed to commit.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum Precondition {
+    /// The record must currently exist.
+    MustExist,
+    /// The record must not currently exist.
+    MustNotExist,
+    /// The record must currently hold exactly this value.
+    ValueEquals(Value),
+}
+
+/// One operation within a transaction.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TransactionOperation {
+    pub collection: String,
+    pub record_id: String,
+    pub op: TransactionOp,
+    pub precondition: Option<Precondition>,
+}
+
+impl TransactionOperation {
+    pub fn set(collection: &str, record_id: &str, value: Value) -> Self {
+        Self {
+            collection: collection.to_string(),
+            record_id: record_id.to_string(),
+            op: TransactionOp::Set(value),
+            precondition: None,
+        }
+    }
+
+    pub fn delete(collection: &str, record_id: &str) -> Self {
+        Self {
+            collection: collection.to_string(),
+            record_id: record_id.to_string(),
+            op: TransactionOp::Delete,
+            precondition: None,
+        }
+    }
+
+    pub fn requiring(mut self, precondition: Precondition) -> Self {
+        self.precondition = Some(precondition);
+        self
+    }
+
+    fn key(&self) -> (String, String) {
+        (self.collection.clone(), self.record_id.clone())
+    }
+}
+
+/// The durable unit of atomicity.
+///
+/// A record is committed if and only if it appears in the log completely and
+/// its checksum verifies. Identity is `transaction_id`, which is what makes
+/// replay idempotent across restart and replication.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TransactionRecord {
+    pub transaction_id: String,
+    pub node_id: String,
+    pub sequence: u64,
+    pub operations: Vec<TransactionOperation>,
+    /// Checksum over the canonical body. Detects a partially written record.
+    pub checksum: String,
+}
+
+impl TransactionRecord {
+    pub fn new(
+        transaction_id: &str,
+        node_id: &str,
+        sequence: u64,
+        operations: Vec<TransactionOperation>,
+    ) -> Self {
+        let mut record = Self {
+            transaction_id: transaction_id.to_string(),
+            node_id: node_id.to_string(),
+            sequence,
+            operations,
+            checksum: String::new(),
+        };
+        record.checksum = record.compute_checksum();
+        record
+    }
+
+    fn compute_checksum(&self) -> String {
+        // The checksum covers everything except itself, in a deterministic
+        // encoding, so any truncation or mutation of the body is detectable.
+        let body = serde_json::json!({
+            "transaction_id": self.transaction_id,
+            "node_id": self.node_id,
+            "sequence": self.sequence,
+            "operations": self.operations,
+        });
+        let mut hasher = Sha256::new();
+        hasher.update(body.to_string().as_bytes());
+        format!("{:x}", hasher.finalize())
+    }
+
+    /// Whether this record was written completely and has not been altered.
+    pub fn verify_checksum(&self) -> bool {
+        self.checksum == self.compute_checksum()
+    }
+
+    /// Records touched by this transaction, for conflict detection.
+    pub fn touched_keys(&self) -> HashSet<(String, String)> {
+        self.operations.iter().map(|operation| operation.key()).collect()
+    }
+}
+
+/// Why a transaction was refused. No state is modified in any of these cases.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum RejectionReason {
+    /// A precondition did not hold. Carries the operation index and a reason.
+    PreconditionFailed { operation_index: usize, detail: String },
+    /// The transaction contained no operations.
+    Empty,
+    /// The transaction writes the same record twice, so its outcome would
+    /// depend on operation order rather than on the transaction as a whole.
+    ConflictingWritesWithinTransaction { collection: String, record_id: String },
+}
+
+/// The outcome of applying a record to derived state.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum ApplyOutcome {
+    /// Every operation was applied.
+    Applied,
+    /// This transaction id was already applied; applying it again is a no-op.
+    AlreadyApplied,
+    /// Nothing was applied.
+    Rejected(RejectionReason),
+}
+
+/// A log entry that could not be read.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum LogDefect {
+    /// The final entry was incomplete: the process died while writing it.
+    /// This transaction never committed and is discarded.
+    TornTail { bytes: usize },
+    /// An entry before the end is unreadable. This is corruption, not a torn
+    /// write, and recovery refuses to guess.
+    InteriorCorruption { line_number: usize, detail: String },
+}
+
+/// What recovery found in the log.
+#[derive(Clone, Debug)]
+pub struct RecoveryReport {
+    pub committed: Vec<TransactionRecord>,
+    /// Present when an incomplete final entry was discarded and truncated.
+    pub discarded_tail: Option<LogDefect>,
+}
+
+/// Where to stop when simulating a crash. Used by the crash-injection harness.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum CrashPoint {
+    /// Die before any byte reaches the log.
+    BeforeWrite,
+    /// Die having written only part of the record: a torn tail.
+    DuringWrite { keep_percent: u8 },
+    /// Die after the bytes were written but before fsync.
+    AfterWriteBeforeFsync,
+    /// Die after the record is durable but before it was applied to state.
+    AfterFsyncBeforeApply,
+    /// Die part-way through applying operations to in-memory state.
+    DuringApply { after_operations: usize },
+    /// Die after everything completed.
+    AfterApply,
+}
+
+/// Append-only transaction log with per-record integrity.
+pub struct AtomicTransactionLog {
+    path: PathBuf,
+    file: File,
+}
+
+impl AtomicTransactionLog {
+    /// Open a log, recovering it first.
+    ///
+    /// An incomplete final entry is truncated away before the log accepts new
+    /// writes. Without this, the next append would splice onto a torn record
+    /// and turn a recoverable tail into interior corruption.
+    pub fn open<P: AsRef<Path>>(path: P) -> Result<(Self, RecoveryReport), std::io::Error> {
+        let path = path.as_ref().to_path_buf();
+        if let Some(parent) = path.parent() {
+            std::fs::create_dir_all(parent)?;
+        }
+
+        let report = Self::read_log(&path)?;
+        if let Some(LogDefect::TornTail { bytes }) = &report.discarded_tail {
+            let file = OpenOptions::new().write(true).open(&path)?;
+            let length = file.metadata()?.len();
+            file.set_len(length.saturating_sub(*bytes as u64))?;
+            file.sync_all()?;
+        }
+
+        let file = OpenOptions::new().create(true).append(true).open(&path)?;
+        Ok((Self { path, file }, report))
+    }
+
+    /// Read the log without modifying it.
+    pub fn read_log(path: &Path) -> Result<RecoveryReport, std::io::Error> {
+        if !path.exists() {
+            return Ok(RecoveryReport { committed: Vec::new(), discarded_tail: None });
+        }
+
+        let reader = BufReader::new(File::open(path)?);
+        let mut committed = Vec::new();
+        let mut defect = None;
+
+        // A line is only trustworthy once we know it is terminated, so entries
+        // are collected with their raw bytes and the final one is judged last.
+        let mut entries: Vec<(usize, String, bool)> = Vec::new();
+        let mut buffer = Vec::new();
+        let mut line_number = 0usize;
+        let mut reader = reader;
+        loop {
+            buffer.clear();
+            let read = reader.read_until(b'\n', &mut buffer)?;
+            if read == 0 {
+                break;
+            }
+            line_number += 1;
+            let terminated = buffer.ends_with(b"\n");
+            let text = String::from_utf8_lossy(&buffer).trim_end_matches('\n').to_string();
+            entries.push((line_number, text, terminated));
+        }
+
+        let last_index = entries.len().saturating_sub(1);
+        for (index, (number, text, terminated)) in entries.into_iter().enumerate() {
+            if text.trim().is_empty() {
+                continue;
+            }
+            let is_last = index == last_index;
+            let parsed = serde_json::from_str::<TransactionRecord>(&text);
+            let usable = match &parsed {
+                Ok(record) => terminated && record.verify_checksum(),
+                Err(_) => false,
+            };
+
+            if usable {
+                committed.push(parsed.expect("checked above"));
+                continue;
+            }
+
+            if is_last {
+                // An unterminated, unparseable, or checksum-failing final entry
+                // is a transaction that was still being written when the
+                // process died. It never committed.
+                defect = Some(LogDefect::TornTail { bytes: text.len() + usize::from(terminated) });
+            } else {
+                return Err(std::io::Error::new(
+                    std::io::ErrorKind::InvalidData,
+                    format!(
+                        "transaction log corrupted at line {}: {}",
+                        number,
+                        parsed.err().map(|error| error.to_string())
+                            .unwrap_or_else(|| "checksum mismatch".to_string())
+                    ),
+                ));
+            }
+        }
+
+        Ok(RecoveryReport { committed, discarded_tail: defect })
+    }
+
+    /// Commit a record: one write, one fsync. This is the atomic commit point.
+    pub fn commit(&mut self, record: &TransactionRecord) -> Result<(), std::io::Error> {
+        let json = serde_json::to_string(record)
+            .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
+        writeln!(self.file, "{}", json)?;
+        self.file.sync_all()?;
+        Ok(())
+    }
+
+    /// Commit while simulating a crash, for the crash-injection harness.
+    ///
+    /// Returns whether the record reached durable storage intact.
+    pub fn commit_with_crash(
+        &mut self,
+        record: &TransactionRecord,
+        crash: &CrashPoint,
+    ) -> Result<bool, std::io::Error> {
+        let json = serde_json::to_string(record)
+            .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?;
+
+        match crash {
+            CrashPoint::BeforeWrite => Ok(false),
+            CrashPoint::DuringWrite { keep_percent } => {
+                // Write only part of the record and never terminate it: exactly
+                // what a process death mid-write leaves behind.
+                let cut = json.len() * (*keep_percent).min(99) as usize / 100;
+                // Cut on a character boundary; the record is ASCII JSON, but a
+                // value could carry multibyte text.
+                let mut cut = cut.max(1).min(json.len().saturating_sub(1)).max(1);
+                while cut > 1 && !json.is_char_boundary(cut) {
+                    cut -= 1;
+                }
+                let partial = &json[..cut];
+                self.file.write_all(partial.as_bytes())?;
+                self.file.sync_all()?;
+                Ok(false)
+            }
+            CrashPoint::AfterWriteBeforeFsync => {
+                writeln!(self.file, "{}", json)?;
+                self.file.flush()?;
+                // Deliberately no sync_all. The record is complete, so whether
+                // it survives is the operating system's choice; either way it
+                // is all-or-nothing, never partial.
+                Ok(true)
+            }
+            _ => {
+                writeln!(self.file, "{}", json)?;
+                self.file.sync_all()?;
+                Ok(true)
+            }
+        }
+    }
+
+    pub fn path(&self) -> &Path {
+        &self.path
+    }
+
+    /// Bytes currently in the log, for harness assertions.
+    pub fn size_bytes(&self) -> Result<u64, std::io::Error> {
+        Ok(self.file.metadata()?.len())
+    }
+
+    /// Append raw bytes. Used only to simulate damage in tests.
+    pub fn append_raw(&mut self, bytes: &[u8]) -> Result<(), std::io::Error> {
+        self.file.write_all(bytes)?;
+        self.file.sync_all()?;
+        Ok(())
+    }
+
+    /// Overwrite bytes at an offset. Used only to simulate corruption in tests.
+    pub fn corrupt_at(&mut self, offset: u64, bytes: &[u8]) -> Result<(), std::io::Error> {
+        let mut file = OpenOptions::new().write(true).open(&self.path)?;
+        file.seek(SeekFrom::Start(offset))?;
+        file.write_all(bytes)?;
+        file.sync_all()?;
+        Ok(())
+    }
+}
+
+/// Applies committed transactions to derived state.
+///
+/// The executor never applies a transaction part-way. Preconditions are
+/// evaluated for every operation before any mutation is staged, and the staged
+/// mutations are then written together.
+pub struct TransactionExecutor {
+    state: CanonicalState,
+    applied: HashSet<String>,
+    applied_order: Vec<String>,
+}
+
+impl TransactionExecutor {
+    pub fn new() -> Self {
+        Self { state: CanonicalState::new(), applied: HashSet::new(), applied_order: Vec::new() }
+    }
+
+    pub fn state(&self) -> &CanonicalState {
+        &self.state
+    }
+
+    pub fn applied_transactions(&self) -> &HashSet<String> {
+        &self.applied
+    }
+
+    pub fn applied_order(&self) -> &[String] {
+        &self.applied_order
+    }
+
+    /// Evaluate every precondition without mutating anything.
+    ///
+    /// This runs before the commit, which is what allows the post-commit apply
+    /// to be infallible.
+    pub fn preflight(&self, record: &TransactionRecord) -> Result<(), RejectionReason> {
+        if record.operations.is_empty() {
+            return Err(RejectionReason::Empty);
+        }
+
+        let mut seen: HashSet<(String, String)> = HashSet::new();
+        for operation in &record.operations {
+            if !seen.insert(operation.key()) {
+                return Err(RejectionReason::ConflictingWritesWithinTransaction {
+                    collection: operation.collection.clone(),
+                    record_id: operation.record_id.clone(),
+                });
+            }
+        }
+
+        for (index, operation) in record.operations.iter().enumerate() {
+            let Some(precondition) = &operation.precondition else { continue };
+            let current = self.state.get_record(&operation.collection, &operation.record_id);
+            let failure = match (precondition, current) {
+                (Precondition::MustExist, None) => Some("record does not exist".to_string()),
+                (Precondition::MustNotExist, Some(_)) => Some("record already exists".to_string()),
+                (Precondition::ValueEquals(expected), Some(actual)) if actual != expected => {
+                    Some("record does not hold the expected value".to_string())
+                }
+                (Precondition::ValueEquals(_), None) => Some("record does not exist".to_string()),
+                _ => None,
+            };
+            if let Some(detail) = failure {
+                return Err(RejectionReason::PreconditionFailed { operation_index: index, detail });
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Apply a committed record. All operations, or none.
+    pub fn apply(&mut self, record: &TransactionRecord) -> ApplyOutcome {
+        if self.applied.contains(&record.transaction_id) {
+            return ApplyOutcome::AlreadyApplied;
+        }
+        if let Err(reason) = self.preflight(record) {
+            return ApplyOutcome::Rejected(reason);
+        }
+
+        // Preflight succeeded, so every mutation below is known to be legal.
+        // Staging them first keeps the mutation phase free of any decision that
+        // could fail half-way through.
+        let staged: Vec<(&TransactionOperation, ())> =
+            record.operations.iter().map(|operation| (operation, ())).collect();
+        for (operation, ()) in staged {
+            match &operation.op {
+                TransactionOp::Set(value) => self.state.set_record(
+                    operation.collection.clone(),
+                    operation.record_id.clone(),
+                    value.clone(),
+                ),
+                TransactionOp::Delete => {
+                    self.state
+                        .records
+                        .remove(&(operation.collection.clone(), operation.record_id.clone()));
+                }
+            }
+        }
+
+        self.applied.insert(record.transaction_id.clone());
+        self.applied_order.push(record.transaction_id.clone());
+        ApplyOutcome::Applied
+    }
+
+    /// Rebuild state by replaying committed records in order.
+    ///
+    /// Replay is idempotent: a record already applied is skipped, so recovery
+    /// can run repeatedly and converge on the same state.
+    pub fn recover(&mut self, records: &[TransactionRecord]) -> Vec<ApplyOutcome> {
+        records.iter().map(|record| self.apply(record)).collect()
+    }
+
+    /// Whether any operation of this transaction is visible in state.
+    ///
+    /// The atomicity assertion in the harness is that this is either all of
+    /// them or none of them.
+    pub fn visible_operations(&self, record: &TransactionRecord) -> usize {
+        record
+            .operations
+            .iter()
+            .filter(|operation| match &operation.op {
+                TransactionOp::Set(value) => {
+                    self.state.get_record(&operation.collection, &operation.record_id) == Some(value)
+                }
+                TransactionOp::Delete => {
+                    self.state.get_record(&operation.collection, &operation.record_id).is_none()
+                }
+            })
+            .count()
+    }
+}
+
+impl Default for TransactionExecutor {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+/// A node that commits transactions durably and derives state from the log.
+///
+/// This is the unit the crash harness kills and restarts.
+pub struct TransactionalNode {
+    log: AtomicTransactionLog,
+    executor: TransactionExecutor,
+    node_id: String,
+    next_sequence: u64,
+}
+
+impl TransactionalNode {
+    /// Start a node, recovering derived state from the durable log.
+    pub fn open<P: AsRef<Path>>(path: P, node_id: &str) -> Result<(Self, RecoveryReport), std::io::Error> {
+        let (log, report) = AtomicTransactionLog::open(path)?;
+        let mut executor = TransactionExecutor::new();
+        executor.recover(&report.committed);
+        let next_sequence = report.committed.iter().map(|record| record.sequence).max().unwrap_or(0) + 1;
+        Ok((Self { log, executor, node_id: node_id.to_string(), next_sequence }, report))
+    }
+
+    pub fn state(&self) -> &CanonicalState {
+        self.executor.state()
+    }
+
+    pub fn executor(&self) -> &TransactionExecutor {
+        &self.executor
+    }
+
+    pub fn log(&mut self) -> &mut AtomicTransactionLog {
+        &mut self.log
+    }
+
+    pub fn build(&mut self, transaction_id: &str, operations: Vec<TransactionOperation>) -> TransactionRecord {
+        let record = TransactionRecord::new(transaction_id, &self.node_id, self.next_sequence, operations);
+        self.next_sequence += 1;
+        record
+    }
+
+    /// Commit a transaction: preflight, durably record, then apply.
+    ///
+    /// A rejected transaction is never written to the log, so a failed
+    /// precondition leaves no trace in durable state.
+    pub fn commit(&mut self, record: &TransactionRecord) -> Result<ApplyOutcome, std::io::Error> {
+        if self.executor.applied_transactions().contains(&record.transaction_id) {
+            return Ok(ApplyOutcome::AlreadyApplied);
+        }
+        if let Err(reason) = self.executor.preflight(record) {
+            return Ok(ApplyOutcome::Rejected(reason));
+        }
+        self.log.commit(record)?;
+        Ok(self.executor.apply(record))
+    }
+
+    /// Commit while simulating a crash at the given point.
+    ///
+    /// Returns what the in-memory executor saw before the process "died". What
+    /// matters is not this value but what a restarted node recovers.
+    pub fn commit_with_crash(
+        &mut self,
+        record: &TransactionRecord,
+        crash: &CrashPoint,
+    ) -> Result<(), std::io::Error> {
+        if self.executor.preflight(record).is_err() {
+            return Ok(());
+        }
+        let durable = self.log.commit_with_crash(record, crash)?;
+        if !durable {
+            return Ok(());
+        }
+        match crash {
+            CrashPoint::AfterFsyncBeforeApply => Ok(()),
+            CrashPoint::DuringApply { after_operations } => {
+                // Deliberately apply only a prefix, in memory, to model a
+                // process that died mid-apply. This state is never persisted;
+                // recovery rebuilds from the log.
+                for operation in record.operations.iter().take(*after_operations) {
+                    if let TransactionOp::Set(value) = &operation.op {
+                        self.executor.state.set_record(
+                            operation.collection.clone(),
+                            operation.record_id.clone(),
+                            value.clone(),
+                        );
+                    }
+                }
+                Ok(())
+            }
+            _ => {
+                self.executor.apply(record);
+                Ok(())
+            }
+        }
+    }
+
+    /// Accept a transaction replicated from another node.
+    ///
+    /// Applying a replicated transaction is the same all-or-nothing path, and
+    /// a transaction already applied here is ignored, so duplicate delivery is
+    /// safe.
+    pub fn accept_replicated(&mut self, record: &TransactionRecord) -> Result<ApplyOutcome, std::io::Error> {
+        if !record.verify_checksum() {
+            return Ok(ApplyOutcome::Rejected(RejectionReason::PreconditionFailed {
+                operation_index: 0,
+                detail: "replicated record failed its integrity check".to_string(),
+            }));
+        }
+        if self.executor.applied_transactions().contains(&record.transaction_id) {
+            return Ok(ApplyOutcome::AlreadyApplied);
+        }
+        if let Err(reason) = self.executor.preflight(record) {
+            return Ok(ApplyOutcome::Rejected(reason));
+        }
+        self.log.commit(record)?;
+        Ok(self.executor.apply(record))
+    }
+}