@feltdb/core 0.8.3 → 0.8.4

package/dist/create/server-source/crates/feltdb/src/phase1c3_acceptance.rs

@@ -468,8 +468,17 @@ mod tests {
             StateHash::from_hex("hash0".to_string()),
         );
 
-        recovered_executor
-            .register_replica("nodeB".to_string(), StateHash::from_hex("hash0".to_string()));
+        // The peer is registered at the state the recovered node actually
+        // holds, read back rather than restated. `load_from_disk` recomputes
+        // the local replica's hash from the replayed log, so a fabricated
+        // literal here would describe a peer that has diverged and the
+        // convergence check below would be asserting the opposite of what this
+        // test is about.
+        let recovered_hash = recovered_executor
+            .get_replica_state("nodeA")
+            .expect("the recovered node has a local replica")
+            .state_hash;
+        recovered_executor.register_replica("nodeB".to_string(), recovered_hash);
 
         let mut recovery = RecoveryOrchestrator::new(recovered_executor, "nodeB".to_string());
         recovery.reconcile_with_peer(&peer)?;

package/dist/create/server-source/crates/feltdb/src/query_execution_diagnostics.rs

@@ -0,0 +1,126 @@
+//! Test instrumentation for the collection read paths a bounded query can take.
+//!
+//! PR33 attributed sustained-workload degradation to bounded queries cloning an
+//! entire collection under the state mutex before predicate, ordering and limit
+//! were applied. PR34 removes that clone, and a benchmark alone cannot prove a
+//! clone is gone: a faster number is evidence of speed, not of mechanism.
+//!
+//! These counters therefore record *which traversal a read used*, so a test can
+//! assert that a bounded query never entered `FeltDb::list_collection`. They
+//! are process-global, monotonic, carry no record data, and are not part of the
+//! query contract: nothing in the query path reads them, and no query result
+//! depends on them. The HTTP surface only exposes them when a server is started
+//! with `FELTDB_QUERY_DIAGNOSTICS=1`.
+//!
+//! PR35 extends them with execution attribution. An index is an optimization, so
+//! "the query got faster" is not evidence that it was used; these counters say
+//! which execution a query actually took, how many candidates the index
+//! produced, and how many records the predicate had to evaluate. That last
+//! figure is the one the PR35 claim rests on: for a selective indexed query it
+//! must track the candidate count rather than the collection size.
+
+use std::sync::atomic::{AtomicU64, Ordering};
+
+static FULL_COLLECTION_MATERIALIZATIONS: AtomicU64 = AtomicU64::new(0);
+static FULL_COLLECTION_RECORDS_CLONED: AtomicU64 = AtomicU64::new(0);
+static COLLECTION_SCANS: AtomicU64 = AtomicU64::new(0);
+static SCAN_RECORDS_VISITED: AtomicU64 = AtomicU64::new(0);
+static SCAN_RECORDS_MATERIALIZED: AtomicU64 = AtomicU64::new(0);
+static QUERIES_TOTAL: AtomicU64 = AtomicU64::new(0);
+static QUERIES_SCAN: AtomicU64 = AtomicU64::new(0);
+static QUERIES_INDEXED: AtomicU64 = AtomicU64::new(0);
+static INDEX_HITS: AtomicU64 = AtomicU64::new(0);
+static INDEX_MISSES: AtomicU64 = AtomicU64::new(0);
+static INDEX_CANDIDATES_EXAMINED: AtomicU64 = AtomicU64::new(0);
+static RECORDS_PREDICATE_EVALUATED: AtomicU64 = AtomicU64::new(0);
+
+/// A point-in-time reading of the collection read counters.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub struct QueryExecutionCounters {
+    /// Calls to `FeltDb::list_collection`, which clones a whole collection.
+    pub full_collection_materializations: u64,
+    /// Records cloned by those calls.
+    pub full_collection_records_cloned: u64,
+    /// Calls to `FeltDb::query_collection`, which evaluates borrowed records.
+    pub collection_scans: u64,
+    /// Records a scan predicate observed, borrowed rather than cloned.
+    pub scan_records_visited: u64,
+    /// Records a scan actually cloned, which is at most the matching set.
+    pub scan_records_materialized: u64,
+    /// Collection reads that executed a query: `queries_scan + queries_indexed`.
+    pub queries_total: u64,
+    /// Queries answered by evaluating the predicate over the whole collection.
+    pub queries_scan: u64,
+    /// Queries answered from equality-index candidates.
+    pub queries_indexed: u64,
+    /// Queries for which at least one condition had an applicable index.
+    pub index_hits: u64,
+    /// Queries for which no condition had an applicable index, which fall back.
+    pub index_misses: u64,
+    /// Candidate record keys the index produced, after intersection.
+    pub index_candidates_examined: u64,
+    /// Records the predicate was actually evaluated against, on either path.
+    /// For a selective indexed query this tracks the candidate count; for a scan
+    /// it is the collection size.
+    pub records_predicate_evaluated: u64,
+}
+
+pub(crate) fn record_full_materialization(records: usize) {
+    FULL_COLLECTION_MATERIALIZATIONS.fetch_add(1, Ordering::Relaxed);
+    FULL_COLLECTION_RECORDS_CLONED.fetch_add(records as u64, Ordering::Relaxed);
+}
+
+pub(crate) fn record_scan(visited: usize, materialized: usize) {
+    COLLECTION_SCANS.fetch_add(1, Ordering::Relaxed);
+    SCAN_RECORDS_VISITED.fetch_add(visited as u64, Ordering::Relaxed);
+    SCAN_RECORDS_MATERIALIZED.fetch_add(materialized as u64, Ordering::Relaxed);
+    QUERIES_TOTAL.fetch_add(1, Ordering::Relaxed);
+    QUERIES_SCAN.fetch_add(1, Ordering::Relaxed);
+    // A scan evaluates the predicate against every record it visits. This is
+    // the O(N) figure PR35 exists to reduce, recorded on the same counter the
+    // indexed path reports to, so the two are directly comparable.
+    RECORDS_PREDICATE_EVALUATED.fetch_add(visited as u64, Ordering::Relaxed);
+}
+
+/// A query for which no condition had an applicable index, so it falls back.
+///
+/// This deliberately does not advance `queries_total`: the scan the caller is
+/// about to run records the query itself, and counting it here would report
+/// every fallback twice.
+pub(crate) fn record_index_miss() {
+    INDEX_MISSES.fetch_add(1, Ordering::Relaxed);
+}
+
+/// A query answered from index candidates.
+///
+/// `examined` is the candidate set after intersection, `evaluated` the records
+/// the predicate actually ran against, and `materialized` the matching set that
+/// was cloned. `evaluated` can be below `examined` only if an index entry has no
+/// record behind it, which would be a maintenance defect; the gap is visible
+/// here rather than hidden.
+pub(crate) fn record_indexed_query(examined: usize, evaluated: usize, materialized: usize) {
+    QUERIES_TOTAL.fetch_add(1, Ordering::Relaxed);
+    QUERIES_INDEXED.fetch_add(1, Ordering::Relaxed);
+    INDEX_HITS.fetch_add(1, Ordering::Relaxed);
+    INDEX_CANDIDATES_EXAMINED.fetch_add(examined as u64, Ordering::Relaxed);
+    RECORDS_PREDICATE_EVALUATED.fetch_add(evaluated as u64, Ordering::Relaxed);
+    SCAN_RECORDS_MATERIALIZED.fetch_add(materialized as u64, Ordering::Relaxed);
+}
+
+/// Read the counters. Monotonic since process start, so tests compare deltas.
+pub fn counters() -> QueryExecutionCounters {
+    QueryExecutionCounters {
+        full_collection_materializations: FULL_COLLECTION_MATERIALIZATIONS.load(Ordering::Relaxed),
+        full_collection_records_cloned: FULL_COLLECTION_RECORDS_CLONED.load(Ordering::Relaxed),
+        collection_scans: COLLECTION_SCANS.load(Ordering::Relaxed),
+        scan_records_visited: SCAN_RECORDS_VISITED.load(Ordering::Relaxed),
+        scan_records_materialized: SCAN_RECORDS_MATERIALIZED.load(Ordering::Relaxed),
+        queries_total: QUERIES_TOTAL.load(Ordering::Relaxed),
+        queries_scan: QUERIES_SCAN.load(Ordering::Relaxed),
+        queries_indexed: QUERIES_INDEXED.load(Ordering::Relaxed),
+        index_hits: INDEX_HITS.load(Ordering::Relaxed),
+        index_misses: INDEX_MISSES.load(Ordering::Relaxed),
+        index_candidates_examined: INDEX_CANDIDATES_EXAMINED.load(Ordering::Relaxed),
+        records_predicate_evaluated: RECORDS_PREDICATE_EVALUATED.load(Ordering::Relaxed),
+    }
+}

package/dist/create/server-source/crates/feltdb/src/state_contract.rs

@@ -25,7 +25,8 @@ use crate::{
     policy_evaluation::{
         Actor, AuthorizationState, PolicyEvaluator, PolicySubject, RecordAuthorizationContext,
     },
-    AtomicMutation, AtomicPrecondition, FeltDb, StoredRow,
+    AtomicMutation, AtomicPrecondition, FeltDb, FlowError, PreconditionFailure, RecordPrecondition,
+    StoredRow,
 };
 use serde::{Deserialize, Serialize};
 use serde_json::{Map, Value};
@@ -853,6 +854,12 @@ pub struct StateFailure {
     pub transaction: Option<String>,
     #[serde(default)]
     pub causal_position: Option<u64>,
+    /// The structured predicate failure, when the refusal was a record
+    /// precondition. Named `failure` because that is what the direct
+    /// authority's refused transaction already calls it: one refusal, one
+    /// shape, whichever endpoint answered.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub failure: Option<Value>,
 }
 impl StateFailure {
     fn new(code: &str, message: &str) -> Self {
@@ -864,6 +871,7 @@ impl StateFailure {
             resource: None,
             transaction: None,
             causal_position: None,
+            failure: None,
         }
     }
     fn storage(error: impl std::fmt::Display) -> Self {
@@ -1615,6 +1623,34 @@ pub enum TransactionPrecondition {
     StateVersion { version: u64 },
     ApplicationRevision { revision_id: String },
     CausalCursor { cursor: BTreeMap<String, u64> },
+    /// What must be true of one record for the transaction to commit.
+    ///
+    /// Distinct from every variant above, and evaluated in a different place
+    /// for a reason: those compare counters that can be read before the commit
+    /// lock is taken, while this one compares the fields a caller reads back —
+    /// the document's `__version`, its authority epoch, its lease — and must
+    /// therefore be evaluated *inside* the same lock as the writes. Checking it
+    /// against the pre-commit snapshot would be a client-side check wearing the
+    /// authority's name: between the read and the write, another writer
+    /// commits.
+    ///
+    /// It is also what makes `putIfAbsent` and `updateIfVersion` mean the same
+    /// thing against a managed authority as against the file runtime. Mapping
+    /// a caller's `expectedVersion` onto `if_version` would not: that fences
+    /// the row's internal storage sequence, which is a different number that
+    /// happens to share a name.
+    Record {
+        collection: String,
+        id: String,
+        #[serde(default)]
+        require_absent: bool,
+        #[serde(default)]
+        expected_version: Option<u64>,
+        #[serde(default)]
+        expected_epoch: Option<u64>,
+        #[serde(default)]
+        expected_lease_id: Option<String>,
+    },
 }
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct TransactionRequest {
@@ -2056,8 +2092,47 @@ fn execute_transaction_with_policies(
         });
     }
     let snapshot = db.state_rows().map_err(StateFailure::storage)?;
+    let namespace = request
+        .state_namespace
+        .as_deref()
+        .unwrap_or(&request.application_id)
+        .to_string();
+    // Record preconditions are collected here and evaluated by the store,
+    // inside the same lock as the writes. Evaluating them against `snapshot`
+    // alongside the counters below would reintroduce exactly the gap they
+    // exist to close.
+    let mut record_preconditions: Vec<RecordPrecondition> = Vec::new();
     for condition in &request.preconditions {
         let (resource, expected, actual, passed) = match condition {
+            TransactionPrecondition::Record {
+                collection,
+                id,
+                require_absent,
+                expected_version,
+                expected_epoch,
+                expected_lease_id,
+            } => {
+                let precondition = RecordPrecondition {
+                    capability: format!("{namespace}:{collection}"),
+                    key: id.clone(),
+                    require_absent: *require_absent,
+                    expected_version: *expected_version,
+                    expected_epoch: *expected_epoch,
+                    expected_lease_id: expected_lease_id.clone(),
+                };
+                if precondition.is_empty() {
+                    // Refused rather than ignored: a precondition that
+                    // constrains nothing reads as protection and provides none.
+                    let mut failure = StateFailure::new(
+                        "VALIDATION_FAILED",
+                        &format!("precondition on {collection}:{id} constrains nothing"),
+                    );
+                    failure.resource = Some(format!("{collection}:{id}"));
+                    return Err(failure);
+                }
+                record_preconditions.push(precondition);
+                continue;
+            }
             TransactionPrecondition::CollectionVersion {
                 collection,
                 version,
@@ -2350,15 +2425,37 @@ fn execute_transaction_with_policies(
         }
     }
     let commit = db
-        .apply_atomic_transaction_content_addressed(
+        .apply_atomic_transaction_guarded(
             &storage_transaction_id,
             Some(&payload_hash),
             request.causal_parent,
             &preconditions,
+            &record_preconditions,
             &mutations,
             Some(serde_json::json!({"transaction_id":transaction_id.clone(),"subject":request.authorization.subject.clone(),"tenant":request.tenant_id.clone(),"application":request.application_id.clone(),"revision":request.revision_id.clone(),"schema_version":request.schema_version,"operations":request.operations.len(),"authorization_decisions":["state:write allowed"],"causal_parent":request.causal_parent,"result":"committed"})),
         )
         .map_err(|error| {
+            // A record precondition that did not hold is a lost race, and it
+            // says which predicate failed and what the authority holds. That
+            // detail is the difference between "re-read and decide again" and
+            // "this deployment is broken", so it is carried through as
+            // structure rather than flattened into the message.
+            if let FlowError::PreconditionFailed(refusal) = &error {
+                let mut failure =
+                    StateFailure::new("PRECONDITION_FAILED", &error.to_string());
+                failure.transaction = Some(transaction_id.clone());
+                failure.causal_position = db.sequence().ok();
+                failure.resource = Some(refusal.key().to_string());
+                failure.failure = serde_json::to_value(refusal.as_ref()).ok();
+                if let PreconditionFailure::Version {
+                    expected, actual, ..
+                } = refusal.as_ref()
+                {
+                    failure.expected = Some(Value::from(*expected));
+                    failure.actual = Some(Value::from(*actual));
+                }
+                return failure;
+            }
             let text = error.to_string();
             if text.contains("PRECONDITION_FAILED") {
                 let mut failure = StateFailure::new("PRECONDITION_FAILED", &text);
@@ -2665,6 +2762,199 @@ mod tests {
             "CONFLICT"
         );
     }
+    /// A conditional write through the canonical contract has to mean the same
+    /// thing it means everywhere else in FeltDB, or `putIfAbsent` and
+    /// `updateIfVersion` become file-runtime privileges.
+    ///
+    /// The predicate under test is the document's `__version` -- the number a
+    /// caller reads back -- not the row's internal storage sequence. Those are
+    /// two different numbers that happen to share a name, and a fence on the
+    /// wrong one is worse than no fence, because it reads as protection.
+    #[test]
+    fn a_record_precondition_fences_the_document_version() {
+        let db = db("record-precondition");
+        let schema = schema();
+        let write = |id: &str,
+                     transaction: &str,
+                     title: &str,
+                     preconditions: Vec<TransactionPrecondition>,
+                     kind: TransactionOperationKind| {
+            execute_transaction(
+                &db,
+                &schema,
+                &TransactionRequest {
+                    transaction_id: Some(transaction.into()),
+                    tenant_id: "tenant".into(),
+                    application_id: "app".into(),
+                    revision_id: "rev".into(),
+                    schema_version: 1,
+                    state_namespace: None,
+                    causal_parent: None,
+                    authorization: auth(),
+                    preconditions,
+                    operations: vec![TransactionOperation {
+                        kind,
+                        collection: "incidents".into(),
+                        id: id.into(),
+                        value: serde_json::json!({ "title": title }),
+                        if_version: None,
+                    }],
+                },
+                None,
+            )
+        };
+        let fence = |version: Option<u64>, absent: bool| {
+            vec![TransactionPrecondition::Record {
+                collection: "incidents".into(),
+                id: "i1".into(),
+                require_absent: absent,
+                expected_version: version,
+                expected_epoch: None,
+                expected_lease_id: None,
+            }]
+        };
+        let version = || {
+            db.state_rows()
+                .unwrap()
+                .into_iter()
+                .find(|row| row.capability == "app:incidents" && row.key == "i1" && !row.deleted)
+                .and_then(|row| row.value.get("__version").and_then(Value::as_u64))
+        };
+
+        // A conditional create is a create at version 1.
+        write(
+            "i1",
+            "create",
+            "First",
+            fence(None, true),
+            TransactionOperationKind::Insert,
+        )
+        .expect("the record is absent, so the create commits");
+        assert_eq!(version(), Some(1));
+
+        // The same create again is a lost race. Whichever check catches it,
+        // the answer is a conflict and nothing was written.
+        let refused = write(
+            "i1",
+            "create-again",
+            "Second",
+            fence(None, true),
+            TransactionOperationKind::Insert,
+        )
+        .expect_err("the record now exists");
+        assert!(
+            refused.code == "CONFLICT" || refused.code == "PRECONDITION_FAILED",
+            "a lost creation race is a conflict, got {}",
+            refused.code,
+        );
+        assert_eq!(version(), Some(1), "a refused transaction writes nothing");
+
+        // A guard on a record the transaction does not write is evaluated
+        // inside the commit lock, and says which predicate failed.
+        let guarded = write(
+            "i2",
+            "guarded",
+            "Other",
+            fence(None, true),
+            TransactionOperationKind::Insert,
+        )
+        .expect_err("i1 exists, so the guard fails and i2 is never written");
+        assert_eq!(guarded.code, "PRECONDITION_FAILED");
+        assert_eq!(
+            guarded
+                .failure
+                .as_ref()
+                .and_then(|value| value.get("predicate").and_then(Value::as_str)),
+            Some("present"),
+        );
+        assert!(
+            db.state_rows()
+                .unwrap()
+                .iter()
+                .all(|row| row.key != "i2" || row.deleted),
+            "a refused guard leaves the staged write unwritten",
+        );
+
+        // A stale fence conflicts and names what the authority actually holds.
+        let stale = write(
+            "i1",
+            "stale",
+            "Third",
+            fence(Some(7), false),
+            TransactionOperationKind::Update,
+        )
+        .expect_err("version 7 is not the version on record");
+        assert_eq!(stale.code, "PRECONDITION_FAILED");
+        assert_eq!(stale.expected, Some(Value::from(7)));
+        assert_eq!(stale.actual, Some(Value::from(1)));
+        assert_eq!(version(), Some(1));
+
+        // The fenced write commits and the authority owns the next version.
+        write(
+            "i1",
+            "fenced",
+            "Fourth",
+            fence(Some(1), false),
+            TransactionOperationKind::Update,
+        )
+        .expect("version 1 is the version on record");
+        assert_eq!(version(), Some(2));
+
+        // Replaying a committed transaction id applies nothing, so it cannot
+        // advance the version a second time.
+        write(
+            "i1",
+            "fenced",
+            "Fourth",
+            fence(Some(1), false),
+            TransactionOperationKind::Update,
+        )
+        .expect("a duplicate transaction id is not a failure");
+        assert_eq!(version(), Some(2));
+    }
+
+    /// A precondition that constrains nothing is refused rather than accepted,
+    /// because one that is silently ignored reads as protection and provides
+    /// none.
+    #[test]
+    fn an_empty_record_precondition_is_refused() {
+        let db = db("empty-precondition");
+        let schema = schema();
+        let failure = execute_transaction(
+            &db,
+            &schema,
+            &TransactionRequest {
+                transaction_id: Some("empty".into()),
+                tenant_id: "tenant".into(),
+                application_id: "app".into(),
+                revision_id: "rev".into(),
+                schema_version: 1,
+                state_namespace: None,
+                causal_parent: None,
+                authorization: auth(),
+                preconditions: vec![TransactionPrecondition::Record {
+                    collection: "incidents".into(),
+                    id: "i1".into(),
+                    require_absent: false,
+                    expected_version: None,
+                    expected_epoch: None,
+                    expected_lease_id: None,
+                }],
+                operations: vec![TransactionOperation {
+                    kind: TransactionOperationKind::Insert,
+                    collection: "incidents".into(),
+                    id: "i1".into(),
+                    value: serde_json::json!({ "title": "First" }),
+                    if_version: None,
+                }],
+            },
+            None,
+        )
+        .expect_err("a precondition that constrains nothing is a caller error");
+        assert_eq!(failure.code, "VALIDATION_FAILED");
+        assert!(db.state_rows().unwrap().iter().all(|row| row.deleted));
+    }
+
     #[test]
     fn validation_failure_rolls_back_every_operation() {
         let db = db("rollback");

package/dist/create/server-source/crates/feltdb/src/sync.rs

@@ -19,6 +19,18 @@ pub struct ChangeLog {
 }
 
 impl ChangeLog {
+    /// Earliest retained sequence for each operation origin.
+    pub fn retained_from(&self) -> HashMap<String, u64> {
+        let mut floors: HashMap<String, u64> = HashMap::new();
+        for operation in &self.pending {
+            floors
+                .entry(operation.instance_id.clone())
+                .and_modify(|sequence| *sequence = (*sequence).min(operation.sequence))
+                .or_insert(operation.sequence);
+        }
+        floors
+    }
+
     /// Create a new empty change log
     pub fn new() -> Self {
         Self::default()