@feltdb/core 0.15.0 → 0.16.0

package/dist/create/server-source/crates/feltdb/src/query_execution_diagnostics.rs

@@ -1,126 +0,0 @@
-//! Test instrumentation for the collection read paths a bounded query can take.
-//!
-//! PR33 attributed sustained-workload degradation to bounded queries cloning an
-//! entire collection under the state mutex before predicate, ordering and limit
-//! were applied. PR34 removes that clone, and a benchmark alone cannot prove a
-//! clone is gone: a faster number is evidence of speed, not of mechanism.
-//!
-//! These counters therefore record *which traversal a read used*, so a test can
-//! assert that a bounded query never entered `FeltDb::list_collection`. They
-//! are process-global, monotonic, carry no record data, and are not part of the
-//! query contract: nothing in the query path reads them, and no query result
-//! depends on them. The HTTP surface only exposes them when a server is started
-//! with `FELTDB_QUERY_DIAGNOSTICS=1`.
-//!
-//! PR35 extends them with execution attribution. An index is an optimization, so
-//! "the query got faster" is not evidence that it was used; these counters say
-//! which execution a query actually took, how many candidates the index
-//! produced, and how many records the predicate had to evaluate. That last
-//! figure is the one the PR35 claim rests on: for a selective indexed query it
-//! must track the candidate count rather than the collection size.
-
-use std::sync::atomic::{AtomicU64, Ordering};
-
-static FULL_COLLECTION_MATERIALIZATIONS: AtomicU64 = AtomicU64::new(0);
-static FULL_COLLECTION_RECORDS_CLONED: AtomicU64 = AtomicU64::new(0);
-static COLLECTION_SCANS: AtomicU64 = AtomicU64::new(0);
-static SCAN_RECORDS_VISITED: AtomicU64 = AtomicU64::new(0);
-static SCAN_RECORDS_MATERIALIZED: AtomicU64 = AtomicU64::new(0);
-static QUERIES_TOTAL: AtomicU64 = AtomicU64::new(0);
-static QUERIES_SCAN: AtomicU64 = AtomicU64::new(0);
-static QUERIES_INDEXED: AtomicU64 = AtomicU64::new(0);
-static INDEX_HITS: AtomicU64 = AtomicU64::new(0);
-static INDEX_MISSES: AtomicU64 = AtomicU64::new(0);
-static INDEX_CANDIDATES_EXAMINED: AtomicU64 = AtomicU64::new(0);
-static RECORDS_PREDICATE_EVALUATED: AtomicU64 = AtomicU64::new(0);
-
-/// A point-in-time reading of the collection read counters.
-#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
-pub struct QueryExecutionCounters {
-    /// Calls to `FeltDb::list_collection`, which clones a whole collection.
-    pub full_collection_materializations: u64,
-    /// Records cloned by those calls.
-    pub full_collection_records_cloned: u64,
-    /// Calls to `FeltDb::query_collection`, which evaluates borrowed records.
-    pub collection_scans: u64,
-    /// Records a scan predicate observed, borrowed rather than cloned.
-    pub scan_records_visited: u64,
-    /// Records a scan actually cloned, which is at most the matching set.
-    pub scan_records_materialized: u64,
-    /// Collection reads that executed a query: `queries_scan + queries_indexed`.
-    pub queries_total: u64,
-    /// Queries answered by evaluating the predicate over the whole collection.
-    pub queries_scan: u64,
-    /// Queries answered from equality-index candidates.
-    pub queries_indexed: u64,
-    /// Queries for which at least one condition had an applicable index.
-    pub index_hits: u64,
-    /// Queries for which no condition had an applicable index, which fall back.
-    pub index_misses: u64,
-    /// Candidate record keys the index produced, after intersection.
-    pub index_candidates_examined: u64,
-    /// Records the predicate was actually evaluated against, on either path.
-    /// For a selective indexed query this tracks the candidate count; for a scan
-    /// it is the collection size.
-    pub records_predicate_evaluated: u64,
-}
-
-pub(crate) fn record_full_materialization(records: usize) {
-    FULL_COLLECTION_MATERIALIZATIONS.fetch_add(1, Ordering::Relaxed);
-    FULL_COLLECTION_RECORDS_CLONED.fetch_add(records as u64, Ordering::Relaxed);
-}
-
-pub(crate) fn record_scan(visited: usize, materialized: usize) {
-    COLLECTION_SCANS.fetch_add(1, Ordering::Relaxed);
-    SCAN_RECORDS_VISITED.fetch_add(visited as u64, Ordering::Relaxed);
-    SCAN_RECORDS_MATERIALIZED.fetch_add(materialized as u64, Ordering::Relaxed);
-    QUERIES_TOTAL.fetch_add(1, Ordering::Relaxed);
-    QUERIES_SCAN.fetch_add(1, Ordering::Relaxed);
-    // A scan evaluates the predicate against every record it visits. This is
-    // the O(N) figure PR35 exists to reduce, recorded on the same counter the
-    // indexed path reports to, so the two are directly comparable.
-    RECORDS_PREDICATE_EVALUATED.fetch_add(visited as u64, Ordering::Relaxed);
-}
-
-/// A query for which no condition had an applicable index, so it falls back.
-///
-/// This deliberately does not advance `queries_total`: the scan the caller is
-/// about to run records the query itself, and counting it here would report
-/// every fallback twice.
-pub(crate) fn record_index_miss() {
-    INDEX_MISSES.fetch_add(1, Ordering::Relaxed);
-}
-
-/// A query answered from index candidates.
-///
-/// `examined` is the candidate set after intersection, `evaluated` the records
-/// the predicate actually ran against, and `materialized` the matching set that
-/// was cloned. `evaluated` can be below `examined` only if an index entry has no
-/// record behind it, which would be a maintenance defect; the gap is visible
-/// here rather than hidden.
-pub(crate) fn record_indexed_query(examined: usize, evaluated: usize, materialized: usize) {
-    QUERIES_TOTAL.fetch_add(1, Ordering::Relaxed);
-    QUERIES_INDEXED.fetch_add(1, Ordering::Relaxed);
-    INDEX_HITS.fetch_add(1, Ordering::Relaxed);
-    INDEX_CANDIDATES_EXAMINED.fetch_add(examined as u64, Ordering::Relaxed);
-    RECORDS_PREDICATE_EVALUATED.fetch_add(evaluated as u64, Ordering::Relaxed);
-    SCAN_RECORDS_MATERIALIZED.fetch_add(materialized as u64, Ordering::Relaxed);
-}
-
-/// Read the counters. Monotonic since process start, so tests compare deltas.
-pub fn counters() -> QueryExecutionCounters {
-    QueryExecutionCounters {
-        full_collection_materializations: FULL_COLLECTION_MATERIALIZATIONS.load(Ordering::Relaxed),
-        full_collection_records_cloned: FULL_COLLECTION_RECORDS_CLONED.load(Ordering::Relaxed),
-        collection_scans: COLLECTION_SCANS.load(Ordering::Relaxed),
-        scan_records_visited: SCAN_RECORDS_VISITED.load(Ordering::Relaxed),
-        scan_records_materialized: SCAN_RECORDS_MATERIALIZED.load(Ordering::Relaxed),
-        queries_total: QUERIES_TOTAL.load(Ordering::Relaxed),
-        queries_scan: QUERIES_SCAN.load(Ordering::Relaxed),
-        queries_indexed: QUERIES_INDEXED.load(Ordering::Relaxed),
-        index_hits: INDEX_HITS.load(Ordering::Relaxed),
-        index_misses: INDEX_MISSES.load(Ordering::Relaxed),
-        index_candidates_examined: INDEX_CANDIDATES_EXAMINED.load(Ordering::Relaxed),
-        records_predicate_evaluated: RECORDS_PREDICATE_EVALUATED.load(Ordering::Relaxed),
-    }
-}

package/dist/create/server-source/crates/feltdb/src/query_performance.rs

@@ -1,218 +0,0 @@
-/// Query Performance Tracking & Latency Contracts
-///
-/// Manages query performance benchmarks and latency SLAs:
-/// - Query latency tracking (p50, p95, p99)
-/// - Latency contract enforcement
-/// - Performance degradation detection
-/// - Query optimization recommendations
-use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
-use std::time::{Duration, Instant};
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct LatencyContract {
-    pub query_type: String,
-    pub p50_target_us: u64,
-    pub p95_target_us: u64,
-    pub p99_target_us: u64,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct QueryMetric {
-    pub query_type: String,
-    pub duration_us: u64,
-    pub timestamp: u64,
-    pub index_name: String,
-    pub result_count: usize,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct QueryLatencyStats {
-    pub query_type: String,
-    pub count: u64,
-    pub p50_us: u64,
-    pub p95_us: u64,
-    pub p99_us: u64,
-    pub min_us: u64,
-    pub max_us: u64,
-    pub avg_us: u64,
-}
-
-#[derive(Debug, Clone)]
-struct QueryTimer {
-    start: Instant,
-}
-
-impl QueryTimer {
-    fn new() -> Self {
-        QueryTimer {
-            start: Instant::now(),
-        }
-    }
-
-    fn elapsed_us(&self) -> u64 {
-        self.start.elapsed().as_micros() as u64
-    }
-}
-
-pub struct QueryPerformanceTracker {
-    contracts: HashMap<String, LatencyContract>,
-    metrics: Vec<QueryMetric>,
-    violations: Vec<String>,
-}
-
-impl QueryPerformanceTracker {
-    pub fn new() -> Self {
-        QueryPerformanceTracker {
-            contracts: HashMap::new(),
-            metrics: Vec::new(),
-            violations: Vec::new(),
-        }
-    }
-
-    pub fn register_contract(&mut self, contract: LatencyContract) {
-        self.contracts.insert(contract.query_type.clone(), contract);
-    }
-
-    pub fn start_timer(&self) -> QueryTimer {
-        QueryTimer::new()
-    }
-
-    pub fn record_query(
-        &mut self,
-        query_type: String,
-        index_name: String,
-        result_count: usize,
-        duration_us: u64,
-    ) {
-        let metric = QueryMetric {
-            query_type: query_type.clone(),
-            duration_us,
-            timestamp: std::time::SystemTime::now()
-                .duration_since(std::time::UNIX_EPOCH)
-                .unwrap_or_default()
-                .as_secs(),
-            index_name,
-            result_count,
-        };
-
-        if let Some(contract) = self.contracts.get(&query_type) {
-            if duration_us > contract.p99_target_us {
-                self.violations.push(format!(
-                    "Query {} violated p99 contract: {} us > {} us",
-                    query_type, duration_us, contract.p99_target_us
-                ));
-            }
-        }
-
-        self.metrics.push(metric);
-    }
-
-    pub fn get_stats(&self, query_type: &str) -> Option<QueryLatencyStats> {
-        let relevant: Vec<&QueryMetric> = self
-            .metrics
-            .iter()
-            .filter(|m| m.query_type == query_type)
-            .collect();
-
-        if relevant.is_empty() {
-            return None;
-        }
-
-        let mut durations: Vec<u64> = relevant.iter().map(|m| m.duration_us).collect();
-        durations.sort();
-
-        let count = durations.len() as u64;
-        let sum: u64 = durations.iter().sum();
-        let avg = sum / count;
-
-        let p50_idx = (count / 2) as usize;
-        let p95_idx = ((count * 95) / 100) as usize;
-        let p99_idx = ((count * 99) / 100) as usize;
-
-        Some(QueryLatencyStats {
-            query_type: query_type.to_string(),
-            count,
-            p50_us: durations.get(p50_idx).copied().unwrap_or(0),
-            p95_us: durations.get(p95_idx).copied().unwrap_or(0),
-            p99_us: durations.get(p99_idx).copied().unwrap_or(0),
-            min_us: *durations.first().unwrap_or(&0),
-            max_us: *durations.last().unwrap_or(&0),
-            avg_us: avg,
-        })
-    }
-
-    pub fn get_violations(&self) -> Vec<String> {
-        self.violations.clone()
-    }
-
-    pub fn clear_violations(&mut self) {
-        self.violations.clear();
-    }
-
-    pub fn export_metrics_csv(&self) -> String {
-        let mut csv = String::from("timestamp,query_type,duration_us,index_name,result_count\n");
-        for metric in &self.metrics {
-            csv.push_str(&format!(
-                "{},{},{},{},{}\n",
-                metric.timestamp,
-                metric.query_type,
-                metric.duration_us,
-                metric.index_name,
-                metric.result_count
-            ));
-        }
-        csv
-    }
-}
-
-impl Default for QueryPerformanceTracker {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_latency_contract() {
-        let mut tracker = QueryPerformanceTracker::new();
-        tracker.register_contract(LatencyContract {
-            query_type: "index_lookup".to_string(),
-            p50_target_us: 100,
-            p95_target_us: 500,
-            p99_target_us: 1000,
-        });
-
-        let timer = tracker.start_timer();
-        std::thread::sleep(Duration::from_micros(50));
-        let elapsed = timer.elapsed_us();
-        // A sleep lasts at least as long as asked, and on a loaded machine
-        // arbitrarily longer, so only the lower bound is a property.
-        assert!(elapsed >= 50, "timer measured {elapsed}us for a 50us sleep");
-
-        tracker.record_query("index_lookup".to_string(), "idx_1".to_string(), 10, elapsed);
-
-        let stats = tracker.get_stats("index_lookup").unwrap();
-        assert_eq!(stats.count, 1);
-        assert_eq!(stats.p50_us, elapsed);
-    }
-
-    #[test]
-    fn test_violation_detection() {
-        let mut tracker = QueryPerformanceTracker::new();
-        tracker.register_contract(LatencyContract {
-            query_type: "slow_query".to_string(),
-            p50_target_us: 100,
-            p95_target_us: 200,
-            p99_target_us: 500,
-        });
-
-        tracker.record_query("slow_query".to_string(), "idx_1".to_string(), 100, 600);
-
-        let violations = tracker.get_violations();
-        assert_eq!(violations.len(), 1);
-    }
-}

package/dist/create/server-source/crates/feltdb/src/read_basis.rs

@@ -1,257 +0,0 @@
-//! Read-basis fences: commit only if the state a decision read is unchanged.
-//!
-//! ```text
-//!     basis = read_basis(records, predicates)       -- one token, under the lock
-//!     ... decide ...
-//!     commit fenced by basis                        -- re-read under the commit lock
-//!         unchanged  ->  commit
-//!         changed    ->  PreconditionFailed(Fence), ZERO writes
-//! ```
-//!
-//! The fence reads stored rows, not bookkeeping a writer maintains, so it holds
-//! whichever write path changed them. Only the named records and the named
-//! collections' matching rows are read: everything else is outside the fence
-//! and never conflicts.
-
-#[cfg(test)]
-mod tests {
-    use crate::{
-        AtomicMutation, FeltDb, FlowError, PreconditionFailure, ReadBasis, ReadCondition,
-        ReadLayout, ReadPredicate, ReadRecord, ReadSet,
-    };
-    use serde_json::json;
-    use tempfile::TempDir;
-
-    fn db(dir: &TempDir) -> FeltDb {
-        FeltDb::open(dir.path().join("app.log")).expect("open")
-    }
-
-    fn set(collection: &str, id: &str, value: serde_json::Value) -> AtomicMutation {
-        AtomicMutation {
-            capability: collection.to_string(),
-            key: format!("{collection}:{id}"),
-            value: Some(value),
-        }
-    }
-
-    fn delete(collection: &str, id: &str) -> AtomicMutation {
-        AtomicMutation {
-            capability: collection.to_string(),
-            key: format!("{collection}:{id}"),
-            value: None,
-        }
-    }
-
-    fn write(db: &FeltDb, id: &str, mutations: &[AtomicMutation]) {
-        db.apply_atomic_transaction(id, None, &[], mutations, None)
-            .expect("write commits");
-    }
-
-    fn reads() -> ReadSet {
-        ReadSet {
-            records: vec![ReadRecord {
-                collection: "A".into(),
-                id: "a1".into(),
-            }],
-            predicates: vec![ReadPredicate {
-                collection: "B".into(),
-                conditions: vec![ReadCondition {
-                    field: "a".into(),
-                    eq: json!("a1"),
-                }],
-            }],
-        }
-    }
-
-    fn seed(db: &FeltDb) {
-        write(
-            db,
-            "seed",
-            &[
-                set("A", "a1", json!({ "id": "a1", "n": 1 })),
-                set("B", "b1", json!({ "id": "b1", "a": "a1" })),
-                set("B", "b2", json!({ "id": "b2", "a": "a2" })),
-            ],
-        );
-    }
-
-    fn fenced(db: &FeltDb, id: &str, basis: &ReadBasis) -> Result<crate::AtomicCommit, FlowError> {
-        db.apply_atomic_transaction_fenced(
-            id,
-            None,
-            None,
-            &[],
-            &[],
-            std::slice::from_ref(basis),
-            &[set("Z", id, json!({ "id": id }))],
-            None,
-        )
-    }
-
-    fn is_fence_conflict(result: Result<crate::AtomicCommit, FlowError>) -> bool {
-        matches!(result, Err(FlowError::PreconditionFailed(failure)) if matches!(*failure, PreconditionFailure::Fence { .. }))
-    }
-
-    #[test]
-    fn an_unchanged_basis_commits_and_a_changed_one_commits_nothing() {
-        let dir = TempDir::new().unwrap();
-        let db = db(&dir);
-        seed(&db);
-        let basis = db.read_basis(&reads()).unwrap();
-        assert!(fenced(&db, "z1", &basis).is_ok());
-        write(&db, "raw", &[set("A", "a1", json!({ "id": "a1", "n": 2 }))]);
-        assert!(is_fence_conflict(fenced(&db, "z2", &basis)));
-        assert!(
-            db.get_value("Z:z2").unwrap().is_none(),
-            "a refused transaction writes nothing"
-        );
-    }
-
-    #[test]
-    fn records_entering_or_leaving_a_predicate_conflict_and_others_do_not() {
-        let dir = TempDir::new().unwrap();
-        let db = db(&dir);
-        seed(&db);
-        let basis = db.read_basis(&reads()).unwrap();
-        write(
-            &db,
-            "outside",
-            &[
-                set("B", "b3", json!({ "id": "b3", "a": "a9" })),
-                set("C", "c1", json!({ "id": "c1" })),
-            ],
-        );
-        assert!(
-            fenced(&db, "z1", &basis).is_ok(),
-            "writes outside the read set never conflict"
-        );
-
-        write(
-            &db,
-            "enter",
-            &[set("B", "b4", json!({ "id": "b4", "a": "a1" }))],
-        );
-        assert!(is_fence_conflict(fenced(&db, "z2", &basis)));
-
-        let basis = db.read_basis(&reads()).unwrap();
-        write(
-            &db,
-            "leave",
-            &[set("B", "b1", json!({ "id": "b1", "a": "a2" }))],
-        );
-        assert!(is_fence_conflict(fenced(&db, "z3", &basis)));
-    }
-
-    #[test]
-    fn delete_and_recreate_with_different_content_conflicts() {
-        let dir = TempDir::new().unwrap();
-        let db = db(&dir);
-        seed(&db);
-        let basis = db.read_basis(&reads()).unwrap();
-        write(&db, "delete", &[delete("A", "a1")]);
-        write(
-            &db,
-            "recreate",
-            &[set("A", "a1", json!({ "id": "a1", "n": 7 }))],
-        );
-        assert!(is_fence_conflict(fenced(&db, "z1", &basis)));
-    }
-
-    #[test]
-    fn tokens_are_deterministic_and_survive_restart() {
-        let dir = TempDir::new().unwrap();
-        let basis = {
-            let db = db(&dir);
-            seed(&db);
-            let first = db.read_basis(&reads()).unwrap();
-            let mut shuffled = reads();
-            shuffled.records.push(ReadRecord {
-                collection: "A".into(),
-                id: "a1".into(),
-            });
-            assert_eq!(
-                first,
-                db.read_basis(&shuffled).unwrap(),
-                "normalised: duplicates do not matter"
-            );
-            first
-        };
-        let db = db(&dir);
-        assert_eq!(db.read_basis(&reads()).unwrap().token, basis.token);
-        assert!(fenced(&db, "z1", &basis).is_ok());
-    }
-
-    #[test]
-    fn invalid_and_foreign_bases_are_refused_as_invalid_not_as_conflicts() {
-        let dir = TempDir::new().unwrap();
-        let db = db(&dir);
-        seed(&db);
-        assert!(db.read_basis(&ReadSet::default()).is_err());
-        let mut foreign = db.read_basis(&reads()).unwrap();
-        foreign.token = format!("fb1:{}", "0".repeat(64));
-        let result = fenced(&db, "z1", &foreign);
-        assert!(
-            matches!(result, Err(FlowError::CapabilityError(ref message)) if message.starts_with("INVALID_READ_SET:"))
-        );
-    }
-
-    #[test]
-    fn namespace_bases_read_namespace_rows_and_are_scoped_to_their_namespace() {
-        let dir = TempDir::new().unwrap();
-        let db = db(&dir);
-        let ns = ReadLayout::Namespace("tenant-a".into());
-        let row = |collection: &str, id: &str, value: serde_json::Value| AtomicMutation {
-            capability: format!("tenant-a:{collection}"),
-            key: id.to_string(),
-            value: Some(value),
-        };
-        write(
-            &db,
-            "seed",
-            &[
-                row("A", "a1", json!({ "id": "a1", "n": 1 })),
-                row("B", "b1", json!({ "id": "b1", "a": "a1" })),
-            ],
-        );
-        let basis = db.read_basis_in(&reads(), &ns).unwrap();
-        assert!(basis.token.starts_with(&ns.token_prefix()));
-        let commit = |id: &str, layout: &ReadLayout, basis: &ReadBasis| {
-            db.apply_atomic_transaction_fenced_in(
-                id,
-                None,
-                None,
-                &[],
-                &[],
-                layout,
-                std::slice::from_ref(basis),
-                &[row("Z", id, json!({ "id": id }))],
-                None,
-            )
-        };
-        let invalid = |result: Result<crate::AtomicCommit, FlowError>| matches!(result, Err(FlowError::CapabilityError(ref message)) if message.starts_with("INVALID_READ_SET:"));
-        // A token from another scope is invalid, never compared.
-        assert!(invalid(commit("z0", &ReadLayout::Direct, &basis)));
-        assert!(invalid(commit(
-            "z0",
-            &ReadLayout::Namespace("tenant-b".into()),
-            &basis
-        )));
-        let direct = db.read_basis(&reads()).unwrap();
-        assert!(invalid(commit("z0", &ns, &direct)));
-        // Direct-layout writes to the same names are outside the namespace.
-        write(
-            &db,
-            "direct",
-            &[set("A", "a1", json!({ "id": "a1", "n": 9 }))],
-        );
-        assert!(commit("z1", &ns, &basis).is_ok());
-        // A namespace write the basis read refuses the commit.
-        write(
-            &db,
-            "raw",
-            &[row("B", "b9", json!({ "id": "b9", "a": "a1" }))],
-        );
-        let result = commit("z2", &ns, &basis);
-        assert!(is_fence_conflict(result));
-    }
-}