@feltdb/core 0.11.9 → 0.12.1

package/dist/create/server-source/crates/feltdb/tests/workload_envelope_contract.rs

@@ -51,6 +51,11 @@ struct Workload {
 struct Measurement {
     elapsed: Duration,
     log_bytes: u64,
+    /// `log_bytes` without this database's instance id. Every record carries
+    /// that id, and it is derived from the log's path and printed in unpadded
+    /// hex, so two logs at different paths can differ in size by a few bytes
+    /// per record while holding identical databases.
+    identity_free_log_bytes: u64,
     revisions: usize,
 }
 
@@ -93,9 +98,14 @@ fn run(directory: &Path, workload: Workload, mode: DurabilityMode) -> Measuremen
         .map(|resource| store.history_of(&format!("bench:{resource}")).len())
         .sum();
 
+    let log = std::fs::read_to_string(&path).unwrap();
+    let instance_id = db.instance_id().unwrap();
+    let identity_bytes = log.matches(instance_id.as_str()).count() * instance_id.len();
+
     Measurement {
         elapsed,
-        log_bytes: std::fs::metadata(&path).unwrap().len(),
+        log_bytes: log.len() as u64,
+        identity_free_log_bytes: (log.len() - identity_bytes) as u64,
         revisions,
     }
 }
@@ -217,7 +227,10 @@ fn a_barrier_per_write_costs_more_than_none() {
 
     // Durability is the only thing that changed. The database is identical.
     assert_eq!(flushed.revisions, synced.revisions);
-    assert_eq!(flushed.log_bytes, synced.log_bytes);
+    assert_eq!(
+        flushed.identity_free_log_bytes,
+        synced.identity_free_log_bytes
+    );
 }
 
 /// Changing the mode changes the contract and nothing else: the same writes

package/dist/create/server-source/crates/feltdb-server/Cargo.toml

@@ -23,3 +23,7 @@ tower-http = { version = "0.6", features = ["cors", "limit", "trace"] }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 uuid = { version = "1.6", features = ["v4"] }
+
+[[bench]]
+name = "query_shape_observer"
+harness = false

package/dist/create/server-source/crates/feltdb-server/benches/query_shape_observer.rs

@@ -0,0 +1,136 @@
+//! Direct per-operation cost of the query-shape observer.
+//!
+//! This isolates what the observer adds to one `POST /query`: building the
+//! structural input, normalizing, and recording one outcome. It is deliberately
+//! not an end-to-end number — an HTTP round trip costs orders of magnitude more
+//! and would hide it — and it says nothing about contention with the database's
+//! state lock, which the observer never takes. The end-to-end comparison lives
+//! in `tools/benchmarks/query-shape-observer.mjs`.
+//!
+//!   cargo bench --locked -p feltdb-server --bench query_shape_observer
+//!
+//! Output is one JSON object. Each figure is the median of `ROUNDS` timed
+//! rounds, in nanoseconds per operation, on this machine and build profile
+//! only.
+
+use feltdb_server::query_shape::{Access, ObserverConfig, QueryShapeObserver, ShapeInput};
+use std::hint::black_box;
+use std::sync::Arc;
+use std::time::Instant;
+
+const ROUNDS: usize = 7;
+const OPERATIONS: usize = 200_000;
+
+fn median(mut values: Vec<f64>) -> f64 {
+    values.sort_by(|a, b| a.partial_cmp(b).unwrap());
+    values[values.len() / 2]
+}
+
+/// One simulated query: the same `Vec` construction the handler performs.
+fn observe(observer: &QueryShapeObserver, collection: &str, field: &str) {
+    let predicates = vec![(field, "eq"), ("kind", "eq")];
+    let order_by = vec![("n", "asc")];
+    let observation = observer.begin(&ShapeInput {
+        collection,
+        predicates: &predicates,
+        order_by: &order_by,
+        limit: 50,
+        has_semantic_clause: false,
+    });
+    observation.finish_executed(Access::Indexed, black_box(40), black_box(40));
+}
+
+fn per_op(mut run: impl FnMut(usize)) -> f64 {
+    median(
+        (0..ROUNDS)
+            .map(|_| {
+                let started = Instant::now();
+                for index in 0..OPERATIONS {
+                    run(index);
+                }
+                started.elapsed().as_nanos() as f64 / OPERATIONS as f64
+            })
+            .collect(),
+    )
+}
+
+fn main() {
+    let observer = QueryShapeObserver::new(ObserverConfig::with_capacity(128));
+
+    // The cost of building the input with no observer at all: the floor.
+    let floor = per_op(|_| {
+        let predicates = vec![("tenant", "eq"), ("kind", "eq")];
+        let order_by = vec![("n", "asc")];
+        black_box((&predicates, &order_by));
+    });
+
+    let one_shape = per_op(|_| observe(&observer, "orders", "tenant"));
+
+    let fixed: Vec<String> = (0..64).map(|i| format!("c{i}")).collect();
+    let sixty_four_shapes = per_op(|index| observe(&observer, &fixed[index % 64], "tenant"));
+
+    // Every operation admits a new shape into a full table: the eviction scan.
+    let churn_names: Vec<String> = (0..4096).map(|i| format!("churn{i}")).collect();
+    let churn = QueryShapeObserver::new(ObserverConfig::with_capacity(128));
+    let eviction_churn = per_op(|index| observe(&churn, &churn_names[index % 4096], "tenant"));
+    let churn_max = QueryShapeObserver::new(ObserverConfig::with_capacity(1024));
+    let eviction_churn_max_capacity =
+        per_op(|index| observe(&churn_max, &churn_names[index % 4096], "tenant"));
+
+    let reserved = per_op(|_| {
+        let order_by = vec![("n", "asc")];
+        black_box(observer.begin(&ShapeInput {
+            collection: "_feltdb.Proposal",
+            predicates: &[],
+            order_by: &order_by,
+            limit: 50,
+            has_semantic_clause: false,
+        }));
+    });
+
+    // Contention: four threads recording into one observer.
+    let shared = Arc::new(QueryShapeObserver::new(ObserverConfig::with_capacity(128)));
+    let contended = median(
+        (0..ROUNDS)
+            .map(|_| {
+                let started = Instant::now();
+                let handles: Vec<_> = (0..4)
+                    .map(|thread| {
+                        let shared = shared.clone();
+                        std::thread::spawn(move || {
+                            let name = format!("t{thread}");
+                            for _ in 0..OPERATIONS / 4 {
+                                observe(&shared, &name, "tenant");
+                            }
+                        })
+                    })
+                    .collect();
+                for handle in handles {
+                    handle.join().unwrap();
+                }
+                started.elapsed().as_nanos() as f64 / OPERATIONS as f64
+            })
+            .collect(),
+    );
+
+    let snapshot = churn_max.snapshot();
+    println!(
+        "{}",
+        serde_json::json!({
+            "benchmark": "query_shape_observer",
+            "profile": if cfg!(debug_assertions) { "debug" } else { "release" },
+            "operations_per_round": OPERATIONS,
+            "rounds": ROUNDS,
+            "unit": "ns/op (median of rounds)",
+            "input_construction_floor_no_observer": floor,
+            "begin_finish_one_shape": one_shape,
+            "begin_finish_64_shapes": sixty_four_shapes,
+            "begin_finish_every_op_evicts_capacity_128": eviction_churn,
+            "begin_finish_every_op_evicts_capacity_1024": eviction_churn_max_capacity,
+            "reserved_collection_skipped": reserved,
+            "four_threads_one_observer_wall_per_op": contended,
+            "capacity_1024_memory_bound_bytes": snapshot.memory_bound_bytes,
+            "capacity_1024_memory_estimate_bytes_at_full": snapshot.memory_estimate_bytes,
+        })
+    );
+}

package/dist/create/server-source/crates/feltdb-server/src/app_state.rs

@@ -86,6 +86,9 @@ pub struct AppState {
     pub state_triggers: Arc<std::sync::Mutex<StateTriggerStore>>,
     pub mesh: Arc<std::sync::Mutex<WorkerMeshStore>>,
     pub readiness_probe: Arc<PathBuf>,
+    /// Opt-in, value-free query-shape observer for `POST /query`, or `None`
+    /// (the default). Diagnostic only: nothing reads it to answer a request.
+    pub query_shapes: Option<Arc<crate::query_shape::QueryShapeObserver>>,
     pub bounded_query_cursors: Arc<std::sync::Mutex<HashMap<String, BoundedQueryCursor>>>,
     /// The deployment's semantic runtime binding for bounded semantic query
     /// composition, or `None` when the deployment configures no runtime. A

package/dist/create/server-source/crates/feltdb-server/src/lib.rs

@@ -27,6 +27,7 @@ pub mod metrics;
 pub mod portable_bundle;
 pub mod principals;
 pub mod providers;
+pub mod query_shape;
 pub mod releases;
 pub mod request_telemetry;
 pub mod sessions;

package/dist/create/server-source/crates/feltdb-server/src/main.rs

@@ -96,6 +96,7 @@ use feltdb_server::{
     portable_bundle::{BundleArtifact, BundleStore, ExportOptions, ImportMode, PortableBundle},
     principals::{PrincipalKind, PrincipalStatus},
     providers::{ProviderManifest, ProviderStatus, ProviderStore},
+    query_shape::{Access, ObserverConfig, QueryShapeObserver, ShapeInput},
     releases::{
         provider_for, AuthorizedDeploymentContext, DeploymentPlan, DeploymentProfile,
         DeploymentProvider, DeploymentProviderKind, DomainBinding, ReleaseManifest, ReleaseStore,
@@ -8766,6 +8767,16 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     // Constructed before the state so the same handle can be stopped after
     // serving: the pipeline outlives the router by exactly one shutdown.
     let audit_handle = AuditLog::new(config.audit.clone());
+    // Opt-in and off by default. A malformed setting is a startup error rather
+    // than a silent default, so a mistyped exclusion list cannot yield an
+    // observer that records what the operator meant to exclude.
+    let query_shapes = ObserverConfig::from_environment()?.map(|config| {
+        println!(
+            "feltdb: query-shape observer enabled (capacity {}, value-free, in-memory)",
+            config.capacity
+        );
+        Arc::new(QueryShapeObserver::new(config))
+    });
     let state = AppState {
         started_at: Instant::now(),
         admission: Arc::new(tokio::sync::Semaphore::new(config.max_in_flight)),
@@ -8829,6 +8840,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         state_triggers: Arc::new(std::sync::Mutex::new(state_trigger_store)),
         mesh: Arc::new(std::sync::Mutex::new(mesh_store)),
         readiness_probe: Arc::new(readiness_probe),
+        query_shapes,
         bounded_query_cursors: Arc::new(std::sync::Mutex::new(HashMap::new())),
         semantic_runtime: configured_semantic_runtime()?,
     };
@@ -13520,7 +13532,7 @@ fn bounded_query_records(
     state: &AppState,
     request: &BoundedQueryRequest,
     execution: BoundedQueryExecution,
-) -> Result<Vec<feltdb::StoredRow>, ApiError> {
+) -> Result<(Vec<feltdb::StoredRow>, Access), ApiError> {
     if execution != BoundedQueryExecution::Scan {
         let equalities = bounded_query_equalities(&request.conditions);
         let indexed =
@@ -13530,7 +13542,7 @@ fn bounded_query_records(
                     bounded_query_matches(row, &request.conditions)
                 })?;
         if let Some(records) = indexed {
-            return Ok(records);
+            return Ok((records, Access::Indexed));
         }
         if execution == BoundedQueryExecution::Index {
             // Only reachable through the test-only header. A forced-index run
@@ -13541,11 +13553,12 @@ fn bounded_query_records(
             ));
         }
     }
-    Ok(state
+    let scanned = state
         .db
         .query_collection(&request.collection, None, |row| {
             bounded_query_matches(row, &request.conditions)
-        })?)
+        })?;
+    Ok((scanned, Access::Scan))
 }
 
 fn issue_bounded_cursor(
@@ -13628,6 +13641,37 @@ async fn execute_bounded_query(
         feltdb::validate_semantic_clause(clause).map_err(semantic_query_error)?;
     }
     let hash = bounded_query_hash(&request)?;
+    // Diagnostic only, and only when opted in. The observation sees structure
+    // (names, operators, directions), never a value, and any failure inside it
+    // is its own: nothing below reads it to decide a result. Dropped without
+    // being finished — an early `?` return, a cancelled request — it records an
+    // error outcome for the shape.
+    let observation = state.query_shapes.as_deref().map(|observer| {
+        let predicates: Vec<(&str, &str)> = request
+            .conditions
+            .iter()
+            .filter_map(|condition| {
+                condition
+                    .operators
+                    .keys()
+                    .next()
+                    .map(|operator| (condition.field.as_str(), operator.as_str()))
+            })
+            .collect();
+        let order_by: Vec<(&str, &str)> = request
+            .order_by
+            .iter()
+            .map(|order| (order.field.as_str(), order.direction.as_str()))
+            .collect();
+        observer.begin(&ShapeInput {
+            collection: &request.collection,
+            predicates: &predicates,
+            order_by: &order_by,
+            limit: request.limit,
+            has_semantic_clause: request.semantic.is_some(),
+        })
+    });
+    let mut executed: Option<(Access, usize)> = None;
     let (records, position) = if let Some(token) = &request.cursor {
         let cursor = state
             .bounded_query_cursors
@@ -13665,7 +13709,8 @@ async fn execute_bounded_query(
         // this surface's contract, so the matching set — not the collection, and
         // not an index bucket — is what gets sorted, and the limit is applied to
         // that ordered set. Nothing below this line knows which execution ran.
-        let matches = bounded_query_records(&state, &request, execution)?;
+        let (matches, access) = bounded_query_records(&state, &request, execution)?;
+        executed = Some((access, matches.len()));
         // Result materialization happens outside the state lock, so it is a cost
         // this request pays alone rather than one it imposes on every other
         // operation. PR36 measures it separately for exactly that reason.
@@ -13724,6 +13769,12 @@ async fn execute_bounded_query(
     } else {
         None
     };
+    if let Some(observation) = observation {
+        match executed {
+            Some((access, matched)) => observation.finish_executed(access, matched, page.len()),
+            None => observation.finish_continuation(page.len()),
+        }
+    }
     // The semantic stage transforms exactly this page. It runs after the
     // deterministic cursor is issued because continuation is by deterministic
     // position: the page may return fewer records than `limit` and still not
@@ -13814,6 +13865,15 @@ fn query_execution_diagnostics_router() -> Router<AppState> {
             get(query_execution_diagnostics),
         );
     }
+    // Query-shape observations name collections and fields, so they are a
+    // separate switch from the counters and exist only when the observer was
+    // opted into. Counts and structure only; never a value.
+    if ObserverConfig::from_environment().ok().flatten().is_some() {
+        router = router.route(
+            "/internal/query-shape-observations",
+            get(query_shape_observations),
+        );
+    }
     // Phase attribution is a separate switch from the execution counters,
     // because it is a separate cost: the counters are a handful of atomic adds
     // per query, while attribution reads the clock at every instrumented
@@ -13833,6 +13893,15 @@ fn query_execution_diagnostics_router() -> Router<AppState> {
     router
 }
 
+/// The observer's retained shapes and counters. Diagnostic evidence about
+/// workload structure, not proof that any change would help.
+async fn query_shape_observations(State(state): State<AppState>) -> Json<Value> {
+    match state.query_shapes.as_deref() {
+        Some(observer) => Json(json!({ "enabled": true, "observations": observer.snapshot() })),
+        None => Json(json!({ "enabled": false })),
+    }
+}
+
 /// Phase attribution: where elapsed time went, as counts and nanoseconds.
 ///
 /// Test instrumentation, mounted only under `FELTDB_WORKLOAD_DIAGNOSTICS=1`.