@feltdb/core 0.8.3 → 0.8.4

package/dist/create/server-source/crates/feltdb-server/src/main.rs

@@ -7861,6 +7861,54 @@ struct SyncPullRequest {
 struct SyncPullResponse {
     operations: Vec<Operation>,
     versions: HashMap<String, u64>,
+    catchup: CatchupStatus,
+}
+
+#[derive(Serialize, Deserialize)]
+struct CatchupStatus {
+    peer_id: String,
+    peer_position: HashMap<String, u64>,
+    authority_position: HashMap<String, u64>,
+    retained_from: HashMap<String, u64>,
+    retained_through: HashMap<String, u64>,
+    lag: u64,
+    recoverability: String,
+    catchup_mode: String,
+    recovery_required: bool,
+}
+
+fn catchup_status(
+    peer_id: String,
+    peer_position: HashMap<String, u64>,
+    authority_position: HashMap<String, u64>,
+    retained_from: HashMap<String, u64>,
+) -> CatchupStatus {
+    let lag = authority_position.iter().map(|(origin, sequence)| {
+        sequence.saturating_sub(peer_position.get(origin).copied().unwrap_or(0))
+    }).sum();
+    let recovery_required = authority_position.iter().any(|(origin, through)| {
+        let position = peer_position.get(origin).copied().unwrap_or(0);
+        let floor = retained_from.get(origin).copied().unwrap_or(through.saturating_add(1));
+        position < *through && position.saturating_add(1) < floor
+    });
+    let catchup_mode = if recovery_required {
+        "snapshot_rebootstrap_required"
+    } else if lag == 0 {
+        "current"
+    } else {
+        "incremental_replay"
+    };
+    CatchupStatus {
+        peer_id,
+        peer_position,
+        retained_through: authority_position.clone(),
+        authority_position,
+        retained_from,
+        lag,
+        recoverability: if recovery_required { "rebootstrap_required" } else { "recoverable" }.into(),
+        catchup_mode: catchup_mode.into(),
+        recovery_required,
+    }
 }
 
 #[derive(Serialize, Deserialize)]
@@ -8250,6 +8298,18 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let config = Config::from_args().map_err(|error| format!("configuration error: {error}"))?;
     validate_production_configuration(&config)?;
     let db = FeltDb::open(&config.data)?;
+    // Indexes are derived execution structures, so they are declared by
+    // configuration and built from the durable state this open just replayed.
+    // Nothing about them is read from disk: a restart re-derives every bucket
+    // from the records that survived, which is why there is no index format,
+    // no index log, and nothing to repair.
+    let declared_indexes = declare_configured_equality_indexes(&db)?;
+    if !declared_indexes.is_empty() {
+        println!(
+            "feltdb: equality indexes maintained for {}",
+            declared_indexes.join(", ")
+        );
+    }
     let peer_token = match std::env::var(&config.peer_token_env) {
         Ok(token) => Some(Arc::<str>::from(token)),
         Err(_) if config.peers.is_empty() => None,
@@ -8287,6 +8347,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let causal_store = CausalStore::load(config.data.with_extension("causal-events.json"))?;
     let provider_store = ProviderStore::load(config.data.with_extension("providers.json"))?;
     let readiness_probe = config.data.with_extension("readiness");
+    // Constructed before the state so the same handle can be stopped after
+    // serving: the pipeline outlives the router by exactly one shutdown.
+    let audit_handle = AuditLog::new(config.audit.clone());
     let state = AppState {
         started_at: Instant::now(),
         ids: Arc::new(AtomicU64::new(db.sequence()?)),
@@ -8299,7 +8362,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             config.data.with_extension("cluster.json"),
             config.peers.clone(),
         )?,
-        audit: AuditLog::new(config.audit.clone()),
+        audit: audit_handle.clone(),
         peer_client,
         peer_token,
         lifecycle_lock: Arc::new(tokio::sync::Mutex::new(())),
@@ -9016,7 +9079,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             axum::routing::delete(delete_certification_fixture),
         )
         .merge(protected)
+        .merge(query_execution_diagnostics_router())
         .layer(middleware::from_fn(protocol_version))
+        .layer(middleware::from_fn(attribute_request))
         .layer(middleware::from_fn_with_state(state.clone(), count_request))
         .layer(RequestBodyLimitLayer::new(1024 * 1024))
         .layer(TraceLayer::new_for_http())
@@ -9049,9 +9114,17 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         "FeltDB server ready at http://{bound_address} (namespace: {})",
         config.namespace
     );
-    axum::serve(listener, app)
+    let serving = axum::serve(listener, app)
         .with_graceful_shutdown(shutdown_signal())
-        .await?;
+        .await;
+    // The audit writer owns the stream, so an orderly stop is what makes the
+    // last durability group durable. Run it whether or not serving ended
+    // cleanly: a server that failed still recorded events, and losing the tail
+    // of the security log because of an unrelated bind error would be its own
+    // defect. Dropping the last handle would do this too; doing it here means
+    // it happens at a point in the process the operator can reason about.
+    audit_handle.shutdown();
+    serving?;
     Ok(())
 }
 
@@ -9081,6 +9154,12 @@ async fn authenticate(
     mut request: Request<axum::body::Body>,
     next: Next,
 ) -> Response {
+    // Everything up to the handoff below is authorization: key lookup, session
+    // resolution and the capability check. It is timed separately from the rest
+    // of the request because "the request costs something outside state" is not
+    // an answer to PR36's question — which part of the request does is.
+    let authorization =
+        feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::Authorization);
     let principal = if state.auth_enabled {
         let token = request
             .headers()
@@ -9271,6 +9350,7 @@ async fn authenticate(
         .key_id
         .clone();
     let target = request.uri().path().to_string();
+    drop(authorization);
     let response = next.run(request).await;
     let status = response.status().as_u16();
     audit(
@@ -9292,6 +9372,18 @@ fn audit(
     outcome: &str,
     status: u16,
 ) {
+    // Every authenticated request records one of these. Since PR38 the request
+    // waits only for ACCEPTANCE — the audit writer's append — and never for a
+    // durability barrier; the barrier happens afterwards, in a bounded group,
+    // on the writer's own threads.
+    //
+    // The error is still discarded, and that is still deliberate: PR37 proved
+    // request success does not depend on audit durability, and PR38 does not
+    // change that policy. What changed is that the failure is no longer
+    // invisible — `AuditLog::record` has already counted it, timestamped it and
+    // degraded the audit health state by the time this returns.
+    let _span =
+        feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::AuditWrite);
     if let Err(error) = state.audit.record(AuditEvent {
         timestamp_ms: 0,
         namespace: &state.namespace,
@@ -9318,6 +9410,21 @@ async fn protocol_version(request: Request<axum::body::Body>, next: Next) -> Res
     next.run(request).await
 }
 
+/// Time a whole request inside the server, from the middleware stack down.
+///
+/// This is the server's own share of a request. Subtracting it from the client's
+/// measured latency leaves transport, connection handling, and the JSON decode
+/// and encode that axum performs in the extractor and the response — none of
+/// which can be bracketed from inside a handler. PR36 reports that difference as
+/// a bounded residual rather than pretending to have decomposed it.
+///
+/// Disabled, this costs one relaxed atomic load per request.
+async fn attribute_request(request: Request<axum::body::Body>, next: Next) -> Response {
+    let _span =
+        feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::HttpHandler);
+    next.run(request).await
+}
+
 async fn health(State(state): State<AppState>) -> Json<HealthResponse<'static>> {
     let membership = match state.cluster.proposal().map(|value| value.phase) {
         Some(ProposalPhase::Preparing | ProposalPhase::Prepared) => "recovering",
@@ -9749,10 +9856,21 @@ async fn create_online_backup(
     Ok(Json(manifest))
 }
 
-async fn network_metrics(
-    State(state): State<AppState>,
-) -> Json<feltdb_server::metrics::MetricsSnapshot> {
-    Json(state.metrics.snapshot())
+/// Network counters, plus the audit subsystem's health.
+///
+/// The audit block is here rather than on a route of its own because PR37's
+/// finding was that losing security evidence was invisible, and a signal an
+/// operator has to know to go and look for is not much better than no signal.
+/// It rides with the metrics they already read.
+async fn network_metrics(State(state): State<AppState>) -> Json<Value> {
+    let mut body = serde_json::to_value(state.metrics.snapshot()).unwrap_or_else(|_| json!({}));
+    if let Some(fields) = body.as_object_mut() {
+        fields.insert(
+            "audit".to_string(),
+            serde_json::to_value(state.audit.health()).unwrap_or(Value::Null),
+        );
+    }
+    Json(body)
 }
 
 async fn sync_pull(
@@ -9760,11 +9878,25 @@ async fn sync_pull(
     Json(request): Json<SyncPullRequest>,
 ) -> Result<Json<SyncPullResponse>, ApiError> {
     authorize_sync_member(&state, &request.requester, request.membership_epoch)?;
-    let operations = state.db.operations_since(&request.versions)?;
+    let peer_position = request.versions;
+    let authority_position = state.db.operation_versions()?;
+    let retained_from = state.db.retained_operation_floors()?;
+    let catchup = catchup_status(request.requester, peer_position.clone(), authority_position.clone(), retained_from);
+    if catchup.recovery_required {
+        return Err(ApiError::structured(
+            StatusCode::CONFLICT,
+            serde_json::to_value(&catchup).unwrap_or_else(|_| json!({
+                "catchup_mode": "snapshot_rebootstrap_required",
+                "recovery_required": true
+            })),
+        ));
+    }
+    let operations = state.db.operations_since(&peer_position)?;
     state.metrics.sync_sent(operations.len() as u64);
     Ok(Json(SyncPullResponse {
         operations,
-        versions: state.db.operation_versions()?,
+        versions: authority_position,
+        catchup,
     }))
 }
 
@@ -11795,7 +11927,15 @@ fn query_scalar_cmp(left: Option<&Value>, right: Option<&Value>) -> std::cmp::Or
     }
 }
 
-fn condition_matches(record: &Value, condition: &BoundedQueryCondition) -> Result<bool, ApiError> {
+/// Apply one condition to the field value the record supplies for it.
+///
+/// Taking the field value rather than the record lets the scan path evaluate a
+/// condition against a borrowed record plus its authority-supplied `recordId`,
+/// without first cloning the record to insert that field.
+fn condition_matches_field(
+    actual: Option<&Value>,
+    condition: &BoundedQueryCondition,
+) -> Result<bool, ApiError> {
     if condition.operators.len() != 1 || condition.field.trim().is_empty() {
         return Err(bounded_query_error(
             "INVALID_QUERY",
@@ -11807,7 +11947,6 @@ fn condition_matches(record: &Value, condition: &BoundedQueryCondition) -> Resul
         .iter()
         .next()
         .expect("validated operator");
-    let actual = record.get(&condition.field);
     if operator == "eq" {
         return Ok(actual == Some(expected));
     }
@@ -11832,11 +11971,166 @@ fn condition_matches(record: &Value, condition: &BoundedQueryCondition) -> Resul
     })
 }
 
+/// The record identity the query surface exposes as `recordId`.
+fn bounded_query_record_id(key: &str) -> &str {
+    key.split_once(':').map(|(_, id)| id).unwrap_or(key)
+}
+
+/// Does a stored record satisfy every condition of a bounded query?
+///
+/// This is the same conjunction the materializing path applied, evaluated
+/// against a borrowed record. `recordId` is authority metadata rather than
+/// caller-controlled document data, so it shadows a document field of that name
+/// exactly where the materializing path inserted it: on object records only.
+fn bounded_query_matches(row: &feltdb::StoredRow, conditions: &[BoundedQueryCondition]) -> bool {
+    let record_id = row
+        .value
+        .is_object()
+        .then(|| Value::String(bounded_query_record_id(&row.key).to_string()));
+    conditions.iter().all(|condition| {
+        let actual = match record_id.as_ref() {
+            Some(id) if condition.field == "recordId" => Some(id),
+            _ => row.value.get(&condition.field),
+        };
+        condition_matches_field(actual, condition).unwrap_or(false)
+    })
+}
+
+/// Declare the equality indexes this authority maintains.
+///
+/// `FELTDB_EQUALITY_INDEXES=orders.status,orders.tenantId` — a comma-separated
+/// list of `collection.field`, split at the first dot so a field name may itself
+/// contain one. Declaration is explicit and static on purpose: PR35 adds an
+/// equality execution primitive, not a planner, so nothing here inspects a
+/// workload, collects statistics, or decides on its own what deserves an index.
+///
+/// Each declaration is populated from authoritative state as it is made, so the
+/// authority never serves a query against a half-built index.
+fn declare_configured_equality_indexes(db: &FeltDb) -> Result<Vec<String>, String> {
+    let Ok(declaration) = std::env::var("FELTDB_EQUALITY_INDEXES") else {
+        return Ok(Vec::new());
+    };
+    let mut declared = Vec::new();
+    for entry in declaration.split(',').map(str::trim).filter(|entry| !entry.is_empty()) {
+        let Some((collection, field)) = entry.split_once('.') else {
+            return Err(format!(
+                "FELTDB_EQUALITY_INDEXES entry {entry:?} must be written as collection.field"
+            ));
+        };
+        db.create_equality_index(collection, field)
+            .map_err(|error| format!("cannot index {entry}: {error}"))?;
+        declared.push(format!("{collection}.{field}"));
+    }
+    Ok(declared)
+}
+
+/// Which execution a bounded query is allowed to take.
+///
+/// `Auto` is the only mode the product has: use the index when one applies,
+/// scan otherwise. `Scan` and `Index` exist so a test can run the *same* query
+/// down both paths and compare the externally visible result, which is the only
+/// way to demonstrate that an optimization changed nothing but cost. They are
+/// not a public API: the mode arrives on an undocumented request header that is
+/// read only when the process was started with `FELTDB_QUERY_DIAGNOSTICS=1`, so
+/// a production authority has exactly one execution mode and no way to be asked
+/// for another. The request body — the query contract itself — is untouched.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum BoundedQueryExecution {
+    Auto,
+    Scan,
+    Index,
+}
+
+/// Test instrumentation is compiled in but inert unless the process opted in,
+/// evaluated once so a request cannot pay for the lookup.
+fn query_diagnostics_enabled() -> bool {
+    static ENABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
+    *ENABLED.get_or_init(|| std::env::var("FELTDB_QUERY_DIAGNOSTICS").as_deref() == Ok("1"))
+}
+
+const QUERY_EXECUTION_HEADER: &str = "x-feltdb-query-execution";
+
+fn bounded_query_execution(headers: &HeaderMap) -> Result<BoundedQueryExecution, ApiError> {
+    if !query_diagnostics_enabled() {
+        return Ok(BoundedQueryExecution::Auto);
+    }
+    match headers
+        .get(QUERY_EXECUTION_HEADER)
+        .and_then(|value| value.to_str().ok())
+    {
+        None | Some("auto") => Ok(BoundedQueryExecution::Auto),
+        Some("scan") => Ok(BoundedQueryExecution::Scan),
+        Some("index") => Ok(BoundedQueryExecution::Index),
+        Some(other) => Err(bounded_query_error(
+            "INVALID_QUERY",
+            format!("unsupported query execution mode: {other}"),
+        )),
+    }
+}
+
+/// The equality conditions of a query, as candidate-selection inputs.
+///
+/// Only `eq` contributes: every other operator is a comparison this index does
+/// not model, and PR35 adds no range index. `recordId` is excluded because on
+/// this surface it is authority metadata that shadows a document field of the
+/// same name on object-valued records only — an index over the document field
+/// would answer a different question from the one `bounded_query_matches` asks,
+/// so a `recordId` condition is never index eligible and its query falls back to
+/// the scan (or is served by candidates from a different, indexed condition).
+///
+/// Conditions the index cannot use are simply absent here. They are not dropped
+/// from the query: `bounded_query_matches` still evaluates every one of them
+/// against authoritative records.
+fn bounded_query_equalities(conditions: &[BoundedQueryCondition]) -> Vec<(&str, &Value)> {
+    conditions
+        .iter()
+        .filter(|condition| condition.field != "recordId" && condition.operators.len() == 1)
+        .filter_map(|condition| {
+            let (operator, expected) = condition.operators.iter().next()?;
+            (operator == "eq").then_some((condition.field.as_str(), expected))
+        })
+        .collect()
+}
+
+/// Evaluate a bounded query's conjunction against authoritative records.
+///
+/// Two executions, one result. The indexed execution narrows *which* records the
+/// predicate sees; `bounded_query_matches` — unchanged, and the sole authority on
+/// what matches — decides the answer on either path. When no index applies the
+/// scan runs exactly as PR34 left it.
+fn bounded_query_records(
+    state: &AppState,
+    request: &BoundedQueryRequest,
+    execution: BoundedQueryExecution,
+) -> Result<Vec<feltdb::StoredRow>, ApiError> {
+    if execution != BoundedQueryExecution::Scan {
+        let equalities = bounded_query_equalities(&request.conditions);
+        let indexed = state.db.query_collection_by_equality(
+            &request.collection,
+            &equalities,
+            |row| bounded_query_matches(row, &request.conditions),
+        )?;
+        if let Some(records) = indexed {
+            return Ok(records);
+        }
+        if execution == BoundedQueryExecution::Index {
+            // Only reachable through the test-only header. A forced-index run
+            // that silently scanned would make an equivalence test prove nothing.
+            return Err(bounded_query_error(
+                "INDEX_UNAVAILABLE",
+                "no equality index applies to this query",
+            ));
+        }
+    }
+    Ok(state.db.query_collection(&request.collection, None, |row| {
+        bounded_query_matches(row, &request.conditions)
+    })?)
+}
+
 fn issue_bounded_cursor(
     state: &AppState,
     mut cursor: BoundedQueryCursor,
-) -> Result<String, ApiError> {
-    let token = uuid::Uuid::new_v4().simple().to_string();
+) -> Result<String, ApiError> {    let token = uuid::Uuid::new_v4().simple().to_string();
     let now = unix_seconds_i64().max(0) as u64;
     cursor.created_at = now;
     let mut cursors = state.bounded_query_cursors.lock().map_err(|_| {
@@ -11859,8 +12153,12 @@ fn issue_bounded_cursor(
 async fn execute_bounded_query(
     State(state): State<AppState>,
     Extension(principal): Extension<Principal>,
+    headers: HeaderMap,
     Json(request): Json<BoundedQueryRequest>,
 ) -> Result<Json<BoundedQueryPage>, ApiError> {
+    let _handler =
+        feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::HandlerBody);
+    let execution = bounded_query_execution(&headers)?;
     validate_segment(&request.collection)?;
     if request.limit == 0 || request.limit > MAX_BOUNDED_QUERY_LIMIT {
         return Err(bounded_query_error(
@@ -11928,30 +12226,40 @@ async fn execute_bounded_query(
         }
         (cursor.records, cursor.position)
     } else {
-        let mut records = Vec::new();
-        for row in state.db.list_collection(&request.collection)? {
+        // Bounded queries evaluate their conjunction against borrowed records
+        // and clone only the matches, so a query never pays to materialize the
+        // records it is about to discard. An equality index, where one applies,
+        // narrows which records the conjunction is evaluated against; it does
+        // not decide the answer and it does not order it. Ordering is part of
+        // this surface's contract, so the matching set — not the collection, and
+        // not an index bucket — is what gets sorted, and the limit is applied to
+        // that ordered set. Nothing below this line knows which execution ran.
+        let matches = bounded_query_records(&state, &request, execution)?;
+        // Result materialization happens outside the state lock, so it is a cost
+        // this request pays alone rather than one it imposes on every other
+        // operation. PR36 measures it separately for exactly that reason.
+        let materialization = feltdb::workload_diagnostics::span(
+            feltdb::workload_diagnostics::Phase::ResultMaterialization,
+        );
+        let mut records = Vec::with_capacity(matches.len());
+        for row in matches {
             let mut value = row.value;
             if let Some(object) = value.as_object_mut() {
-                let id = row
-                    .key
-                    .split_once(':')
-                    .map(|(_, id)| id)
-                    .unwrap_or(&row.key);
                 // `recordId` is authority metadata for this query surface, not
                 // caller-controlled document data. It is the final total-order
                 // tie-breaker even when a document contains a field by that name.
-                object.insert("recordId".into(), Value::String(id.to_string()));
-            }
-            if request
-                .conditions
-                .iter()
-                .all(|condition| condition_matches(&value, condition).unwrap_or(false))
-            {
-                records.push(value);
+                object.insert(
+                    "recordId".into(),
+                    Value::String(bounded_query_record_id(&row.key).to_string()),
+                );
             }
+            records.push(value);
         }
+        drop(materialization);
         // The authority always adds its immutable record identity as the final
         // tie-breaker, so equal user sort values still form a total order.
+        let ordering =
+            feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::Ordering);
         records.sort_by(|left, right| {
             for order in &request.order_by {
                 let comparison = query_scalar_cmp(left.get(&order.field), right.get(&order.field));
@@ -11965,6 +12273,7 @@ async fn execute_bounded_query(
             }
             query_scalar_cmp(left.get("recordId"), right.get("recordId"))
         });
+        drop(ordering);
         (Arc::new(records), 0)
     };
     let end = position.saturating_add(request.limit).min(records.len());
@@ -11991,6 +12300,131 @@ async fn execute_bounded_query(
     }))
 }
 
+/// Collection-read counters, mounted only when the process opts in.
+///
+/// This is test instrumentation for the PR34 no-materialization proof, not part
+/// of the query contract, so a server only carries the route when it is started
+/// with `FELTDB_QUERY_DIAGNOSTICS=1`. It returns counts, never record data.
+fn query_execution_diagnostics_router() -> Router<AppState> {
+    let mut router = Router::new();
+    if std::env::var("FELTDB_QUERY_DIAGNOSTICS").as_deref() == Ok("1") {
+        router = router.route(
+            "/internal/query-execution-diagnostics",
+            get(query_execution_diagnostics),
+        );
+    }
+    // Phase attribution is a separate switch from the execution counters,
+    // because it is a separate cost: the counters are a handful of atomic adds
+    // per query, while attribution reads the clock at every instrumented
+    // boundary. Keeping them apart lets a run take execution counts without
+    // paying for timing, which is how the headline PR36 figures are measured.
+    if std::env::var("FELTDB_WORKLOAD_DIAGNOSTICS").as_deref() == Ok("1") {
+        router = router
+            .route(
+                "/internal/workload-diagnostics",
+                get(workload_phase_diagnostics),
+            )
+            .route(
+                "/internal/workload-diagnostics/reset",
+                axum::routing::post(reset_workload_phase_diagnostics),
+            );
+    }
+    router
+}
+
+/// Phase attribution: where elapsed time went, as counts and nanoseconds.
+///
+/// Test instrumentation, mounted only under `FELTDB_WORKLOAD_DIAGNOSTICS=1`.
+/// It reports durations and call counts and nothing else — no record, no field,
+/// no value, no id — and no product surface reads it.
+///
+/// `parent` is part of the payload because the phases nest: a caller that summed
+/// every row would double-count the work reported inside `state_lock_hold` and
+/// `indexed_query`. The nesting is declared here so an analysis can compute
+/// exclusive time instead of guessing at it.
+async fn workload_phase_diagnostics() -> Json<Value> {
+    let phases: Vec<Value> = feltdb::workload_diagnostics::counters()
+        .into_iter()
+        .map(|counter| {
+            json!({
+                "phase": counter.phase.name(),
+                "parent": counter.phase.parent().map(|parent| parent.name()),
+                "calls": counter.calls,
+                "nanos": counter.nanos,
+                "meanNanos": counter.mean_nanos(),
+            })
+        })
+        .collect();
+    Json(json!({
+        "enabled": feltdb::workload_diagnostics::enabled(),
+        "phases": phases,
+    }))
+}
+
+/// Zero the phase counters, so a measured window starts from a clean base.
+async fn reset_workload_phase_diagnostics() -> Json<Value> {
+    feltdb::workload_diagnostics::reset();
+    Json(json!({ "reset": true }))
+}
+
+/// Ask the diagnostics route to also verify the index against records.
+#[derive(Deserialize)]
+struct QueryDiagnosticsScope {
+    #[serde(default)]
+    verify: Option<String>,
+}
+
+async fn query_execution_diagnostics(
+    State(state): State<AppState>,
+    Query(scope): Query<QueryDiagnosticsScope>,
+) -> Json<Value> {
+    let counters = feltdb::query_execution_diagnostics::counters();
+    // Verifying rebuilds the whole index from records to compare against the
+    // live one, so it is linear in collection size and is asked for explicitly.
+    // A counter read that silently paid for a rebuild would make every
+    // measurement that brackets a query with a counter read measure the rebuild.
+    let consistency = match scope.verify.as_deref() {
+        Some("1") => state
+            .db
+            .verify_equality_index()
+            .map(|outcome| Value::Bool(outcome.is_ok()))
+            .unwrap_or(Value::Null),
+        _ => Value::Null,
+    };
+    Json(json!({
+        "fullCollectionMaterializations": counters.full_collection_materializations,
+        "fullCollectionRecordsCloned": counters.full_collection_records_cloned,
+        "collectionScans": counters.collection_scans,
+        "scanRecordsVisited": counters.scan_records_visited,
+        "scanRecordsMaterialized": counters.scan_records_materialized,
+        "queriesTotal": counters.queries_total,
+        "queriesScan": counters.queries_scan,
+        "queriesIndexed": counters.queries_indexed,
+        "indexHits": counters.index_hits,
+        "indexMisses": counters.index_misses,
+        "indexCandidatesExamined": counters.index_candidates_examined,
+        "recordsPredicateEvaluated": counters.records_predicate_evaluated,
+        "equalityIndexes": state
+            .db
+            .equality_indexes()
+            .map(|indexes| {
+                indexes
+                    .into_iter()
+                    .map(|(collection, field)| format!("{collection}.{field}"))
+                    .collect::<Vec<_>>()
+            })
+            .unwrap_or_default(),
+        "equalityIndexStats": state.db.equality_index_stats().map(|stats| json!({
+            "indexedFields": stats.indexed_fields,
+            "valueBuckets": stats.value_buckets,
+            "entries": stats.entries,
+        })).unwrap_or(Value::Null),
+        // Null unless `?verify=1` asked for the check; a null here means "not
+        // checked", never "checked and inconsistent".
+        "equalityIndexConsistent": consistency,
+    }))
+}
+
 async fn list_records(
     State(state): State<AppState>,
     Path(collection): Path<String>,
@@ -12015,6 +12449,8 @@ async fn get_record(
     State(state): State<AppState>,
     Path((collection, id)): Path<(String, String)>,
 ) -> Result<Json<RecordResponse>, ApiError> {
+    let _handler =
+        feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::HandlerBody);
     if collection == PROPOSAL_COLLECTION
         || collection == PROPOSAL_EVENT_COLLECTION
         || collection == CONVERGENCE_COLLECTION
@@ -12123,6 +12559,8 @@ async fn commit_transaction(
     State(state): State<AppState>,
     Json(request): Json<AtomicTxRequest>,
 ) -> Result<(StatusCode, Json<AtomicTxResponse>), ApiError> {
+    let _handler =
+        feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::HandlerBody);
     if request.transaction_id.trim().is_empty() {
         return Err(ApiError(
             StatusCode::BAD_REQUEST,
@@ -12272,6 +12710,8 @@ async fn create_record(
     Path(collection): Path<String>,
     Json(mut record): Json<CreateRecord>,
 ) -> Result<(StatusCode, Json<RecordResponse>), ApiError> {
+    let _handler =
+        feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::HandlerBody);
     if collection == PROPOSAL_COLLECTION
         || collection == PROPOSAL_EVENT_COLLECTION
         || collection == CONVERGENCE_COLLECTION
@@ -12316,6 +12756,8 @@ async fn update_record(
     Path((collection, id)): Path<(String, String)>,
     Json(changes): Json<Map<String, Value>>,
 ) -> Result<Json<RecordResponse>, ApiError> {
+    let _handler =
+        feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::HandlerBody);
     if collection == PROPOSAL_COLLECTION
         || collection == PROPOSAL_EVENT_COLLECTION
         || collection == CONVERGENCE_COLLECTION
@@ -12353,6 +12795,8 @@ async fn compare_and_set_record(
     Path((collection, id)): Path<(String, String)>,
     Json(request): Json<CasRecordRequest>,
 ) -> Result<Response, ApiError> {
+    let _handler =
+        feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::HandlerBody);
     let key = record_key(&collection, &id)?;
     let result = state.db.compare_and_set_json(
         &key,
@@ -12442,6 +12886,8 @@ async fn delete_record(
     State(state): State<AppState>,
     Path((collection, id)): Path<(String, String)>,
 ) -> Result<StatusCode, ApiError> {
+    let _handler =
+        feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::HandlerBody);
     if collection == PROPOSAL_COLLECTION
         || collection == PROPOSAL_EVENT_COLLECTION
         || collection == CONVERGENCE_COLLECTION