@feltdb/core 0.8.5 → 0.8.7

package/dist/create/server-source/crates/feltdb-server/src/main.rs

@@ -6883,7 +6883,8 @@ async fn application_openapi() -> Json<Value> {
         },
         "security": [{ "bearer": [] }],
         "paths": {
-            "/health": { "get": { "security": [], "responses": { "200": { "description": "Runtime health" } } } },
+            "/health": { "get": { "security": [], "responses": { "200": { "description": "Process liveness" } } } },
+            "/health/ready": { "get": { "security": [], "responses": { "200": { "description": "Storage and scoped-operation readiness" }, "503": { "description": "Not ready" } } } },
             "/application": { "get": { "responses": { "200": { "description": "Active application contract identity" } } } },
             "/schema": { "get": { "responses": { "200": { "description": "Active application schema" } } } },
             "/query": { "post": { "responses": { "200": { "description": "Bounded authorized query" } } } },
@@ -8091,6 +8092,8 @@ struct Config {
     peers: Vec<String>,
     peer_token_env: String,
     sync_interval: Duration,
+    max_in_flight: usize,
+    request_deadline: Duration,
     audit: PathBuf,
     allowed_origins: Vec<axum::http::HeaderValue>,
     worker_enabled: bool,
@@ -8119,6 +8122,11 @@ impl Config {
         let mut peers = Vec::new();
         let mut peer_token_env = "FELTDB_PEER_TOKEN".to_string();
         let mut sync_interval = Duration::from_secs(2);
+        // Concurrency past this point does not buy throughput: the database
+        // serializes every read and write on one lock. It only decides whether
+        // an overload is shed quickly or queued invisibly.
+        let mut max_in_flight = 64usize;
+        let mut request_deadline = Duration::from_secs(10);
         let mut audit = None;
         let mut allowed_origins = std::env::var("FELTDB_ALLOWED_ORIGINS")
             .ok()
@@ -8165,6 +8173,23 @@ impl Config {
                     }
                     sync_interval = Duration::from_millis(milliseconds);
                 }
+                "--max-in-flight" => {
+                    max_in_flight = value()?
+                        .parse()
+                        .map_err(|_| "invalid --max-in-flight".to_string())?;
+                    if max_in_flight == 0 {
+                        return Err("--max-in-flight must be at least 1".to_string());
+                    }
+                }
+                "--request-deadline-ms" => {
+                    let milliseconds: u64 = value()?
+                        .parse()
+                        .map_err(|_| "invalid --request-deadline-ms".to_string())?;
+                    if milliseconds < 100 {
+                        return Err("--request-deadline-ms must be at least 100".to_string());
+                    }
+                    request_deadline = Duration::from_millis(milliseconds);
+                }
                 "--audit" => audit = Some(PathBuf::from(value()?)),
                 "--allow-origin" => allowed_origins.push(
                     value()?
@@ -8204,6 +8229,8 @@ impl Config {
             peers,
             peer_token_env,
             sync_interval,
+            max_in_flight,
+            request_deadline,
             audit,
             allowed_origins,
             worker_enabled,
@@ -8357,10 +8384,16 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let audit_handle = AuditLog::new(config.audit.clone());
     let state = AppState {
         started_at: Instant::now(),
+        admission: Arc::new(tokio::sync::Semaphore::new(config.max_in_flight)),
+        request_deadline: config.request_deadline,
         ids: Arc::new(AtomicU64::new(db.sequence()?)),
         db,
         namespace: Arc::from(config.namespace.clone()),
         auth_enabled: config.auth_enabled,
+        // Argon2 is deliberately memory-hard. Bound concurrent legacy/session
+        // verification so a burst waits asynchronously instead of exhausting
+        // a small production VM with one allocation per request.
+        authentication_workers: Arc::new(tokio::sync::Semaphore::new(2)),
         keys: KeyStore::load(&config.keys)?,
         metrics: Metrics::default(),
         cluster: ClusterStore::load(
@@ -9063,6 +9096,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     let mut app = Router::new()
         .route("/health", get(health))
+        .route("/health/ready", get(health_ready))
         .route("/v1/health", get(health))
         .route("/v1/openapi.json", get(application_openapi))
         .route("/v1/auth/signup", axum::routing::post(auth_sign_up))
@@ -9086,6 +9120,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         )
         .merge(protected)
         .merge(query_execution_diagnostics_router())
+        .layer(middleware::from_fn_with_state(state.clone(), admit_request))
         .layer(middleware::from_fn(protocol_version))
         .layer(middleware::from_fn(attribute_request))
         .layer(middleware::from_fn_with_state(state.clone(), count_request))
@@ -9171,16 +9206,52 @@ async fn authenticate(
             .headers()
             .get(AUTHORIZATION)
             .and_then(|header| header.to_str().ok())
-            .and_then(|header| header.strip_prefix("Bearer "));
-        let machine = token.and_then(|token| state.keys.authenticate(token, &state.namespace));
+            .and_then(|header| header.strip_prefix("Bearer "))
+            .map(str::to_owned);
+        let keys = state.keys.clone();
+        let identities = state.identities.clone();
+        let namespace = state.namespace.clone();
+        let authentication_worker = match state.authentication_workers.clone().acquire_owned().await
+        {
+            Ok(permit) => permit,
+            Err(error) => {
+                tracing::error!(%error, "authentication worker limit closed");
+                return ApiError(
+                    StatusCode::SERVICE_UNAVAILABLE,
+                    "authentication temporarily unavailable".into(),
+                )
+                .into_response();
+            }
+        };
+        let (machine, actor) = match tokio::task::spawn_blocking(move || {
+            let _authentication_worker = authentication_worker;
+            let machine = token
+                .as_deref()
+                .and_then(|token| keys.authenticate(token, &namespace));
+            let actor = if machine.is_none() {
+                token
+                    .as_deref()
+                    .and_then(|token| identities.authenticate_session(token))
+            } else {
+                None
+            };
+            (machine, actor)
+        })
+        .await
+        {
+            Ok(authentication) => authentication,
+            Err(error) => {
+                tracing::error!(%error, "bearer authentication worker failed");
+                return ApiError(
+                    StatusCode::SERVICE_UNAVAILABLE,
+                    "authentication temporarily unavailable".into(),
+                )
+                .into_response();
+            }
+        };
         // A valid service key is already authoritative. Do not feed that
         // high-entropy bearer token through the Argon2-backed human-session
         // verifier as well; doing so blocks the request worker needlessly.
-        let actor = if machine.is_none() {
-            token.and_then(|token| state.identities.authenticate_session(token))
-        } else {
-            None
-        };
         let human = request
             .headers()
             .get(COOKIE)
@@ -9431,6 +9502,234 @@ async fn attribute_request(request: Request<axum::body::Body>, next: Next) -> Re
     next.run(request).await
 }
 
+/// The overall status, from the conditions that actually bear on it.
+///
+/// Storage participates. Before it did not, so a database that had discarded an
+/// incomplete final append reported exactly what a cleanly replayed one did —
+/// which is the difference between an operator learning their last write was
+/// lost and never finding out.
+fn overall_status(
+    membership_healthy: bool,
+    clock_healthy: bool,
+    storage: &feltdb::StorageHealth,
+) -> &'static str {
+    if membership_healthy && clock_healthy && storage.is_clean() {
+        "healthy"
+    } else {
+        "degraded"
+    }
+}
+
+#[cfg(test)]
+mod admission_tests {
+    use super::overloaded;
+    use axum::http::StatusCode;
+    use axum::response::IntoResponse;
+
+    /// An overload is refused with a status and a hint, not a hang.
+    ///
+    /// The managed incident produced client-side timeouts and upstream 502s
+    /// because the server had no inbound deadline and no admission bound: a
+    /// request that could not proceed simply waited. A caller cannot act on
+    /// that. It can act on this.
+    #[test]
+    fn an_overload_is_an_explicit_refusal_with_a_retry_hint() {
+        let response = overloaded("server is at its in-flight request limit").into_response();
+        assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
+        assert_eq!(
+            response
+                .headers()
+                .get(axum::http::header::RETRY_AFTER)
+                .and_then(|value| value.to_str().ok()),
+            Some("1"),
+            "a caller is told when to come back"
+        );
+    }
+
+    /// 503 rather than 500: the request was refused, not mishandled.
+    #[test]
+    fn an_overload_is_not_reported_as_a_server_fault() {
+        let response = overloaded("request exceeded the server deadline").into_response();
+        assert_ne!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
+        assert!(response.status().is_server_error());
+    }
+}
+
+#[cfg(test)]
+mod health_status_tests {
+    use super::overall_status;
+    use feltdb::StorageHealth;
+
+    const RECOVERED: StorageHealth = StorageHealth::RecoveredIncompleteWrite {
+        byte_offset: 128,
+        discarded_bytes: 40,
+    };
+
+    /// Storage participates in the overall status.
+    ///
+    /// It did not before: a database that had discarded an incomplete final
+    /// append reported exactly what a cleanly replayed one did, so the
+    /// distinction the durable-log work established never reached an operator.
+    #[test]
+    fn a_recovered_database_is_reported_degraded() {
+        assert_eq!(overall_status(true, true, &StorageHealth::Clean), "healthy");
+        assert_eq!(overall_status(true, true, &RECOVERED), "degraded");
+    }
+
+    /// And the conditions that already counted still do, so this narrows
+    /// nothing.
+    #[test]
+    fn membership_and_clock_still_count() {
+        assert_eq!(
+            overall_status(false, true, &StorageHealth::Clean),
+            "degraded"
+        );
+        assert_eq!(
+            overall_status(true, false, &StorageHealth::Clean),
+            "degraded"
+        );
+        assert_eq!(overall_status(false, false, &RECOVERED), "degraded");
+    }
+
+    /// The reported storage value names an observation rather than asserting
+    /// the property that is still unproven.
+    #[test]
+    fn the_storage_label_is_an_observation_not_a_promise() {
+        assert_eq!(StorageHealth::Clean.label(), "clean");
+        assert_eq!(RECOVERED.label(), "recovered-incomplete-write");
+        for label in [StorageHealth::Clean.label(), RECOVERED.label()] {
+            assert_ne!(label, "durable", "health must not claim durability");
+        }
+    }
+}
+
+/// Shed load at the door, and abandon work the caller is no longer waiting for.
+///
+/// Two failures this replaces, both observed in production:
+///
+/// - **No inbound deadline.** A slow operation ran until the *client* gave up,
+///   and then kept running, so the load that caused the stall outlived every
+///   request that reported it.
+/// - **No admission bound.** Requests queued on the database's single lock
+///   until an upstream proxy returned 502. Concurrency past the bound buys no
+///   throughput — every read and write serializes on that lock — so all it
+///   decides is whether an overload is refused quickly or hidden.
+///
+/// Both now produce an explicit `503` with `Retry-After`, which is a signal a
+/// caller can act on rather than a timeout it has to interpret.
+async fn admit_request(
+    State(state): State<AppState>,
+    request: axum::extract::Request,
+    next: Next,
+) -> axum::response::Response {
+    let Ok(_permit) = state.admission.clone().try_acquire_owned() else {
+        state.metrics.request_shed();
+        return overloaded("server is at its in-flight request limit");
+    };
+
+    match tokio::time::timeout(state.request_deadline, next.run(request)).await {
+        Ok(response) => response,
+        Err(_) => {
+            state.metrics.request_deadline_exceeded();
+            overloaded("request exceeded the server deadline")
+        }
+    }
+}
+
+/// A refusal a caller can act on: an explicit status, a stable code, and a hint.
+fn overloaded(detail: &str) -> axum::response::Response {
+    let mut response = (
+        StatusCode::SERVICE_UNAVAILABLE,
+        Json(json!({
+            "error": detail,
+            "code": "SERVICE_OVERLOADED",
+        })),
+    )
+        .into_response();
+    response.headers_mut().insert(
+        axum::http::header::RETRY_AFTER,
+        axum::http::HeaderValue::from_static("1"),
+    );
+    response
+}
+
+/// Readiness: whether this server can actually serve a scoped operation.
+///
+/// `/health` answers "is this process alive". It reads cluster state and the
+/// lease clock and never touches storage, so it cannot fail for a database
+/// reason — which is exactly how a deployment reported `200 healthy` while
+/// every scoped read timed out.
+///
+/// This endpoint takes the database lock and performs a real bounded read, off
+/// the runtime and under the same deadline a request gets. **A server whose
+/// scoped operations are stalled reports `503` here.**
+async fn health_ready(State(state): State<AppState>) -> axum::response::Response {
+    let probe = state.clone();
+    let started = Instant::now();
+    let outcome = tokio::time::timeout(
+        state.request_deadline,
+        tokio::task::spawn_blocking(move || {
+            // A real scoped operation: acquire the state lock and read through
+            // it. Bounded, so a large database does not make readiness slow.
+            let health = probe.db.health();
+            let _ = probe
+                .db
+                .list_collection_page("_flow_capabilities", None, 1)?;
+            Ok::<_, feltdb::FlowError>(health)
+        }),
+    )
+    .await;
+
+    let waited_ms = started.elapsed().as_millis() as u64;
+    match outcome {
+        Ok(Ok(Ok(health))) => {
+            let ready = health.is_nominal();
+            let body = Json(json!({
+                "ready": ready,
+                "storage": health.storage.label(),
+                "durable_format": health.durable_format.to_string(),
+                "probe_ms": waited_ms,
+            }));
+            if ready {
+                (StatusCode::OK, body).into_response()
+            } else {
+                // Usable, but not in the condition a clean open produces.
+                (StatusCode::SERVICE_UNAVAILABLE, body).into_response()
+            }
+        }
+        Ok(Ok(Err(error))) => (
+            StatusCode::SERVICE_UNAVAILABLE,
+            Json(json!({
+                "ready": false,
+                "error": error.to_string(),
+                "code": "STORAGE_UNAVAILABLE",
+                "probe_ms": waited_ms,
+            })),
+        )
+            .into_response(),
+        Ok(Err(error)) => (
+            StatusCode::SERVICE_UNAVAILABLE,
+            Json(json!({
+                "ready": false,
+                "error": error.to_string(),
+                "code": "STORAGE_PROBE_FAILED",
+                "probe_ms": waited_ms,
+            })),
+        )
+            .into_response(),
+        Err(_) => (
+            StatusCode::SERVICE_UNAVAILABLE,
+            Json(json!({
+                "ready": false,
+                "error": "storage probe exceeded the server deadline",
+                "code": "STORAGE_STALLED",
+                "probe_ms": waited_ms,
+            })),
+        )
+            .into_response(),
+    }
+}
+
 async fn health(State(state): State<AppState>) -> Json<HealthResponse<'static>> {
     let membership = match state.cluster.proposal().map(|value| value.phase) {
         Some(ProposalPhase::Preparing | ProposalPhase::Prepared) => "recovering",
@@ -9438,16 +9737,16 @@ async fn health(State(state): State<AppState>) -> Json<HealthResponse<'static>>
         _ => "healthy",
     };
     let clock = state.lease_clock.is_healthy();
+    // Observed, not asserted. An incompatible or corrupt database never opens,
+    // so a running server has already established its format; what remains to
+    // report is whether replay discarded anything.
+    let database = state.db.health();
     Json(HealthResponse {
-        status: if membership == "healthy" && clock {
-            "healthy"
-        } else {
-            "degraded"
-        },
+        status: overall_status(membership == "healthy", clock, &database.storage),
         version: env!("CARGO_PKG_VERSION"),
         git_commit: option_env!("FELTDB_GIT_COMMIT").unwrap_or("unknown"),
         runtime: "self-hosted",
-        storage: "durable",
+        storage: database.storage.label(),
         fabric: "healthy",
         execution: if state.worker_enabled {
             "autonomous"
@@ -9488,22 +9787,33 @@ async fn managed_diagnostics(State(state): State<AppState>) -> Json<ManagedDiagn
 }
 
 async fn readiness(State(state): State<AppState>) -> Result<Json<Value>, ApiError> {
-    if let Some(parent) = state.readiness_probe.parent() {
-        std::fs::create_dir_all(parent)
+    let readiness_probe = state.readiness_probe.clone();
+    let db = state.db.clone();
+    let (state_version, storage) = tokio::task::spawn_blocking(move || {
+        if let Some(parent) = readiness_probe.parent() {
+            std::fs::create_dir_all(parent)
+                .map_err(|error| ApiError(StatusCode::SERVICE_UNAVAILABLE, error.to_string()))?;
+        }
+        let mut probe = OpenOptions::new()
+            .create(true)
+            .write(true)
+            .truncate(true)
+            .open(readiness_probe.as_ref())
             .map_err(|error| ApiError(StatusCode::SERVICE_UNAVAILABLE, error.to_string()))?;
-    }
-    let mut probe = OpenOptions::new()
-        .create(true)
-        .write(true)
-        .truncate(true)
-        .open(state.readiness_probe.as_ref())
-        .map_err(|error| ApiError(StatusCode::SERVICE_UNAVAILABLE, error.to_string()))?;
-    probe
-        .write_all(b"feltdb-ready-v1\n")
-        .and_then(|_| probe.sync_all())
-        .map_err(|error| ApiError(StatusCode::SERVICE_UNAVAILABLE, error.to_string()))?;
-    let state_version = state.db.sequence()?;
-    let storage = state.db.instance_id()?;
+        probe
+            .write_all(b"feltdb-ready-v1\n")
+            .and_then(|_| probe.sync_all())
+            .map_err(|error| ApiError(StatusCode::SERVICE_UNAVAILABLE, error.to_string()))?;
+        Ok::<_, ApiError>((db.sequence()?, db.instance_id()?))
+    })
+    .await
+    .map_err(|error| {
+        tracing::error!(%error, "readiness worker failed");
+        ApiError(
+            StatusCode::SERVICE_UNAVAILABLE,
+            "readiness check temporarily unavailable".into(),
+        )
+    })??;
     Ok(Json(json!({
         "status": "ready",
         "authority": "rust",
@@ -10072,9 +10382,25 @@ fn start_peer_sessions(
                     }
                 }
             }
-            match state.db.compact_operation_log(&peers) {
-                Ok(removed) => state.metrics.operations_compacted(removed as u64),
-                Err(error) => tracing::warn!(%error, "operation compaction failed"),
+            // Policy-driven, and off the runtime. Compaction rewrites the log
+            // while holding the lock that serializes every read and write, so
+            // running it on every tick stopped the world on every tick — and
+            // running it inline blocked a runtime worker while it did.
+            let compactor = state.clone();
+            let compaction_peers = peers.clone();
+            match tokio::task::spawn_blocking(move || {
+                compactor.db.maybe_compact_operation_log(&compaction_peers)
+            })
+            .await
+            {
+                Ok(Ok(outcome)) => {
+                    state.metrics.operations_compacted(outcome.pruned() as u64);
+                    if outcome.rewrote_log() {
+                        tracing::debug!(pruned = outcome.pruned(), "durable log rewritten");
+                    }
+                }
+                Ok(Err(error)) => tracing::warn!(%error, "operation compaction failed"),
+                Err(error) => tracing::warn!(%error, "operation compaction task failed"),
             }
             tokio::time::sleep(interval).await;
         }
@@ -10395,10 +10721,10 @@ fn execute_builtin_capability(
             Ok(Value::Array(
                 state
                     .db
-                    .list_collection(collection)?
+                    .query_collection(collection, Some(limit), |row| {
+                        row.value.to_string().to_lowercase().contains(&needle)
+                    })?
                     .into_iter()
-                    .filter(|row| row.value.to_string().to_lowercase().contains(&needle))
-                    .take(limit)
                     .map(|row| row.value)
                     .collect(),
             ))
@@ -11439,11 +11765,11 @@ async fn get_provenance(
     Path((collection, id)): Path<(String, String)>,
 ) -> Result<Json<Value>, ApiError> {
     let key = record_key(&collection, &id)?;
+    // A keyed lookup. This used to materialize the whole collection — cloning
+    // every row under the global state lock — to find one record.
     let row = state
         .db
-        .list_collection(&collection)?
-        .into_iter()
-        .find(|row| row.key == key)
+        .get_collection_record(&collection, &key)?
         .ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "record not found".to_string()))?;
     let operation = row.operation.as_ref();
     Ok(Json(json!({
@@ -11473,12 +11799,14 @@ async fn search_collection(
         ));
     }
     let needle = request.query.to_lowercase();
+    // Borrows each row and stops at the limit, rather than cloning the whole
+    // collection and filtering afterwards. Only matches are cloned.
     let records = state
         .db
-        .list_collection(&collection)?
+        .query_collection(&collection, Some(request.limit), |row| {
+            row.value.to_string().to_lowercase().contains(&needle)
+        })?
         .into_iter()
-        .filter(|row| row.value.to_string().to_lowercase().contains(&needle))
-        .take(request.limit)
         .map(record_response)
         .collect();
     Ok(Json(records))

package/dist/create/server-source/crates/feltdb-server/src/metrics.rs

@@ -21,6 +21,8 @@ struct Inner {
     peer_sync_successes: AtomicU64,
     peer_sync_failures: AtomicU64,
     operations_compacted: AtomicU64,
+    requests_shed: AtomicU64,
+    request_deadlines_exceeded: AtomicU64,
 }
 
 #[derive(Serialize)]
@@ -35,6 +37,12 @@ pub struct MetricsSnapshot {
     pub peer_sync_successes: u64,
     pub peer_sync_failures: u64,
     pub operations_compacted: u64,
+    /// Requests refused at the door because the server was already at its
+    /// in-flight limit. Non-zero means load is being shed rather than queued.
+    pub requests_shed: u64,
+    /// Requests the server abandoned for exceeding its own deadline. Non-zero
+    /// means scoped operations are stalling, whatever `/health` says.
+    pub request_deadlines_exceeded: u64,
 }
 
 impl Metrics {
@@ -65,6 +73,14 @@ impl Metrics {
             .sync_operations_received
             .fetch_add(count, Ordering::Relaxed);
     }
+    pub fn request_shed(&self) {
+        self.inner.requests_shed.fetch_add(1, Ordering::Relaxed);
+    }
+    pub fn request_deadline_exceeded(&self) {
+        self.inner
+            .request_deadlines_exceeded
+            .fetch_add(1, Ordering::Relaxed);
+    }
     pub fn peer_sync_success(&self) {
         self.inner
             .peer_sync_successes
@@ -92,6 +108,11 @@ impl Metrics {
             peer_sync_successes: self.inner.peer_sync_successes.load(Ordering::Relaxed),
             peer_sync_failures: self.inner.peer_sync_failures.load(Ordering::Relaxed),
             operations_compacted: self.inner.operations_compacted.load(Ordering::Relaxed),
+            requests_shed: self.inner.requests_shed.load(Ordering::Relaxed),
+            request_deadlines_exceeded: self
+                .inner
+                .request_deadlines_exceeded
+                .load(Ordering::Relaxed),
         }
     }
 }