@feltdb/core 0.8.4 → 0.8.6

package/dist/create/server-source/crates/feltdb-server/src/main.rs

@@ -62,7 +62,7 @@ use feltdb::{
     },
     workload::{CreateWorkload, WorkloadStore},
     AtomicMutation, DatabaseSnapshot, FeltDb, FlowError, JsonCasResult, Operation,
-    PeerAdvertisement, PeerId, RecordPrecondition, StoredRow,
+    PeerAdvertisement, PeerId, RecordPrecondition, StateTriggerStore, StoredRow,
 };
 use feltdb_server::{
     app_state::{AppState, BoundedQueryCursor},
@@ -1421,9 +1421,10 @@ struct WorkerReconcileRequest {
 fn workload_error(error: feltdb::workload::WorkloadError) -> ApiError {
     let status = match error.code.as_str() {
         "WORKLOAD_NOT_FOUND" => StatusCode::NOT_FOUND,
-        "WORKLOAD_STALE_FENCE" | "WORKLOAD_TRANSITION_DENIED" | "WORKLOAD_NOT_READY" => {
-            StatusCode::CONFLICT
-        }
+        "WORKLOAD_STALE_FENCE"
+        | "WORKLOAD_TRANSITION_DENIED"
+        | "WORKLOAD_NOT_READY"
+        | "WORKLOAD_NOT_YET_ELIGIBLE" => StatusCode::CONFLICT,
         "WORKLOAD_UNAUTHORIZED" | "WORKLOAD_CAPABILITY_DENIED" => StatusCode::FORBIDDEN,
         "WORKLOAD_STORAGE_FAILURE" => StatusCode::SERVICE_UNAVAILABLE,
         _ => StatusCode::UNPROCESSABLE_ENTITY,
@@ -6882,7 +6883,8 @@ async fn application_openapi() -> Json<Value> {
         },
         "security": [{ "bearer": [] }],
         "paths": {
-            "/health": { "get": { "security": [], "responses": { "200": { "description": "Runtime health" } } } },
+            "/health": { "get": { "security": [], "responses": { "200": { "description": "Process liveness" } } } },
+            "/health/ready": { "get": { "security": [], "responses": { "200": { "description": "Storage and scoped-operation readiness" }, "503": { "description": "Not ready" } } } },
             "/application": { "get": { "responses": { "200": { "description": "Active application contract identity" } } } },
             "/schema": { "get": { "responses": { "200": { "description": "Active application schema" } } } },
             "/query": { "post": { "responses": { "200": { "description": "Bounded authorized query" } } } },
@@ -8090,6 +8092,8 @@ struct Config {
     peers: Vec<String>,
     peer_token_env: String,
     sync_interval: Duration,
+    max_in_flight: usize,
+    request_deadline: Duration,
     audit: PathBuf,
     allowed_origins: Vec<axum::http::HeaderValue>,
     worker_enabled: bool,
@@ -8118,6 +8122,11 @@ impl Config {
         let mut peers = Vec::new();
         let mut peer_token_env = "FELTDB_PEER_TOKEN".to_string();
         let mut sync_interval = Duration::from_secs(2);
+        // Concurrency past this point does not buy throughput: the database
+        // serializes every read and write on one lock. It only decides whether
+        // an overload is shed quickly or queued invisibly.
+        let mut max_in_flight = 64usize;
+        let mut request_deadline = Duration::from_secs(10);
         let mut audit = None;
         let mut allowed_origins = std::env::var("FELTDB_ALLOWED_ORIGINS")
             .ok()
@@ -8164,6 +8173,23 @@ impl Config {
                     }
                     sync_interval = Duration::from_millis(milliseconds);
                 }
+                "--max-in-flight" => {
+                    max_in_flight = value()?
+                        .parse()
+                        .map_err(|_| "invalid --max-in-flight".to_string())?;
+                    if max_in_flight == 0 {
+                        return Err("--max-in-flight must be at least 1".to_string());
+                    }
+                }
+                "--request-deadline-ms" => {
+                    let milliseconds: u64 = value()?
+                        .parse()
+                        .map_err(|_| "invalid --request-deadline-ms".to_string())?;
+                    if milliseconds < 100 {
+                        return Err("--request-deadline-ms must be at least 100".to_string());
+                    }
+                    request_deadline = Duration::from_millis(milliseconds);
+                }
                 "--audit" => audit = Some(PathBuf::from(value()?)),
                 "--allow-origin" => allowed_origins.push(
                     value()?
@@ -8203,6 +8229,8 @@ impl Config {
             peers,
             peer_token_env,
             sync_interval,
+            max_in_flight,
+            request_deadline,
             audit,
             allowed_origins,
             worker_enabled,
@@ -8289,6 +8317,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     if manage_workloads()? {
         return Ok(());
     }
+    if manage_state_triggers()? {
+        return Ok(());
+    }
     if manage_backup()? {
         return Ok(());
     }
@@ -8336,6 +8367,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let grant_store = GrantStore::load(config.data.with_extension("grants.json"))?;
     let sync_store = SyncStore::load(config.data.with_extension("sync.json"))?;
     let workload_store = WorkloadStore::load(config.data.with_extension("workloads.json"))?;
+    let state_trigger_store = StateTriggerStore::load(config.data.with_extension("triggers.json"))?;
     let mesh_store = WorkerMeshStore::load(config.data.with_extension("workers.json"))?;
     let content_store = ContentStore::new(config.data.with_extension("content"))?;
     let artifact_store = ArtifactStore::load(
@@ -8352,10 +8384,16 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let audit_handle = AuditLog::new(config.audit.clone());
     let state = AppState {
         started_at: Instant::now(),
+        admission: Arc::new(tokio::sync::Semaphore::new(config.max_in_flight)),
+        request_deadline: config.request_deadline,
         ids: Arc::new(AtomicU64::new(db.sequence()?)),
         db,
         namespace: Arc::from(config.namespace.clone()),
         auth_enabled: config.auth_enabled,
+        // Argon2 is deliberately memory-hard. Bound concurrent legacy/session
+        // verification so a burst waits asynchronously instead of exhausting
+        // a small production VM with one allocation per request.
+        authentication_workers: Arc::new(tokio::sync::Semaphore::new(2)),
         keys: KeyStore::load(&config.keys)?,
         metrics: Metrics::default(),
         cluster: ClusterStore::load(
@@ -8403,6 +8441,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         grants: Arc::new(std::sync::Mutex::new(grant_store)),
         sync: Arc::new(std::sync::Mutex::new(sync_store)),
         workloads: Arc::new(std::sync::Mutex::new(workload_store)),
+        state_triggers: Arc::new(std::sync::Mutex::new(state_trigger_store)),
         mesh: Arc::new(std::sync::Mutex::new(mesh_store)),
         readiness_probe: Arc::new(readiness_probe),
         bounded_query_cursors: Arc::new(std::sync::Mutex::new(HashMap::new())),
@@ -9057,6 +9096,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     let mut app = Router::new()
         .route("/health", get(health))
+        .route("/health/ready", get(health_ready))
         .route("/v1/health", get(health))
         .route("/v1/openapi.json", get(application_openapi))
         .route("/v1/auth/signup", axum::routing::post(auth_sign_up))
@@ -9080,6 +9120,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         )
         .merge(protected)
         .merge(query_execution_diagnostics_router())
+        .layer(middleware::from_fn_with_state(state.clone(), admit_request))
         .layer(middleware::from_fn(protocol_version))
         .layer(middleware::from_fn(attribute_request))
         .layer(middleware::from_fn_with_state(state.clone(), count_request))
@@ -9165,16 +9206,52 @@ async fn authenticate(
             .headers()
             .get(AUTHORIZATION)
             .and_then(|header| header.to_str().ok())
-            .and_then(|header| header.strip_prefix("Bearer "));
-        let machine = token.and_then(|token| state.keys.authenticate(token, &state.namespace));
+            .and_then(|header| header.strip_prefix("Bearer "))
+            .map(str::to_owned);
+        let keys = state.keys.clone();
+        let identities = state.identities.clone();
+        let namespace = state.namespace.clone();
+        let authentication_worker = match state.authentication_workers.clone().acquire_owned().await
+        {
+            Ok(permit) => permit,
+            Err(error) => {
+                tracing::error!(%error, "authentication worker limit closed");
+                return ApiError(
+                    StatusCode::SERVICE_UNAVAILABLE,
+                    "authentication temporarily unavailable".into(),
+                )
+                .into_response();
+            }
+        };
+        let (machine, actor) = match tokio::task::spawn_blocking(move || {
+            let _authentication_worker = authentication_worker;
+            let machine = token
+                .as_deref()
+                .and_then(|token| keys.authenticate(token, &namespace));
+            let actor = if machine.is_none() {
+                token
+                    .as_deref()
+                    .and_then(|token| identities.authenticate_session(token))
+            } else {
+                None
+            };
+            (machine, actor)
+        })
+        .await
+        {
+            Ok(authentication) => authentication,
+            Err(error) => {
+                tracing::error!(%error, "bearer authentication worker failed");
+                return ApiError(
+                    StatusCode::SERVICE_UNAVAILABLE,
+                    "authentication temporarily unavailable".into(),
+                )
+                .into_response();
+            }
+        };
         // A valid service key is already authoritative. Do not feed that
         // high-entropy bearer token through the Argon2-backed human-session
         // verifier as well; doing so blocks the request worker needlessly.
-        let actor = if machine.is_none() {
-            token.and_then(|token| state.identities.authenticate_session(token))
-        } else {
-            None
-        };
         let human = request
             .headers()
             .get(COOKIE)
@@ -9425,6 +9502,234 @@ async fn attribute_request(request: Request<axum::body::Body>, next: Next) -> Re
     next.run(request).await
 }
 
+/// The overall status, from the conditions that actually bear on it.
+///
+/// Storage participates. Before it did not, so a database that had discarded an
+/// incomplete final append reported exactly what a cleanly replayed one did —
+/// which is the difference between an operator learning their last write was
+/// lost and never finding out.
+fn overall_status(
+    membership_healthy: bool,
+    clock_healthy: bool,
+    storage: &feltdb::StorageHealth,
+) -> &'static str {
+    if membership_healthy && clock_healthy && storage.is_clean() {
+        "healthy"
+    } else {
+        "degraded"
+    }
+}
+
+#[cfg(test)]
+mod admission_tests {
+    use super::overloaded;
+    use axum::http::StatusCode;
+    use axum::response::IntoResponse;
+
+    /// An overload is refused with a status and a hint, not a hang.
+    ///
+    /// The managed incident produced client-side timeouts and upstream 502s
+    /// because the server had no inbound deadline and no admission bound: a
+    /// request that could not proceed simply waited. A caller cannot act on
+    /// that. It can act on this.
+    #[test]
+    fn an_overload_is_an_explicit_refusal_with_a_retry_hint() {
+        let response = overloaded("server is at its in-flight request limit").into_response();
+        assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
+        assert_eq!(
+            response
+                .headers()
+                .get(axum::http::header::RETRY_AFTER)
+                .and_then(|value| value.to_str().ok()),
+            Some("1"),
+            "a caller is told when to come back"
+        );
+    }
+
+    /// 503 rather than 500: the request was refused, not mishandled.
+    #[test]
+    fn an_overload_is_not_reported_as_a_server_fault() {
+        let response = overloaded("request exceeded the server deadline").into_response();
+        assert_ne!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
+        assert!(response.status().is_server_error());
+    }
+}
+
+#[cfg(test)]
+mod health_status_tests {
+    use super::overall_status;
+    use feltdb::StorageHealth;
+
+    const RECOVERED: StorageHealth = StorageHealth::RecoveredIncompleteWrite {
+        byte_offset: 128,
+        discarded_bytes: 40,
+    };
+
+    /// Storage participates in the overall status.
+    ///
+    /// It did not before: a database that had discarded an incomplete final
+    /// append reported exactly what a cleanly replayed one did, so the
+    /// distinction the durable-log work established never reached an operator.
+    #[test]
+    fn a_recovered_database_is_reported_degraded() {
+        assert_eq!(overall_status(true, true, &StorageHealth::Clean), "healthy");
+        assert_eq!(overall_status(true, true, &RECOVERED), "degraded");
+    }
+
+    /// And the conditions that already counted still do, so this narrows
+    /// nothing.
+    #[test]
+    fn membership_and_clock_still_count() {
+        assert_eq!(
+            overall_status(false, true, &StorageHealth::Clean),
+            "degraded"
+        );
+        assert_eq!(
+            overall_status(true, false, &StorageHealth::Clean),
+            "degraded"
+        );
+        assert_eq!(overall_status(false, false, &RECOVERED), "degraded");
+    }
+
+    /// The reported storage value names an observation rather than asserting
+    /// the property that is still unproven.
+    #[test]
+    fn the_storage_label_is_an_observation_not_a_promise() {
+        assert_eq!(StorageHealth::Clean.label(), "clean");
+        assert_eq!(RECOVERED.label(), "recovered-incomplete-write");
+        for label in [StorageHealth::Clean.label(), RECOVERED.label()] {
+            assert_ne!(label, "durable", "health must not claim durability");
+        }
+    }
+}
+
+/// Shed load at the door, and abandon work the caller is no longer waiting for.
+///
+/// Two failures this replaces, both observed in production:
+///
+/// - **No inbound deadline.** A slow operation ran until the *client* gave up,
+///   and then kept running, so the load that caused the stall outlived every
+///   request that reported it.
+/// - **No admission bound.** Requests queued on the database's single lock
+///   until an upstream proxy returned 502. Concurrency past the bound buys no
+///   throughput — every read and write serializes on that lock — so all it
+///   decides is whether an overload is refused quickly or hidden.
+///
+/// Both now produce an explicit `503` with `Retry-After`, which is a signal a
+/// caller can act on rather than a timeout it has to interpret.
+async fn admit_request(
+    State(state): State<AppState>,
+    request: axum::extract::Request,
+    next: Next,
+) -> axum::response::Response {
+    let Ok(_permit) = state.admission.clone().try_acquire_owned() else {
+        state.metrics.request_shed();
+        return overloaded("server is at its in-flight request limit");
+    };
+
+    match tokio::time::timeout(state.request_deadline, next.run(request)).await {
+        Ok(response) => response,
+        Err(_) => {
+            state.metrics.request_deadline_exceeded();
+            overloaded("request exceeded the server deadline")
+        }
+    }
+}
+
+/// A refusal a caller can act on: an explicit status, a stable code, and a hint.
+fn overloaded(detail: &str) -> axum::response::Response {
+    let mut response = (
+        StatusCode::SERVICE_UNAVAILABLE,
+        Json(json!({
+            "error": detail,
+            "code": "SERVICE_OVERLOADED",
+        })),
+    )
+        .into_response();
+    response.headers_mut().insert(
+        axum::http::header::RETRY_AFTER,
+        axum::http::HeaderValue::from_static("1"),
+    );
+    response
+}
+
+/// Readiness: whether this server can actually serve a scoped operation.
+///
+/// `/health` answers "is this process alive". It reads cluster state and the
+/// lease clock and never touches storage, so it cannot fail for a database
+/// reason — which is exactly how a deployment reported `200 healthy` while
+/// every scoped read timed out.
+///
+/// This endpoint takes the database lock and performs a real bounded read, off
+/// the runtime and under the same deadline a request gets. **A server whose
+/// scoped operations are stalled reports `503` here.**
+async fn health_ready(State(state): State<AppState>) -> axum::response::Response {
+    let probe = state.clone();
+    let started = Instant::now();
+    let outcome = tokio::time::timeout(
+        state.request_deadline,
+        tokio::task::spawn_blocking(move || {
+            // A real scoped operation: acquire the state lock and read through
+            // it. Bounded, so a large database does not make readiness slow.
+            let health = probe.db.health();
+            let _ = probe
+                .db
+                .list_collection_page("_flow_capabilities", None, 1)?;
+            Ok::<_, feltdb::FlowError>(health)
+        }),
+    )
+    .await;
+
+    let waited_ms = started.elapsed().as_millis() as u64;
+    match outcome {
+        Ok(Ok(Ok(health))) => {
+            let ready = health.is_nominal();
+            let body = Json(json!({
+                "ready": ready,
+                "storage": health.storage.label(),
+                "durable_format": health.durable_format.to_string(),
+                "probe_ms": waited_ms,
+            }));
+            if ready {
+                (StatusCode::OK, body).into_response()
+            } else {
+                // Usable, but not in the condition a clean open produces.
+                (StatusCode::SERVICE_UNAVAILABLE, body).into_response()
+            }
+        }
+        Ok(Ok(Err(error))) => (
+            StatusCode::SERVICE_UNAVAILABLE,
+            Json(json!({
+                "ready": false,
+                "error": error.to_string(),
+                "code": "STORAGE_UNAVAILABLE",
+                "probe_ms": waited_ms,
+            })),
+        )
+            .into_response(),
+        Ok(Err(error)) => (
+            StatusCode::SERVICE_UNAVAILABLE,
+            Json(json!({
+                "ready": false,
+                "error": error.to_string(),
+                "code": "STORAGE_PROBE_FAILED",
+                "probe_ms": waited_ms,
+            })),
+        )
+            .into_response(),
+        Err(_) => (
+            StatusCode::SERVICE_UNAVAILABLE,
+            Json(json!({
+                "ready": false,
+                "error": "storage probe exceeded the server deadline",
+                "code": "STORAGE_STALLED",
+                "probe_ms": waited_ms,
+            })),
+        )
+            .into_response(),
+    }
+}
+
 async fn health(State(state): State<AppState>) -> Json<HealthResponse<'static>> {
     let membership = match state.cluster.proposal().map(|value| value.phase) {
         Some(ProposalPhase::Preparing | ProposalPhase::Prepared) => "recovering",
@@ -9432,16 +9737,16 @@ async fn health(State(state): State<AppState>) -> Json<HealthResponse<'static>>
         _ => "healthy",
     };
     let clock = state.lease_clock.is_healthy();
+    // Observed, not asserted. An incompatible or corrupt database never opens,
+    // so a running server has already established its format; what remains to
+    // report is whether replay discarded anything.
+    let database = state.db.health();
     Json(HealthResponse {
-        status: if membership == "healthy" && clock {
-            "healthy"
-        } else {
-            "degraded"
-        },
+        status: overall_status(membership == "healthy", clock, &database.storage),
         version: env!("CARGO_PKG_VERSION"),
         git_commit: option_env!("FELTDB_GIT_COMMIT").unwrap_or("unknown"),
         runtime: "self-hosted",
-        storage: "durable",
+        storage: database.storage.label(),
         fabric: "healthy",
         execution: if state.worker_enabled {
             "autonomous"
@@ -9482,22 +9787,33 @@ async fn managed_diagnostics(State(state): State<AppState>) -> Json<ManagedDiagn
 }
 
 async fn readiness(State(state): State<AppState>) -> Result<Json<Value>, ApiError> {
-    if let Some(parent) = state.readiness_probe.parent() {
-        std::fs::create_dir_all(parent)
+    let readiness_probe = state.readiness_probe.clone();
+    let db = state.db.clone();
+    let (state_version, storage) = tokio::task::spawn_blocking(move || {
+        if let Some(parent) = readiness_probe.parent() {
+            std::fs::create_dir_all(parent)
+                .map_err(|error| ApiError(StatusCode::SERVICE_UNAVAILABLE, error.to_string()))?;
+        }
+        let mut probe = OpenOptions::new()
+            .create(true)
+            .write(true)
+            .truncate(true)
+            .open(readiness_probe.as_ref())
             .map_err(|error| ApiError(StatusCode::SERVICE_UNAVAILABLE, error.to_string()))?;
-    }
-    let mut probe = OpenOptions::new()
-        .create(true)
-        .write(true)
-        .truncate(true)
-        .open(state.readiness_probe.as_ref())
-        .map_err(|error| ApiError(StatusCode::SERVICE_UNAVAILABLE, error.to_string()))?;
-    probe
-        .write_all(b"feltdb-ready-v1\n")
-        .and_then(|_| probe.sync_all())
-        .map_err(|error| ApiError(StatusCode::SERVICE_UNAVAILABLE, error.to_string()))?;
-    let state_version = state.db.sequence()?;
-    let storage = state.db.instance_id()?;
+        probe
+            .write_all(b"feltdb-ready-v1\n")
+            .and_then(|_| probe.sync_all())
+            .map_err(|error| ApiError(StatusCode::SERVICE_UNAVAILABLE, error.to_string()))?;
+        Ok::<_, ApiError>((db.sequence()?, db.instance_id()?))
+    })
+    .await
+    .map_err(|error| {
+        tracing::error!(%error, "readiness worker failed");
+        ApiError(
+            StatusCode::SERVICE_UNAVAILABLE,
+            "readiness check temporarily unavailable".into(),
+        )
+    })??;
     Ok(Json(json!({
         "status": "ready",
         "authority": "rust",
@@ -9566,10 +9882,15 @@ fn manage_keys() -> Result<bool, Box<dyn std::error::Error>> {
                     .map(str::to_string)
                     .collect()
             };
-            let key = store.create(
+            // `--worker <id>` issues a credential that authenticates as that
+            // one worker rather than as a service. It is a narrowing: the
+            // worker lifecycle routes accept it only for its own worker id, and
+            // it cannot satisfy a check that wants a human or a service.
+            let key = store.create_for_worker(
                 name,
                 values("--scope", "state:read,state:write,events:read"),
                 values("--namespace", "default"),
+                option("--worker"),
             )?;
             println!(
                 "API key created (id: {}). This secret will not be shown again:\n{}",
@@ -9584,7 +9905,7 @@ fn manage_keys() -> Result<bool, Box<dyn std::error::Error>> {
             }
             println!("API key revoked: {id}");
         }
-        _ => return Err("usage: feltdb-server keys <create|list|revoke> [--keys path] [--name name] [--scope scopes] [--namespace names] [--id key-id]".into()),
+        _ => return Err("usage: feltdb-server keys <create|list|revoke> [--keys path] [--name name] [--scope scopes] [--namespace names] [--worker worker-id] [--id key-id]".into()),
     }
     Ok(true)
 }
@@ -9682,6 +10003,61 @@ fn manage_workloads() -> Result<bool, Box<dyn std::error::Error>> {
     Ok(true)
 }
 
+fn manage_state_triggers() -> Result<bool, Box<dyn std::error::Error>> {
+    let arguments: Vec<String> = std::env::args().collect();
+    if arguments.get(1).map(String::as_str) != Some("trigger") {
+        return Ok(false);
+    }
+    let command = arguments.get(2).map(String::as_str).unwrap_or("list");
+    let option = |name: &str| {
+        arguments
+            .iter()
+            .position(|value| value == name)
+            .and_then(|index| arguments.get(index + 1))
+            .cloned()
+    };
+    let data = PathBuf::from(option("--data").unwrap_or_else(|| "./data/feltdb.log".into()));
+    let trigger_path = PathBuf::from(option("--store").unwrap_or_else(|| {
+        data.with_extension("triggers.json")
+            .to_string_lossy()
+            .into_owned()
+    }));
+    let mut triggers = StateTriggerStore::load(trigger_path)?;
+    match command {
+        "define" => {
+            let input = option("--input").ok_or("trigger define requires --input trigger.json")?;
+            let trigger: feltdb::StateTrigger = serde_json::from_slice(&std::fs::read(input)?)?;
+            triggers.define(trigger)?;
+            println!("{}", serde_json::to_string_pretty(&triggers.list())?);
+        }
+        "list" => println!("{}", serde_json::to_string_pretty(&triggers.list())?),
+        "status" => println!(
+            "{}",
+            serde_json::to_string_pretty(&json!({
+                "triggers": triggers.triggers.len(),
+                "cursors": triggers.cursor_versions(),
+            }))?
+        ),
+        "evaluate" => {
+            let db = FeltDb::open(&data)?;
+            let workload_path = PathBuf::from(
+                option("--workload-store")
+                    .unwrap_or_else(|| data.with_extension("workloads.json").to_string_lossy().into_owned()),
+            );
+            let mut workloads = WorkloadStore::load(workload_path)?;
+            let produced = triggers.evaluate(&db, &mut workloads, unix_seconds_i64())?;
+            println!("{}", serde_json::to_string_pretty(&produced)?);
+        }
+        _ => {
+            return Err(
+                "usage: feltdb-server trigger <define|list|status|evaluate> [--data path] [--store path] [--workload-store path] [--input trigger.json]"
+                    .into(),
+            )
+        }
+    }
+    Ok(true)
+}
+
 fn manage_backup() -> Result<bool, Box<dyn std::error::Error>> {
     let arguments: Vec<String> = std::env::args().collect();
     if arguments.get(1).map(String::as_str) != Some("backup") {
@@ -10006,9 +10382,25 @@ fn start_peer_sessions(
                     }
                 }
             }
-            match state.db.compact_operation_log(&peers) {
-                Ok(removed) => state.metrics.operations_compacted(removed as u64),
-                Err(error) => tracing::warn!(%error, "operation compaction failed"),
+            // Policy-driven, and off the runtime. Compaction rewrites the log
+            // while holding the lock that serializes every read and write, so
+            // running it on every tick stopped the world on every tick — and
+            // running it inline blocked a runtime worker while it did.
+            let compactor = state.clone();
+            let compaction_peers = peers.clone();
+            match tokio::task::spawn_blocking(move || {
+                compactor.db.maybe_compact_operation_log(&compaction_peers)
+            })
+            .await
+            {
+                Ok(Ok(outcome)) => {
+                    state.metrics.operations_compacted(outcome.pruned() as u64);
+                    if outcome.rewrote_log() {
+                        tracing::debug!(pruned = outcome.pruned(), "durable log rewritten");
+                    }
+                }
+                Ok(Err(error)) => tracing::warn!(%error, "operation compaction failed"),
+                Err(error) => tracing::warn!(%error, "operation compaction task failed"),
             }
             tokio::time::sleep(interval).await;
         }
@@ -10103,6 +10495,197 @@ async fn run_worker_pass(state: &AppState) -> Result<(), ApiError> {
         }
         execute_agent_run(state, &row.value, &worker).await?;
     }
+
+    evaluate_state_triggers(state)?;
+    execute_eligible_workloads(state, &worker)?;
+    Ok(())
+}
+
+/// How long an autonomous claim holds its lease.
+///
+/// Long enough that an ordinary capability program finishes inside it, short
+/// enough that a worker dying mid-execution returns the work quickly. The lease
+/// is the only thing that makes worker death recoverable, so this is a real
+/// bound rather than a formality.
+const WORKLOAD_LEASE_MS: u64 = 30_000;
+
+fn state_trigger_store(
+    state: &AppState,
+) -> Result<std::sync::MutexGuard<'_, feltdb::StateTriggerStore>, ApiError> {
+    state.state_triggers.lock().map_err(|_| {
+        ApiError(
+            StatusCode::INTERNAL_SERVER_ERROR,
+            "state trigger store unavailable".into(),
+        )
+    })
+}
+
+fn state_trigger_error(error: feltdb::StateTriggerError) -> ApiError {
+    let status = match error.code.as_str() {
+        "TRIGGER_INVALID" | "TRIGGER_UNAUTHORIZED" => StatusCode::BAD_REQUEST,
+        "WORKLOAD_UNAUTHORIZED" => StatusCode::FORBIDDEN,
+        "TRIGGER_STATE_REPLAY_FAILED" | "TRIGGER_STORAGE_FAILURE" | "WORKLOAD_STORAGE_FAILURE" => {
+            StatusCode::SERVICE_UNAVAILABLE
+        }
+        _ => StatusCode::CONFLICT,
+    };
+    ApiError(
+        status,
+        serde_json::to_string(&error).unwrap_or(error.message),
+    )
+}
+
+fn evaluate_state_triggers(state: &AppState) -> Result<(), ApiError> {
+    let mut triggers = state_trigger_store(state)?;
+    if triggers.triggers.is_empty() {
+        return Ok(());
+    }
+    let mut workloads = workload_store(state)?;
+    triggers
+        .evaluate(&state.db, &mut workloads, unix_seconds_i64())
+        .map_err(state_trigger_error)?;
+    Ok(())
+}
+
+/// The durable workload store, or a service error if its lock is poisoned.
+fn workload_store(
+    state: &AppState,
+) -> Result<std::sync::MutexGuard<'_, feltdb::workload::WorkloadStore>, ApiError> {
+    state.workloads.lock().map_err(|_| {
+        ApiError(
+            StatusCode::INTERNAL_SERVER_ERROR,
+            "workload store unavailable".into(),
+        )
+    })
+}
+
+/// The capabilities this node can actually run.
+///
+/// This is what the worker presents to [`WorkloadStore::claim`], which requires
+/// the workload's `capability_snapshot` to be a subset of it. That check is
+/// therefore load-bearing rather than ceremonial: a workload whose declared
+/// capabilities this node cannot execute is never claimed by it, and stays
+/// `Ready` for a node that can.
+fn worker_capabilities(state: &AppState) -> Result<BTreeSet<String>, ApiError> {
+    let mut capabilities: BTreeSet<String> = ["search", "identity"]
+        .into_iter()
+        .map(str::to_string)
+        .collect();
+    for row in state.db.list_collection("_flow_capabilities")? {
+        if let Some((_, name)) = row.key.split_once(':') {
+            capabilities.insert(name.to_string());
+        }
+    }
+    Ok(capabilities)
+}
+
+/// Claim and execute the durable workloads this node is eligible to run.
+///
+/// This is the composition that makes `WorkloadStore` an execution primitive
+/// rather than a work data model. It adds no queue and no worker system: the
+/// scan is the pass that already runs, the claim is the existing lease and
+/// fencing, the execution is the existing capability interpreter, and the
+/// transitions are the existing state machine.
+///
+/// Temporal eligibility needs no special handling here — `Workload::is_eligible_at`
+/// filters candidates and `WorkloadStore::claim` refuses anything early, so
+/// `not_before` and retry backoff gate this path by construction.
+///
+/// The store lock is released before execution. A capability program reads
+/// application state through the same database, so holding the workload lock
+/// across it would make every other workload operation wait on unrelated work.
+fn execute_eligible_workloads(state: &AppState, worker: &str) -> Result<(), ApiError> {
+    let capabilities = worker_capabilities(state)?;
+    let now = unix_seconds_i64();
+    let candidates: Vec<(String, String, Value)> = {
+        let store = workload_store(state)?;
+        store
+            .workloads
+            .values()
+            // Selection, not enforcement. `WorkloadStore::claim` independently
+            // refuses work that is early or beyond this node's capabilities, and
+            // that is where the guarantee lives; filtering here only avoids
+            // attempting a claim, every pass, that is known to be refused.
+            .filter(|workload| {
+                workload.state == feltdb::workload::WorkloadState::Ready
+                    && workload.is_eligible_at(now)
+                    && workload.capability_snapshot.is_subset(&capabilities)
+            })
+            .map(|workload| {
+                (
+                    workload.workload_id.clone(),
+                    workload.definition_id.clone(),
+                    workload.input.clone(),
+                )
+            })
+            .collect()
+    };
+
+    for (id, definition, input) in candidates {
+        // Unlike the filter above, this one is enforcement: nothing downstream
+        // knows whether `definition_id` names something runnable. Claiming a
+        // workload only to dead-letter it would consume work another node could
+        // have executed, so an unrunnable definition is left untouched.
+        if !is_executable_capability(state, &definition)? {
+            continue;
+        }
+        let claim = match workload_store(state)?.claim(
+            &id,
+            worker,
+            &capabilities,
+            unix_seconds_i64(),
+            WORKLOAD_LEASE_MS,
+        ) {
+            Ok(claim) => claim,
+            // Another worker took it, or eligibility moved between the scan and
+            // the claim. Both are ordinary outcomes of concurrent workers, not
+            // failures of this pass.
+            Err(_) => continue,
+        };
+        workload_store(state)?
+            .start(
+                &id,
+                &claim.claim_id,
+                worker,
+                claim.fencing_token,
+                unix_seconds_i64(),
+            )
+            .map_err(workload_error)?;
+
+        match execute_named_capability(state, &definition, &input) {
+            Ok(output) => {
+                workload_store(state)?
+                    .complete(
+                        &id,
+                        &claim.claim_id,
+                        worker,
+                        claim.fencing_token,
+                        &format!("result_{id}_{}", claim.fencing_token),
+                        output,
+                        vec![],
+                        1,
+                        unix_seconds_i64(),
+                    )
+                    .map_err(workload_error)?;
+            }
+            Err(error) => {
+                // A capability program that fails does so deterministically, so
+                // this failure kind is deliberately outside the default
+                // `retry_on` set: retrying it would burn attempts to reach the
+                // same dead letter. Only the message is carried, because the
+                // store refuses failure text that looks like a secret.
+                let _ = workload_store(state)?.fail(
+                    &id,
+                    &claim.claim_id,
+                    worker,
+                    claim.fencing_token,
+                    "execution_error",
+                    &format!("capability {definition} failed with status {}", error.0),
+                    unix_seconds_i64(),
+                );
+            }
+        }
+    }
     Ok(())
 }
 
@@ -10138,10 +10721,10 @@ fn execute_builtin_capability(
             Ok(Value::Array(
                 state
                     .db
-                    .list_collection(collection)?
+                    .query_collection(collection, Some(limit), |row| {
+                        row.value.to_string().to_lowercase().contains(&needle)
+                    })?
                     .into_iter()
-                    .filter(|row| row.value.to_string().to_lowercase().contains(&needle))
-                    .take(limit)
                     .map(|row| row.value)
                     .collect(),
             ))
@@ -11182,11 +11765,11 @@ async fn get_provenance(
     Path((collection, id)): Path<(String, String)>,
 ) -> Result<Json<Value>, ApiError> {
     let key = record_key(&collection, &id)?;
+    // A keyed lookup. This used to materialize the whole collection — cloning
+    // every row under the global state lock — to find one record.
     let row = state
         .db
-        .list_collection(&collection)?
-        .into_iter()
-        .find(|row| row.key == key)
+        .get_collection_record(&collection, &key)?
         .ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "record not found".to_string()))?;
     let operation = row.operation.as_ref();
     Ok(Json(json!({
@@ -11216,12 +11799,14 @@ async fn search_collection(
         ));
     }
     let needle = request.query.to_lowercase();
+    // Borrows each row and stops at the limit, rather than cloning the whole
+    // collection and filtering afterwards. Only matches are cloned.
     let records = state
         .db
-        .list_collection(&collection)?
+        .query_collection(&collection, Some(request.limit), |row| {
+            row.value.to_string().to_lowercase().contains(&needle)
+        })?
         .into_iter()
-        .filter(|row| row.value.to_string().to_lowercase().contains(&needle))
-        .take(request.limit)
         .map(record_response)
         .collect();
     Ok(Json(records))
@@ -13008,6 +13593,69 @@ async fn shutdown_signal() {
     tokio::select! { _ = ctrl_c => {}, _ = terminate => {} }
 }
 
+#[cfg(test)]
+mod worker_identity_tests {
+    use super::grant_subject;
+    use feltdb::authorization::Subject as GrantSubject;
+    use feltdb_server::auth::Principal;
+
+    fn principal(subject_type: &str, identity: Option<&str>) -> Principal {
+        Principal {
+            key_id: "key-1".into(),
+            scopes: vec!["*".into()],
+            subject_type: subject_type.into(),
+            identity_id: identity.map(str::to_string),
+            session_id: None,
+        }
+    }
+
+    /// The workload worker lifecycle compares the authenticated subject against
+    /// the `worker_id` in the request, so which subject each principal maps to
+    /// *is* the security boundary. Nothing but the authenticated principal
+    /// participates in this mapping.
+    #[test]
+    fn only_a_worker_principal_maps_to_a_worker_subject() {
+        assert_eq!(
+            grant_subject(&principal("worker", Some("worker-x"))),
+            GrantSubject::Worker("worker-x".into()),
+        );
+        // Every other principal maps somewhere a worker check can never accept,
+        // no matter what identity it carries or what scopes it holds.
+        for (subject_type, expected) in [
+            ("human", GrantSubject::Human("worker-x".into())),
+            ("user", GrantSubject::Human("worker-x".into())),
+            ("service", GrantSubject::Service("worker-x".into())),
+            ("api_key", GrantSubject::Service("worker-x".into())),
+            ("agent", GrantSubject::Agent("worker-x".into())),
+        ] {
+            let subject = grant_subject(&principal(subject_type, Some("worker-x")));
+            assert_eq!(subject, expected, "{subject_type} mapped unexpectedly");
+            assert_ne!(
+                subject,
+                GrantSubject::Worker("worker-x".into()),
+                "{subject_type} must never satisfy a worker check"
+            );
+        }
+    }
+
+    /// A worker principal is one worker, not any worker.
+    #[test]
+    fn a_worker_subject_is_bound_to_its_own_identity() {
+        let subject = grant_subject(&principal("worker", Some("worker-x")));
+        assert_ne!(subject, GrantSubject::Worker("worker-y".into()));
+    }
+
+    /// Without an identity the principal falls back to its key id, so a worker
+    /// credential missing its worker id cannot silently become a wildcard.
+    #[test]
+    fn a_worker_principal_without_an_identity_falls_back_to_its_key_id() {
+        assert_eq!(
+            grant_subject(&principal("worker", None)),
+            GrantSubject::Worker("key-1".into()),
+        );
+    }
+}
+
 #[cfg(test)]
 mod authority_gate_tests {
     use super::{api_error_body, authorize_transaction_collections, state_authorization};