@feltdb/core 0.13.0 → 0.15.0

package/dist/create/server-source/crates/feltdb-server/src/main.rs

@@ -88,9 +88,11 @@ use feltdb_server::{
     cluster::{ClusterConfig, ClusterStore, MembershipProposal, ProposalPhase},
     connections::{ConnectionProvider, ConnectionStatus, ConnectionStore, SecretReference},
     content::ContentStore,
+    control_authorization::{self, PrincipalClass},
+    console_accounts::{self, ConsoleAccountStore},
     control_plane,
     identity::IdentityStore,
-    key_management::{create_key, list_keys, revoke_key},
+    key_management::{create_key, list_keys, revoke_key, rotate_key},
     key_provider::provider_from_environment,
     leases::LeaseStore,
     metrics::Metrics,
@@ -200,11 +202,13 @@ struct HealthResponse<'a> {
     public_contract: &'a str,
     capability_fingerprint: String,
     runtime: &'a str,
+    /// Replay outcome of the durable log at open (`clean` or
+    /// `recovered-incomplete-write`). Liveness only: it is not a probe of the
+    /// volume now and not a durability claim — see `/ready` and `/v1/status`.
     storage: &'a str,
-    fabric: &'a str,
+    /// Configuration, not a probe: whether this process runs the autonomous
+    /// worker.
     execution: &'a str,
-    workflows: &'a str,
-    agents: &'a str,
     membership: &'a str,
 }
 
@@ -1466,6 +1470,10 @@ struct WorkloadFailRequest {
     fence: WorkloadFenceRequest,
     kind: String,
     error: String,
+    /// `none` (failed before any external effect) or `unknown` (an effect may
+    /// have happened). Omitted: the policy decides by `kind`.
+    #[serde(default)]
+    effect: Option<String>,
 }
 #[derive(Deserialize)]
 struct WorkloadContextRequest {
@@ -1496,6 +1504,45 @@ struct WorkerReconcileRequest {
     reconciliation: WorkerReconciliation,
     grant: feltdb::authorization::SignedGrant,
 }
+/// Whether this node commits Durable Work.
+///
+/// Durable Work's guarantees rest on one serialization point for each
+/// workload: the guarded commit of one store. Peers replicate by causal
+/// order with last-writer-wins for concurrent writes, so two nodes that both
+/// committed work would each pass their own guard and silently diverge. A node
+/// without peers is therefore always the authority (the single-node default);
+/// in a peered cluster exactly one node, started with `--work-authority`,
+/// commits Durable Work, and every other node serves it read-only from its
+/// replicated store. A node that learns a peer is also the authority stops
+/// being one.
+fn is_work_authority(state: &AppState) -> bool {
+    if state
+        .work_authority_conflict
+        .load(std::sync::atomic::Ordering::SeqCst)
+    {
+        return false;
+    }
+    state.work_authority || state.cluster.get().peers.is_empty()
+}
+/// Refuse a Durable Work write on a node that is not the authority.
+fn require_work_authority(state: &AppState) -> Result<(), ApiError> {
+    if is_work_authority(state) {
+        return Ok(());
+    }
+    let conflict = state
+        .work_authority_conflict
+        .load(std::sync::atomic::Ordering::SeqCst);
+    Err(ApiError::structured(
+        StatusCode::MISDIRECTED_REQUEST,
+        json!({"code": "WORKLOAD_NOT_AUTHORITY",
+            "message": if conflict {
+                "more than one node is configured as the Durable Work authority; Durable Work writes are refused until that is corrected"
+            } else {
+                "this node replicates Durable Work read-only; send the request to the node started with --work-authority"
+            },
+            "details": {"reason": if conflict { "authority_conflict" } else { "replica" }, "retryable": false}}),
+    ))
+}
 fn workload_error(error: feltdb::workload::WorkloadError) -> ApiError {
     let status = match error.code.as_str() {
         "WORKLOAD_NOT_FOUND" => StatusCode::NOT_FOUND,
@@ -1505,6 +1552,8 @@ fn workload_error(error: feltdb::workload::WorkloadError) -> ApiError {
         | "WORKLOAD_NOT_YET_ELIGIBLE"
         | "WORKLOAD_ATTEMPTS_EXHAUSTED"
         | "WORKLOAD_RETRY_EXHAUSTED"
+        | "WORKLOAD_RESULT_CONFLICT"
+        | "WORKLOAD_TIMED_OUT"
         | "WORKLOAD_CONFLICT" => StatusCode::CONFLICT,
         "WORKLOAD_UNAUTHORIZED" | "WORKLOAD_CAPABILITY_DENIED" => StatusCode::FORBIDDEN,
         "WORKLOAD_STORAGE_FAILURE" => StatusCode::SERVICE_UNAVAILABLE,
@@ -1597,6 +1646,7 @@ async fn create_workload(
     Extension(principal): Extension<Principal>,
     Json(mut input): Json<CreateWorkload>,
 ) -> Result<(StatusCode, HeaderMap, Json<Value>), ApiError> {
+    require_work_authority(&state)?;
     let subject = grant_subject(&principal);
     input.requested_by = subject.clone();
     input.session_id = principal.session_id.clone();
@@ -1624,10 +1674,18 @@ async fn create_workload(
             if error.code == "IDEMPOTENCY_KEY_REUSED" {
                 // AppPort's contract for a reused key (§13.1): CONFLICT, with
                 // the reason, not retryable.
+                // `fields` names the execution-affecting request fields that
+                // differ from the admitted workload.
+                let fields = error
+                    .details
+                    .as_ref()
+                    .and_then(|details| details.get("fields"))
+                    .cloned()
+                    .unwrap_or_else(|| json!([]));
                 ApiError::structured(
                     StatusCode::CONFLICT,
                     json!({"code": "CONFLICT", "message": error.message,
-                        "details": {"reason": "IDEMPOTENCY_KEY_REUSED", "retryable": false}}),
+                        "details": {"reason": "IDEMPOTENCY_KEY_REUSED", "retryable": false, "fields": fields}}),
                 )
             } else {
                 workload_error(error)
@@ -1668,7 +1726,8 @@ async fn list_workloads(
                 "workload store unavailable".into(),
             )
         })?
-        .list(&tenant, &target.application_id);
+        .fresh_list(&tenant, &target.application_id)
+        .map_err(workload_error)?;
     Ok(Json(json!({"workloads":values,"grant_id":target.grant_id})))
 }
 async fn get_workload(
@@ -1685,7 +1744,7 @@ async fn get_workload(
                 "workload store unavailable".into(),
             )
         })?
-        .get(&id)
+        .fresh(&id)
         .map_err(workload_error)?
         .clone();
     workload_read_scope(&state, &principal, &workload.application_id)?;
@@ -1715,11 +1774,12 @@ async fn claim_workload(
     Path(id): Path<String>,
     Json(input): Json<WorkloadClaimRequest>,
 ) -> Result<Json<Value>, ApiError> {
+    require_work_authority(&state)?;
     let workload = state
         .workloads
         .lock()
         .unwrap()
-        .get(&id)
+        .fresh(&id)
         .map_err(workload_error)?
         .clone();
     let subject = grant_subject(&principal);
@@ -1773,7 +1833,7 @@ fn authorize_fence(
         .workloads
         .lock()
         .unwrap()
-        .get(id)
+        .fresh(id)
         .map_err(workload_error)?
         .clone();
     let subject = grant_subject(principal);
@@ -1800,6 +1860,7 @@ async fn start_workload(
     Path(id): Path<String>,
     Json(input): Json<WorkloadFenceRequest>,
 ) -> Result<StatusCode, ApiError> {
+    require_work_authority(&state)?;
     authorize_fence(&state, &principal, &id, &input, "workload:execute")?;
     state
         .workloads
@@ -1821,6 +1882,7 @@ async fn heartbeat_workload(
     Path(id): Path<String>,
     Json(input): Json<WorkloadHeartbeatRequest>,
 ) -> Result<Json<Value>, ApiError> {
+    require_work_authority(&state)?;
     authorize_fence(&state, &principal, &id, &input.fence, "workload:execute")?;
     let claim = state
         .workloads
@@ -1843,6 +1905,7 @@ async fn checkpoint_workload(
     Path(id): Path<String>,
     Json(input): Json<WorkloadCheckpointRequest>,
 ) -> Result<Json<Value>, ApiError> {
+    require_work_authority(&state)?;
     authorize_fence(&state, &principal, &id, &input.fence, "workload:execute")?;
     let checkpoint = state
         .workloads
@@ -1865,6 +1928,7 @@ async fn complete_workload(
     Path(id): Path<String>,
     Json(input): Json<WorkloadCompleteRequest>,
 ) -> Result<Json<Value>, ApiError> {
+    require_work_authority(&state)?;
     authorize_fence(&state, &principal, &id, &input.fence, "workload:execute")?;
     let result = state
         .workloads
@@ -1891,18 +1955,20 @@ async fn fail_workload(
     Path(id): Path<String>,
     Json(input): Json<WorkloadFailRequest>,
 ) -> Result<Json<Value>, ApiError> {
+    require_work_authority(&state)?;
     authorize_fence(&state, &principal, &id, &input.fence, "workload:execute")?;
     let status = state
         .workloads
         .lock()
         .unwrap()
-        .fail(
+        .fail_with_effect(
             &id,
             &input.fence.claim_id,
             &input.fence.worker_id,
             input.fence.fencing_token,
             &input.kind,
             &input.error,
+            input.effect.as_deref(),
             unix_seconds_i64(),
         )
         .map_err(workload_error)?;
@@ -1915,6 +1981,7 @@ async fn confirm_workload_cancelled(
     Path(id): Path<String>,
     Json(input): Json<WorkloadFenceRequest>,
 ) -> Result<StatusCode, ApiError> {
+    require_work_authority(&state)?;
     authorize_fence(&state, &principal, &id, &input, "workload:execute")?;
     state
         .workloads
@@ -1936,6 +2003,7 @@ async fn workload_execution_context(
     Path(id): Path<String>,
     Json(input): Json<WorkloadContextRequest>,
 ) -> Result<Json<Value>, ApiError> {
+    require_work_authority(&state)?;
     authorize_fence(&state, &principal, &id, &input.fence, "workload:execute")?;
     let context = state
         .workloads
@@ -1958,11 +2026,12 @@ async fn cancel_workload(
     Path(id): Path<String>,
     Json(input): Json<WorkloadControlRequest>,
 ) -> Result<Json<Value>, ApiError> {
+    require_work_authority(&state)?;
     let workload = state
         .workloads
         .lock()
         .unwrap()
-        .get(&id)
+        .fresh(&id)
         .map_err(workload_error)?
         .clone();
     workload_decision(
@@ -1988,11 +2057,12 @@ async fn retry_workload(
     Path(id): Path<String>,
     Json(input): Json<WorkloadControlRequest>,
 ) -> Result<StatusCode, ApiError> {
+    require_work_authority(&state)?;
     let workload = state
         .workloads
         .lock()
         .unwrap()
-        .get(&id)
+        .fresh(&id)
         .map_err(workload_error)?
         .clone();
     workload_decision(
@@ -2021,7 +2091,7 @@ async fn workload_history(
         .workloads
         .lock()
         .unwrap()
-        .get(&id)
+        .fresh(&id)
         .map_err(workload_error)?
         .clone();
     workload_read_scope(&state, &principal, &workload.application_id)?;
@@ -2038,7 +2108,7 @@ async fn workload_result(
         .workloads
         .lock()
         .unwrap()
-        .get(&id)
+        .fresh(&id)
         .map_err(workload_error)?
         .clone();
     workload_read_scope(&state, &principal, &workload.application_id)?;
@@ -2223,7 +2293,8 @@ async fn reconcile_worker(
 ) -> Result<Json<Value>, ApiError> {
     authorize_worker_action(&state, &principal, &id, &input.grant, "worker:heartbeat")?;
     input.reconciliation.worker_id = id;
-    let workloads = state.workloads.lock().unwrap();
+    let mut workloads = state.workloads.lock().unwrap();
+    workloads.refresh().map_err(workload_error)?;
     let result = state
         .mesh
         .lock()
@@ -2314,7 +2385,7 @@ async fn eligible_workers(
         .workloads
         .lock()
         .unwrap()
-        .get(&id)
+        .fresh(&id)
         .map_err(workload_error)?
         .clone();
     workload_read_scope(&state, &principal, &workload.application_id)?;
@@ -6957,6 +7028,139 @@ async fn auth_sign_up(
         .map_err(control_error)?;
     Ok((StatusCode::CREATED, auth_response(identity, session, token)))
 }
+#[derive(Deserialize)]
+struct ConsolePasswordRequest {
+    email: String,
+    password: String,
+}
+#[derive(Deserialize)]
+struct ConsoleGithubRequest {
+    github_id: Value,
+    login: String,
+}
+
+fn console_account_error(error: console_accounts::AccountError) -> ApiError {
+    ApiError::structured(
+        StatusCode::from_u16(error.status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR),
+        json!({ "code": error.code, "message": error.message }),
+    )
+}
+
+/// Authenticate a website call to `/v1/console/accounts/*` and parse its body.
+fn console_request<T: serde::de::DeserializeOwned>(
+    state: &AppState,
+    method: &Method,
+    uri: &axum::http::Uri,
+    headers: &HeaderMap,
+    body: &[u8],
+) -> Result<T, ApiError> {
+    let Some(secret) = state.console_accounts.secret() else {
+        return Err(ApiError::structured(
+            StatusCode::SERVICE_UNAVAILABLE,
+            json!({
+                "code": "CONSOLE_ACCOUNTS_UNCONFIGURED",
+                "message": "console accounts require FELTDB_SESSION_SECRET (at least 32 bytes) on the control plane",
+            }),
+        ));
+    };
+    let header = |name: &str| headers.get(name).and_then(|value| value.to_str().ok());
+    let now = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_secs();
+    if let Err(error) = console_accounts::verify_console_request(
+        secret,
+        method.as_str(),
+        uri.path(),
+        header(console_accounts::TIMESTAMP_HEADER),
+        header(console_accounts::SIGNATURE_HEADER),
+        body,
+        now,
+    ) {
+        state.metrics.authentication_failure();
+        audit(state, "anonymous", "console.accounts", uri.path(), "denied", 401);
+        return Err(console_account_error(error));
+    }
+    serde_json::from_slice(body)
+        .map_err(|error| ApiError(StatusCode::BAD_REQUEST, format!("invalid request body: {error}")))
+}
+
+/// Argon2 work and the account file write run off the request worker.
+async fn console_blocking<F>(work: F) -> Result<Json<Value>, ApiError>
+where
+    F: FnOnce() -> Result<console_accounts::ResolvedAccount, console_accounts::AccountError>
+        + Send
+        + 'static,
+{
+    match tokio::task::spawn_blocking(work).await {
+        Ok(Ok(account)) => Ok(Json(json!({
+            "user_id": account.user_id,
+            "identity": account.identity,
+            "display_identity": account.display_identity,
+        }))),
+        Ok(Err(error)) => Err(console_account_error(error)),
+        Err(_) => Err(ApiError(
+            StatusCode::INTERNAL_SERVER_ERROR,
+            "console account operation failed".into(),
+        )),
+    }
+}
+
+async fn console_password_signup(
+    State(state): State<AppState>,
+    method: Method,
+    uri: axum::http::Uri,
+    headers: HeaderMap,
+    body: axum::body::Bytes,
+) -> Result<(StatusCode, Json<Value>), ApiError> {
+    let input: ConsolePasswordRequest = console_request(&state, &method, &uri, &headers, &body)?;
+    let created = console_blocking(move || {
+        state
+            .console_accounts
+            .password_signup(&state.tenancy, &input.email, &input.password)
+    })
+    .await?;
+    Ok((StatusCode::CREATED, created))
+}
+
+async fn console_password_signin(
+    State(state): State<AppState>,
+    method: Method,
+    uri: axum::http::Uri,
+    headers: HeaderMap,
+    body: axum::body::Bytes,
+) -> Result<Json<Value>, ApiError> {
+    let input: ConsolePasswordRequest = console_request(&state, &method, &uri, &headers, &body)?;
+    console_blocking(move || {
+        state
+            .console_accounts
+            .password_signin(&input.email, &input.password)
+    })
+    .await
+}
+
+async fn console_github_resolve(
+    State(state): State<AppState>,
+    method: Method,
+    uri: axum::http::Uri,
+    headers: HeaderMap,
+    body: axum::body::Bytes,
+) -> Result<Json<Value>, ApiError> {
+    let input: ConsoleGithubRequest = console_request(&state, &method, &uri, &headers, &body)?;
+    // GitHub user ids are numbers; accept the number or its decimal string.
+    let github_id = match &input.github_id {
+        Value::Number(number) if number.is_u64() => number.to_string(),
+        Value::String(text) => text.clone(),
+        _ => String::new(),
+    };
+    console_blocking(move || {
+        state
+            .console_accounts
+            .github_resolve(&state.tenancy, &github_id, &input.login)
+    })
+    .await
+}
+
 async fn auth_sign_in(
     State(state): State<AppState>,
     Json(input): Json<SignInRequest>,
@@ -7595,7 +7799,9 @@ fn state_contract_error(error: feltdb::state_contract::StateFailure) -> ApiError
         "PRECONDITION_FAILED" | "CONFLICT" | "IDEMPOTENCY_CONFLICT" => StatusCode::CONFLICT,
         "UNKNOWN_COLLECTION" => StatusCode::NOT_FOUND,
         "SEMANTIC_UNSUPPORTED" => StatusCode::NOT_IMPLEMENTED,
-        "STORAGE_FAILURE" | "STATE_UNAVAILABLE" => StatusCode::SERVICE_UNAVAILABLE,
+        "STORAGE_FAILURE" | "STATE_UNAVAILABLE" | "DURABILITY_FAILURE" => {
+            StatusCode::SERVICE_UNAVAILABLE
+        }
         _ => StatusCode::UNPROCESSABLE_ENTITY,
     };
     ApiError(
@@ -8232,6 +8438,10 @@ struct SyncPullResponse {
     operations: Vec<Operation>,
     versions: HashMap<String, u64>,
     catchup: CatchupStatus,
+    /// Whether the answering node is configured as the Durable Work authority
+    /// (`--work-authority`). Absent from older peers.
+    #[serde(default)]
+    durable_work_authority: bool,
 }
 
 #[derive(Serialize, Deserialize)]
@@ -8480,6 +8690,7 @@ struct Config {
     audit: PathBuf,
     allowed_origins: Vec<axum::http::HeaderValue>,
     worker_enabled: bool,
+    work_authority: bool,
     advertise_url: Option<String>,
     maximum_clock_drift_ms: u64,
 }
@@ -8528,6 +8739,7 @@ impl Config {
             .transpose()?
             .unwrap_or_default();
         let mut worker_enabled = std::env::var("FELTDB_WORKER_ENABLED").as_deref() == Ok("true");
+        let mut work_authority = std::env::var("FELTDB_WORK_AUTHORITY").as_deref() == Ok("true");
         let mut advertise_url = std::env::var("FELTDB_ADVERTISE_URL")
             .ok()
             .map(|value| value.trim_end_matches('/').to_string());
@@ -8580,6 +8792,7 @@ impl Config {
                         .map_err(|_| "invalid --allow-origin".to_string())?,
                 ),
                 "--worker" => worker_enabled = true,
+                "--work-authority" => work_authority = true,
                 "--advertise-url" => {
                     advertise_url = Some(value()?.trim_end_matches('/').to_string())
                 }
@@ -8594,7 +8807,7 @@ impl Config {
                     }
                 }
                 "--help" | "-h" => {
-                    println!("feltdb-server [--host 127.0.0.1] [--port 7700] [--advertise-url https://node.example] [--namespace default] [--data ./data/feltdb.log] [--auth] [--keys ./data/api-keys.json] [--audit ./data/feltdb.audit.log] [--allow-origin https://app.example] [--peer https://peer] [--peer-token-env FELTDB_PEER_TOKEN] [--sync-interval-ms 2000] [--worker]");
+                    println!("feltdb-server [--host 127.0.0.1] [--port 7700] [--advertise-url https://node.example] [--namespace default] [--data ./data/feltdb.log] [--auth] [--keys ./data/api-keys.json] [--audit ./data/feltdb.audit.log] [--allow-origin https://app.example] [--peer https://peer] [--peer-token-env FELTDB_PEER_TOKEN] [--sync-interval-ms 2000] [--worker] [--work-authority]");
                     std::process::exit(0);
                 }
                 _ => return Err(format!("unknown argument: {argument}")),
@@ -8617,6 +8830,7 @@ impl Config {
             audit,
             allowed_origins,
             worker_enabled,
+            work_authority,
             advertise_url,
             maximum_clock_drift_ms,
         })
@@ -8687,10 +8901,28 @@ fn validate_production_configuration(config: &Config) -> Result<(), Box<dyn std:
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
     tracing_subscriber::fmt()
         .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
+        // Every log line passes through credential redaction, whatever wrote it.
+        .with_writer(feltdb_server::redaction::RedactingStdout)
         .init();
+    if matches!(
+        std::env::args().nth(1).as_deref(),
+        Some("version" | "--version" | "-V")
+    ) {
+        // One line an operator script can print instead of a hard-coded banner.
+        println!(
+            "feltdb-server {} (release {}, commit {})",
+            env!("CARGO_PKG_VERSION"),
+            option_env!("FELTDB_RELEASE_ID").unwrap_or("unknown"),
+            option_env!("FELTDB_GIT_COMMIT").unwrap_or("unknown")
+        );
+        return Ok(());
+    }
     if manage_keys()? {
         return Ok(());
     }
+    if manage_console_accounts()? {
+        return Ok(());
+    }
     if manage_authorization()? {
         return Ok(());
     }
@@ -8827,6 +9059,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         lifecycle_lock: Arc::new(tokio::sync::Mutex::new(())),
         state_transaction_admission: Arc::new(tokio::sync::Semaphore::new(2)),
         worker_enabled: config.worker_enabled,
+        work_authority: config.work_authority,
+        work_authority_conflict: Arc::new(std::sync::atomic::AtomicBool::new(false)),
         leases: LeaseStore::load(config.data.with_extension("leases.json"))?,
         content: content_store,
         artifacts: artifact_store,
@@ -8848,6 +9082,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                 .into_bytes(),
             config.data.with_extension("revoked-sessions.json"),
         )?,
+        console_accounts: ConsoleAccountStore::load(
+            config.data.with_extension("console-accounts.json"),
+            console_accounts::configured_secret(std::env::var("FELTDB_SESSION_SECRET").ok()),
+        )?,
         principals: principal_authority,
         connections: ConnectionStore::load_with_provider(
             config.data.with_extension("connections.json"),
@@ -8936,6 +9174,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         .route("/api/tenants/{tenant_id}", get(get_tenant))
         .route("/api/keys", get(list_keys).post(create_key))
         .route("/api/keys/{id}", axum::routing::delete(revoke_key))
+        .route("/api/keys/{id}/rotate", axum::routing::post(rotate_key))
+        // Operator observability: authenticated, `operator:read`.
+        .route("/metrics", get(network_metrics))
+        .route("/v1/status", get(operator_status))
         .route(
             "/api/tenants/{tenant_id}/applications",
             get(list_applications).post(create_application_control),
@@ -9596,23 +9838,23 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         .route("/v1/openapi.json", get(application_openapi))
         .route("/v1/auth/signup", axum::routing::post(auth_sign_up))
         .route("/v1/auth/signin", axum::routing::post(auth_sign_in))
-        .route("/internal/intents/by-hash", get(internal_find_intent))
+        // Console accounts: callable only by the website, which signs each
+        // request with FELTDB_SESSION_SECRET (see console_accounts.rs).
         .route(
-            "/internal/intents",
-            axum::routing::post(internal_create_intent),
+            "/v1/console/accounts/password/signup",
+            axum::routing::post(console_password_signup),
         )
-        .route("/internal/intents/{id}", put(internal_replace_intent))
         .route(
-            "/internal/intent-events",
-            axum::routing::post(internal_create_intent_event),
+            "/v1/console/accounts/password/signin",
+            axum::routing::post(console_password_signin),
         )
-        .route("/ready", get(readiness))
-        .route("/runtime", get(runtime))
-        .route("/metrics", get(network_metrics))
         .route(
-            "/api/certification/fixtures/{application_id}",
-            axum::routing::delete(delete_certification_fixture),
+            "/v1/console/accounts/github/resolve",
+            axum::routing::post(console_github_resolve),
         )
+        .route("/ready", get(readiness))
+        .route("/runtime", get(runtime))
+        .merge(service_secret_routes())
         .merge(protected)
         .merge(query_execution_diagnostics_router())
         .layer(middleware::from_fn_with_state(state.clone(), admit_request))
@@ -9620,7 +9862,17 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         .layer(middleware::from_fn(attribute_request))
         .layer(middleware::from_fn_with_state(state.clone(), count_request))
         .layer(RequestBodyLimitLayer::new(1024 * 1024))
-        .layer(TraceLayer::new_for_http())
+        .layer(middleware::from_fn(redact_error_responses))
+        // Spans carry the path, never the query string, where clients put tokens.
+        .layer(TraceLayer::new_for_http().make_span_with(
+            |request: &Request<axum::body::Body>| {
+                tracing::debug_span!(
+                    "request",
+                    method = %request.method(),
+                    path = %request.uri().path(),
+                )
+            },
+        ))
         .with_state(state);
     if !config.allowed_origins.is_empty() {
         app = app.layer(
@@ -9664,6 +9916,32 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     Ok(())
 }
 
+/// Error bodies can quote client input (axum's JSON rejections echo the
+/// offending value); a credential sent by mistake must not come back out, or
+/// land in whatever logs the response. Success bodies are untouched: a key's
+/// secret is returned exactly once, at creation or rotation.
+async fn redact_error_responses(request: Request<axum::body::Body>, next: Next) -> Response {
+    let response = next.run(request).await;
+    if response.status().as_u16() < 400 {
+        return response;
+    }
+    let (parts, body) = response.into_parts();
+    let bytes = match axum::body::to_bytes(body, 1024 * 1024).await {
+        Ok(bytes) => bytes,
+        Err(_) => return Response::from_parts(parts, axum::body::Body::empty()),
+    };
+    let body = match std::str::from_utf8(&bytes) {
+        Ok(text) => match feltdb_server::redaction::redact(text) {
+            std::borrow::Cow::Owned(redacted) => axum::body::Body::from(redacted),
+            std::borrow::Cow::Borrowed(_) => axum::body::Body::from(bytes),
+        },
+        Err(_) => axum::body::Body::from(bytes),
+    };
+    let mut parts = parts;
+    parts.headers.remove(axum::http::header::CONTENT_LENGTH);
+    Response::from_parts(parts, body)
+}
+
 async fn count_request(
     State(state): State<AppState>,
     request: Request<axum::body::Body>,
@@ -9696,7 +9974,7 @@ async fn authenticate(
     // an answer to PR36's question — which part of the request does is.
     let authorization =
         feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::Authorization);
-    let principal = if state.auth_enabled {
+    let (principal, class) = if state.auth_enabled {
         let token = request
             .headers()
             .get(AUTHORIZATION)
@@ -9759,8 +10037,8 @@ async fn authenticate(
             })
             .and_then(|token| state.sessions.verify(token));
         match (machine, actor, human) {
-            (Some(principal), _, _) => principal,
-            (None, Some(session), _) => Principal {
+            (Some(principal), _, _) => (principal, PrincipalClass::Machine),
+            (None, Some(session), _) => (Principal {
                 key_id: session.subject_id,
                 scopes: vec![
                     "application:read".into(),
@@ -9775,24 +10053,55 @@ async fn authenticate(
                 subject_type: "human".into(),
                 identity_id: Some(session.identity_id),
                 session_id: Some(session.session_id),
-            },
+            }, PrincipalClass::EndUser),
             (None, None, Some(session)) => {
-                if state
-                    .tenancy
-                    .register_platform_user(&session.user_id, &session.identity)
-                    .is_err()
-                    || !state.tenancy.platform_user_active(&session.user_id)
+                // A signed cookie is not enough: its user id must belong to a
+                // console account (created through the verified account
+                // routes, or bound/claimed for a pre-0.15.0 user), and its
+                // identity must be that account's. Sessions the website
+                // minted before accounts existed are refused here.
+                let account = match state.console_accounts.account_for_user(&session.user_id) {
+                    Some(account) if account.display_identity == session.identity.trim().to_lowercase() => account,
+                    _ => {
+                        state.metrics.authentication_failure();
+                        audit(
+                            &state,
+                            &session.user_id,
+                            "authenticate",
+                            request.uri().path(),
+                            "denied",
+                            401,
+                        );
+                        return ApiError::structured(
+                            StatusCode::UNAUTHORIZED,
+                            json!({
+                                "code": "CONSOLE_ACCOUNT_REQUIRED",
+                                "message": "this console session is not bound to a console account; sign in again",
+                            }),
+                        )
+                        .into_response();
+                    }
+                };
+                if (!state.tenancy.platform_user_exists(&account.user_id)
+                    && state
+                        .tenancy
+                        .register_platform_user(&account.user_id, &account.display_identity)
+                        .is_err())
+                    || !state.tenancy.platform_user_active(&account.user_id)
                 {
                     return ApiError(StatusCode::FORBIDDEN, "platform user is suspended".into())
                         .into_response();
                 }
-                Principal {
-                    key_id: session.user_id,
-                    scopes: vec![],
-                    subject_type: "human".into(),
-                    identity_id: Some(session.identity),
-                    session_id: Some(session.id),
-                }
+                (
+                    Principal {
+                        key_id: account.user_id,
+                        scopes: vec![],
+                        subject_type: "human".into(),
+                        identity_id: Some(account.display_identity),
+                        session_id: Some(session.id),
+                    },
+                    PrincipalClass::ConsoleUser,
+                )
             }
             (None, None, None) => {
                 state.metrics.authentication_failure();
@@ -9812,17 +10121,43 @@ async fn authenticate(
             }
         }
     } else {
-        Principal {
-            key_id: "development".to_string(),
-            scopes: vec!["*".to_string()],
-            subject_type: "service".into(),
-            identity_id: None,
-            session_id: None,
-        }
+        (
+            Principal {
+                key_id: "development".to_string(),
+                scopes: vec!["*".to_string()],
+                subject_type: "service".into(),
+                identity_id: None,
+                session_id: None,
+            },
+            PrincipalClass::Development,
+        )
     };
 
     let path = request.uri().path();
-    let required = if path == "/events" {
+    // The control plane (`/api/*`) is decided per route, before any handler:
+    // machine keys need the route's capability, console users are held to
+    // tenant/application membership by the handler, and application end-user
+    // sessions are refused. There is no path family without a decision.
+    let control = control_authorization::control_plane_requirement(request.method(), path);
+    if let Some(requirement) = control {
+        if let Err(denial) = control_authorization::decide(class, &principal.scopes, requirement) {
+            state.metrics.authentication_failure();
+            let action = match requirement {
+                control_authorization::Requirement::Capability(scope) => scope,
+                _ => "control-plane",
+            };
+            audit(&state, &principal.key_id, action, path, "denied", 403);
+            return ApiError(StatusCode::FORBIDDEN, denial.body()).into_response();
+        }
+    }
+    let required = if let Some(requirement) = control {
+        match requirement {
+            control_authorization::Requirement::Capability(scope) => scope,
+            _ => "session:self",
+        }
+    } else if path == "/metrics" || path == "/v1/status" {
+        control_authorization::OPERATOR_READ
+    } else if path == "/events" {
         "events:read"
     } else if path == "/v1/state/diff"
         || path == "/v1/state/conflicts"
@@ -9882,6 +10217,24 @@ async fn authenticate(
     } else {
         "state:write"
     };
+    // An end-user session belongs to one application (self-serve sign-up
+    // registers it there). Its state scopes are meaningful only on the
+    // application API, which checks that membership per request. The legacy
+    // routes (`/collections`, `/transactions`, `/cas`, `/events`, ...) are
+    // scoped to the whole server namespace, so a session there would read and
+    // write every application's records.
+    if class == PrincipalClass::EndUser && !path.starts_with("/v1/") {
+        state.metrics.authentication_failure();
+        audit(&state, &principal.key_id, required, path, "denied", 403);
+        let denial = control_authorization::Denial::new(
+            control_authorization::END_USER_SESSION_FORBIDDEN,
+            format!(
+                "end-user sessions can only use the application API (/v1/...) of their own application; {path} is a namespace-wide route — use an application-scoped service key on the server side"
+            ),
+        )
+        .with_scope(required);
+        return ApiError(StatusCode::FORBIDDEN, denial.body()).into_response();
+    }
     let subject = if principal.subject_type == "human" {
         Subject::User(principal.key_id.clone())
     } else {
@@ -9899,16 +10252,26 @@ async fn authenticate(
     let human_contract = principal.subject_type == "human"
         && path.starts_with("/v1/applications/")
         && path.split('/').any(|segment| segment == "contract");
+    // Two non-control routes are decided by who is asking rather than by a
+    // scope, and are listed here rather than exempted: a user's own contract
+    // drafts (keyed by that user) and the authenticated diagnostics summary.
     let authenticated_diagnostics = path == "/v1/diagnostics";
-    if !path.starts_with("/api/")
-        && !human_contract
-        && !authenticated_diagnostics
-        && !decision.allowed
-    {
+    let decided_by_route = control.is_some() || human_contract || authenticated_diagnostics;
+    if !decided_by_route && !decision.allowed {
         state.metrics.authentication_failure();
         audit(&state, &principal.key_id, required, path, "denied", 403);
-        return ApiError(StatusCode::FORBIDDEN, format!("missing scope: {required}"))
-            .into_response();
+        let denial = if class == PrincipalClass::EndUser
+            && (path == "/metrics" || path == "/v1/status" || path.starts_with("/admin/"))
+        {
+            control_authorization::Denial::new(
+                control_authorization::END_USER_SESSION_FORBIDDEN,
+                format!("missing scope {required} — end-user sessions cannot use operator routes; use an operator key"),
+            )
+            .with_scope(required)
+        } else {
+            control_authorization::Denial::missing_scope(required)
+        };
+        return ApiError(StatusCode::FORBIDDEN, denial.body()).into_response();
     }
     if path == "/events" {
         state.metrics.event_connection();
@@ -9927,6 +10290,7 @@ async fn authenticate(
 
     request.extensions_mut().insert(principal.clone());
     request.extensions_mut().insert(authenticated.clone());
+    request.extensions_mut().insert(class);
 
     let principal_id = request
         .extensions()
@@ -10272,14 +10636,14 @@ async fn health(State(state): State<AppState>) -> Json<HealthResponse<'static>>
         capability_fingerprint: control_plane::public_contract_capability_fingerprint(),
         runtime: "self-hosted",
         storage: database.storage.label(),
-        fabric: "healthy",
+        // `fabric`, `workflows` and `agents` were constant "healthy" /
+        // "available" here whatever their state (GAP-10). Nothing checks them,
+        // so nothing reports them.
         execution: if state.worker_enabled {
             "autonomous"
         } else {
             "state-coordinated"
         },
-        workflows: "available",
-        agents: "available",
         membership: if clock {
             membership
         } else {
@@ -10417,10 +10781,19 @@ fn manage_keys() -> Result<bool, Box<dyn std::error::Error>> {
                 values("--namespace", "default"),
                 option("--worker"),
             )?;
-            println!(
-                "API key created (id: {}). This secret will not be shown again:\n{}",
-                key.id, key.secret
-            );
+            // `--secret-file <path>` writes the secret to an owner-only file
+            // and prints only the id, so a bootstrap script never has to put a
+            // credential on stdout (and from there into platform logs).
+            match option("--secret-file") {
+                Some(file) => {
+                    write_secret_file(std::path::Path::new(&file), &key.secret)?;
+                    println!("API key created (id: {}). Secret written to {file} (mode 0600).", key.id);
+                }
+                None => println!(
+                    "API key created (id: {}). This secret will not be shown again:\n{}",
+                    key.id, key.secret
+                ),
+            }
         }
         "list" => println!("{}", serde_json::to_string_pretty(&store.list()?)?),
         "revoke" => {
@@ -10430,11 +10803,93 @@ fn manage_keys() -> Result<bool, Box<dyn std::error::Error>> {
             }
             println!("API key revoked: {id}");
         }
-        _ => return Err("usage: feltdb-server keys <create|list|revoke> [--keys path] [--name name] [--scope scopes] [--namespace names] [--worker worker-id] [--id key-id]".into()),
+        _ => return Err("usage: feltdb-server keys <create|list|revoke> [--keys path] [--name name] [--scope scopes] [--namespace names] [--worker worker-id] [--secret-file path] [--id key-id]".into()),
+    }
+    Ok(true)
+}
+
+/// `feltdb-server console-accounts <claim|list> --data <state.log> ...`
+///
+/// `claim --email <email> --password-file <file>` binds a console password
+/// account to the existing pre-0.15.0 platform user registered under that
+/// e-mail, so its owner keeps their tenants. Before 0.15.0 the website did not
+/// verify passwords, so such a user cannot be claimed by self-service sign-up;
+/// the operator verifies the person out of band and runs this. The password
+/// is read from a file (first line) so it never appears in the process list.
+fn manage_console_accounts() -> Result<bool, Box<dyn std::error::Error>> {
+    let arguments: Vec<String> = std::env::args().collect();
+    if arguments.get(1).map(String::as_str) != Some("console-accounts") {
+        return Ok(false);
+    }
+    let usage = "usage: feltdb-server console-accounts <claim|list> --data <state.log path> [--email email --password-file path]";
+    let option = |name: &str| {
+        arguments
+            .iter()
+            .position(|argument| argument == name)
+            .and_then(|index| arguments.get(index + 1))
+            .cloned()
+    };
+    let data = PathBuf::from(
+        option("--data").unwrap_or_else(|| "./data/feltdb.log".to_string()),
+    );
+    let accounts = ConsoleAccountStore::load(data.with_extension("console-accounts.json"), None)?;
+    let tenancy = TenancyStore::load(data.with_extension("platform.json"))?;
+    match arguments.get(2).map(String::as_str).unwrap_or("") {
+        "claim" => {
+            let email = option("--email").ok_or("console-accounts claim requires --email")?;
+            let file = option("--password-file")
+                .ok_or("console-accounts claim requires --password-file")?;
+            let password = std::fs::read_to_string(&file)?
+                .lines()
+                .next()
+                .unwrap_or_default()
+                .to_string();
+            let claimed = accounts
+                .claim_password(&tenancy, &email, &password)
+                .map_err(|error| format!("{}: {}", error.code, error.message))?;
+            println!(
+                "Console account claimed: {} is bound to existing platform user {}.",
+                claimed.identity, claimed.user_id
+            );
+        }
+        "list" => {
+            let listed: Vec<Value> = accounts
+                .list()
+                .into_iter()
+                .map(|account| {
+                    json!({
+                        "identity": account.identity,
+                        "user_id": account.user_id,
+                        "provider": account.provider,
+                        "display_identity": account.display_identity,
+                        "origin": account.origin,
+                        "created_at": account.created_at,
+                    })
+                })
+                .collect();
+            println!("{}", serde_json::to_string_pretty(&listed)?);
+        }
+        _ => return Err(usage.into()),
     }
     Ok(true)
 }
 
+/// Create `path` owner-only (0600) and write `secret` to it; refuses to
+/// overwrite, so a second bootstrap never silently replaces a delivered secret.
+fn write_secret_file(path: &std::path::Path, secret: &str) -> std::io::Result<()> {
+    let mut options = OpenOptions::new();
+    options.write(true).create_new(true);
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::OpenOptionsExt;
+        options.mode(0o600);
+    }
+    let mut file = options.open(path)?;
+    file.write_all(secret.as_bytes())?;
+    file.write_all(b"\n")?;
+    file.sync_all()
+}
+
 fn manage_authorization() -> Result<bool, Box<dyn std::error::Error>> {
     let arguments: Vec<String> = std::env::args().collect();
     let family = arguments.get(1).map(String::as_str).unwrap_or("");
@@ -10961,6 +11416,37 @@ fn manage_application_runtime() -> Result<bool, Box<dyn std::error::Error>> {
     Ok(true)
 }
 
+/// Where an online backup bundle may be written.
+///
+/// With `FELTDB_BACKUP_DIR` set (the managed deployments set it), the request
+/// names a bundle, not a path: a single relative component placed inside that
+/// directory. Without it the server is self-hosted and the operator's path is
+/// used as given, as before.
+fn backup_output_path(
+    requested: &str,
+    backup_dir: Option<std::ffi::OsString>,
+) -> Result<PathBuf, ApiError> {
+    let Some(root) = backup_dir.filter(|value| !value.is_empty()) else {
+        return Ok(PathBuf::from(requested));
+    };
+    let name = std::path::Path::new(requested);
+    let mut components = name.components();
+    let single = matches!(
+        (components.next(), components.next()),
+        (Some(std::path::Component::Normal(_)), None)
+    );
+    if !single {
+        return Err(ApiError(
+            StatusCode::BAD_REQUEST,
+            format!(
+                "backup output must be a bundle name, not a path: this server writes bundles inside FELTDB_BACKUP_DIR ({}); send e.g. {{\"output\":\"backup-2026-10-28\"}}",
+                std::path::Path::new(&root).display()
+            ),
+        ));
+    }
+    Ok(PathBuf::from(root).join(name))
+}
+
 async fn create_online_backup(
     State(state): State<AppState>,
     Json(request): Json<OnlineBackupRequest>,
@@ -10990,7 +11476,7 @@ async fn create_online_backup(
         })
         .cloned()
         .collect();
-    let output = PathBuf::from(request.output);
+    let output = backup_output_path(&request.output, std::env::var_os("FELTDB_BACKUP_DIR"))?;
     let source = feltdb_server::backup::BackupSource {
         data: state
             .backup_artifacts
@@ -11002,14 +11488,22 @@ async fn create_online_backup(
         server_version: Some(env!("CARGO_PKG_VERSION").into()),
         durable_format: Some(feltdb::DURABLE_FORMAT_VERSION),
     };
+    let records = operator_record_path(&state, BACKUP_RECORDS);
     let manifest = tokio::task::spawn_blocking(move || {
-        let _guard = guard;
-        feltdb_server::backup::create_bundle_with_source(
-            &output,
-            &artifacts,
-            &[("database.snapshot.json".to_string(), snapshot_bytes)],
-            source,
-        )
+        let manifest = {
+            let _guard = guard;
+            feltdb_server::backup::create_bundle_with_source(
+                &output,
+                &artifacts,
+                &[("database.snapshot.json".to_string(), snapshot_bytes)],
+                source,
+            )?
+        };
+        // Verified after the barrier is released: restoring into scratch and
+        // opening the result is what turns "a bundle exists" into "a bundle
+        // restores", and the record says which of the two is true.
+        record_online_backup(&records, &output, &manifest);
+        Ok::<_, std::io::Error>(manifest)
     })
     .await
     .map_err(|error| {
@@ -11033,6 +11527,236 @@ async fn create_online_backup(
 /// finding was that losing security evidence was invisible, and a signal an
 /// operator has to know to go and look for is not much better than no signal.
 /// It rides with the metrics they already read.
+const BACKUP_RECORDS: &str = "backups.json";
+
+/// Operator records live beside the durable log (`<data>.<name>`), on the
+/// same volume, like the readiness probe they are derived from.
+fn operator_record_path(state: &AppState, name: &str) -> PathBuf {
+    state.readiness_probe.with_extension(name)
+}
+
+fn read_records(path: &std::path::Path) -> Vec<Value> {
+    std::fs::read(path)
+        .ok()
+        .and_then(|bytes| serde_json::from_slice::<Vec<Value>>(&bytes).ok())
+        .unwrap_or_default()
+}
+
+/// Append one record; bounded so the file cannot grow without limit.
+fn append_record(path: &std::path::Path, record: Value) -> std::io::Result<()> {
+    let mut records = read_records(path);
+    records.push(record);
+    let excess = records.len().saturating_sub(100);
+    records.drain(..excess);
+    let temporary = path.with_extension("json.tmp");
+    let mut file = std::fs::File::create(&temporary)?;
+    file.write_all(&serde_json::to_vec_pretty(&records).map_err(std::io::Error::other)?)?;
+    file.sync_all()?;
+    std::fs::rename(&temporary, path)
+}
+
+fn record_online_backup(
+    records: &std::path::Path,
+    output: &std::path::Path,
+    manifest: &feltdb_server::backup::BackupManifest,
+) {
+    let started = Instant::now();
+    let restore = verify_restorable(output).map_err(|error| error.to_string());
+    let record = json!({
+        "kind": "online_backup",
+        "bundle_id": feltdb_server::backup::backup_id(output).ok(),
+        "output": output.display().to_string(),
+        "created_ms": manifest.created_ms,
+        "files": manifest.entries.len(),
+        "verified": restore.is_ok(),
+        "restore_verification": match &restore {
+            Ok(detail) => json!({
+                "method": "bundle verified, restored into a scratch directory on this host and opened",
+                "verified_at_ms": unix_millis(),
+                "duration_ms": started.elapsed().as_millis() as u64,
+                "detail": detail,
+            }),
+            Err(error) => json!({ "method": "scratch restore", "error": error }),
+        },
+    });
+    if let Err(error) = append_record(records, record) {
+        tracing::error!(%error, "failed to record online backup");
+    }
+}
+
+fn unix_millis() -> u64 {
+    std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_millis() as u64
+}
+
+/// The routes authenticated by a shared service secret rather than an API
+/// key. They are routed only when deliberately configured, so an authority
+/// that is not part of a managed Intent Engine deployment or a certification
+/// run has no such surface at all.
+fn service_secret_routes() -> Router<AppState> {
+    let mut routes = Router::new();
+    let intent_secret = std::env::var("FELTDB_INTENT_SERVICE_SECRET").unwrap_or_default();
+    if intent_secret.len() >= 32 {
+        routes = routes
+            .route("/internal/intents/by-hash", get(internal_find_intent))
+            .route(
+                "/internal/intents",
+                axum::routing::post(internal_create_intent),
+            )
+            .route("/internal/intents/{id}", put(internal_replace_intent))
+            .route(
+                "/internal/intent-events",
+                axum::routing::post(internal_create_intent_event),
+            );
+    } else if !intent_secret.is_empty() {
+        tracing::warn!("FELTDB_INTENT_SERVICE_SECRET is shorter than 32 bytes; /internal/intents* is not routed");
+    }
+    let certification = matches!(
+        std::env::var("FELTDB_CERTIFICATION_MODE").as_deref(),
+        Ok("1" | "true")
+    );
+    if certification {
+        tracing::warn!("FELTDB_CERTIFICATION_MODE is set: certification fixture deletion is routed");
+        routes = routes.route(
+            "/api/certification/fixtures/{application_id}",
+            axum::routing::delete(delete_certification_fixture),
+        );
+    }
+    routes
+}
+
+/// A real write → fsync → read-back probe of the storage volume.
+fn storage_probe(path: &std::path::Path) -> Value {
+    let started = Instant::now();
+    let token = format!("feltdb-status-probe-{}\n", uuid::Uuid::new_v4().simple());
+    let mut wrote = false;
+    let mut synced = false;
+    let result = (|| -> std::io::Result<bool> {
+        if let Some(parent) = path.parent() {
+            std::fs::create_dir_all(parent)?;
+        }
+        let mut file = OpenOptions::new()
+            .create(true)
+            .write(true)
+            .truncate(true)
+            .open(path)?;
+        file.write_all(token.as_bytes())?;
+        wrote = true;
+        file.sync_all()?;
+        synced = true;
+        Ok(std::fs::read(path)? == token.as_bytes())
+    })();
+    let read_back = matches!(result, Ok(true));
+    json!({
+        "ok": read_back,
+        "write": wrote,
+        "fsync": synced,
+        "read_back": read_back,
+        "checked_at_ms": unix_millis(),
+        "latency_us": started.elapsed().as_micros() as u64,
+        "error": result.err().map(|error| error.to_string()),
+    })
+}
+
+/// `GET /v1/status` — the operator's view, needs `operator:read`.
+///
+/// Every field is observed by this request or read from a record this server
+/// wrote when the event happened; nothing is asserted. Availability (this
+/// process answers and its volume accepts a synced write) is reported apart
+/// from verified durability (a backup of this state was restored and opened).
+async fn operator_status(State(state): State<AppState>) -> Result<Json<Value>, ApiError> {
+    let probe_path = operator_record_path(&state, "status-probe");
+    let backups_path = operator_record_path(&state, BACKUP_RECORDS);
+    let (probe, backups) = tokio::task::spawn_blocking(move || {
+        (storage_probe(&probe_path), read_records(&backups_path))
+    })
+    .await
+    .map_err(|error| {
+        tracing::error!(%error, "status worker failed");
+        ApiError(StatusCode::SERVICE_UNAVAILABLE, "status check temporarily unavailable".into())
+    })?;
+    let database = state.db.health();
+    let last_backup = backups
+        .iter()
+        .rev()
+        .find(|record| record["kind"] == "online_backup")
+        .map(|record| {
+            json!({
+                "bundle_id": record["bundle_id"],
+                "created_ms": record["created_ms"],
+                "output": record["output"],
+                "verified": record["verified"],
+            })
+        });
+    let last_verified_restore = backups
+        .iter()
+        .rev()
+        .find(|record| record["verified"] == true)
+        .map(|record| {
+            json!({
+                "bundle_id": record["bundle_id"],
+                "verified_at_ms": record["restore_verification"]["verified_at_ms"],
+                "method": record["restore_verification"]["method"],
+            })
+        });
+    let clock = state.lease_clock.is_healthy();
+    let membership_healthy = !matches!(
+        state.cluster.proposal().map(|value| value.phase),
+        Some(ProposalPhase::Preparing | ProposalPhase::Prepared | ProposalPhase::Aborted)
+    );
+    let available = probe["ok"] == true;
+    Ok(Json(json!({
+        "status": if available && overall_status(membership_healthy, clock, &database.storage) == "healthy" { "available" } else { "degraded" },
+        "release": {
+            "version": env!("CARGO_PKG_VERSION"),
+            "release_id": option_env!("FELTDB_RELEASE_ID").unwrap_or("unknown"),
+            "git_commit": option_env!("FELTDB_GIT_COMMIT").unwrap_or("unknown"),
+            "build_timestamp": option_env!("FELTDB_BUILD_TIMESTAMP").unwrap_or("unknown"),
+            "deployment_id": std::env::var("FELTDB_DEPLOYMENT_ID").or_else(|_| std::env::var("FLY_IMAGE_REF")).ok(),
+            "promoted_revision": std::env::var("FELTDB_PROMOTED_REVISION").ok(),
+        },
+        "capabilities": {
+            "public_contract": "feltdb.public.substrate@1.0",
+            "capability_fingerprint": control_plane::public_contract_capability_fingerprint(),
+            "protocol_version": PROTOCOL_VERSION,
+            "authentication": state.auth_enabled,
+            "autonomous_workers": state.worker_enabled,
+            "configured_peers": state.cluster.get().peers.len(),
+            "durable_work_authority": is_work_authority(&state),
+        },
+        "availability": {
+            "serving": true,
+            "storage_writable": available,
+            "clock": if clock { "healthy" } else { "unhealthy" },
+            "membership": if membership_healthy { "healthy" } else { "recovering-or-attention-required" },
+        },
+        "storage": {
+            "replay": database.storage.label(),
+            "durable_format": database.durable_format.to_string(),
+            "probe": probe,
+        },
+        "backups": {
+            "last": last_backup,
+            "recorded": backups.len(),
+        },
+        "restore": {
+            "last_verified": last_verified_restore,
+            "status": if last_verified_restore.is_some() { "verified" } else { "never_verified" },
+            "off_host": "not observable by this server",
+        },
+        "durability": {
+            "verified": last_verified_restore.is_some(),
+            "basis": if last_verified_restore.is_some() {
+                "a backup of this authority was restored into scratch on this host and opened; off-host recovery and power-loss durability are not proven by this endpoint"
+            } else {
+                "no backup of this authority has been restored and verified; availability is not durability"
+            },
+        },
+    })))
+}
+
 async fn network_metrics(State(state): State<AppState>) -> Json<Value> {
     let mut body = serde_json::to_value(state.metrics.snapshot()).unwrap_or_else(|_| json!({}));
     if let Some(fields) = body.as_object_mut() {
@@ -11075,6 +11799,7 @@ async fn sync_pull(
         operations,
         versions: authority_position,
         catchup,
+        durable_work_authority: state.work_authority,
     }))
 }
 
@@ -11239,10 +11964,13 @@ fn start_workload_recovery(state: &AppState) {
     let state = state.clone();
     tokio::spawn(async move {
         loop {
-            if state.cluster.get().is_member {
+            if state.cluster.get().is_member && is_work_authority(&state) {
                 let outcome = match (state.mesh.lock(), state.workloads.lock()) {
+                    // Deadlines first: work past one is timed out under its
+                    // own decision, not recovered as a lost worker.
                     (Ok(mut mesh), Ok(mut workloads)) => mesh
-                        .recover_expired_work(unix_seconds_i64(), &mut workloads)
+                        .enforce_deadlines(unix_seconds_i64(), &mut workloads)
+                        .and_then(|_| mesh.recover_expired_work(unix_seconds_i64(), &mut workloads))
                         .map(|_| ())
                         .map_err(|error| error.message),
                     _ => Err("worker mesh or workload store unavailable".to_string()),
@@ -11325,8 +12053,12 @@ async fn run_worker_pass(state: &AppState) -> Result<(), ApiError> {
         }
     }
 
-    evaluate_state_triggers(state)?;
-    execute_eligible_workloads(state, &worker)?;
+    // Triggers admit work and the pass claims it: Durable Work writes, so
+    // only on the authority.
+    if is_work_authority(state) {
+        evaluate_state_triggers(state)?;
+        execute_eligible_workloads(state, &worker)?;
+    }
     Ok(())
 }
 
@@ -12141,6 +12873,17 @@ async fn sync_with_peer(
         .json::<SyncPullResponse>()
         .await?;
 
+    // Two nodes configured as the Durable Work authority would each commit
+    // work the other cannot see in time; both refuse until one is restarted
+    // without `--work-authority`.
+    if state.work_authority && pull.durable_work_authority {
+        if !state
+            .work_authority_conflict
+            .swap(true, std::sync::atomic::Ordering::SeqCst)
+        {
+            tracing::error!(%peer, "two nodes are configured as the Durable Work authority; refusing Durable Work writes");
+        }
+    }
     let mut incoming = pull.operations;
     incoming.sort_by(|left, right| {
         left.instance_id
@@ -14977,7 +15720,7 @@ async fn commit_transaction(
                         "committed": false,
                         "code": "PRECONDITION_FAILED",
                         "error": failure.to_string(),
-                        "failure": *failure,
+                        "failure": failure.public_json(&feltdb::ReadLayout::Direct),
                     }),
                 ),
                 FlowError::CapabilityError(message) if message.starts_with("INVALID_READ_SET:") => {
@@ -14994,7 +15737,29 @@ async fn commit_transaction(
                         }),
                     )
                 }
-                other => ApiError(StatusCode::CONFLICT, other.to_string()),
+                // The commit could not be made durable. It is neither a lost
+                // race nor a success, and answering 409 here once let a caller
+                // read a storage failure as "someone else won". The outcome is
+                // to be treated as unknown and reconciled by retrying with the
+                // same transaction id.
+                storage @ (FlowError::Io(_) | FlowError::Serde(_)) => ApiError::structured(
+                    StatusCode::SERVICE_UNAVAILABLE,
+                    json!({
+                        "committed": "unknown",
+                        "code": "DURABILITY_FAILURE",
+                        "message": storage.to_string(),
+                        "transaction_id": request.transaction_id,
+                    }),
+                ),
+                other => ApiError::structured(
+                    StatusCode::INTERNAL_SERVER_ERROR,
+                    json!({
+                        "committed": "unknown",
+                        "code": "EXECUTION_FAILED",
+                        "message": other.to_string(),
+                        "transaction_id": request.transaction_id,
+                    }),
+                ),
             }
         })?;
     let operation_ids = state
@@ -16437,7 +17202,8 @@ async fn control_plane_agent(
                 "workload store unavailable".into(),
             )
         })?
-        .list(&tenant, &application_id)
+        .fresh_list(&tenant, &application_id)
+        .map_err(workload_error)?
         .into_iter()
         .filter(|workload| {
             serde_json::to_value(&workload.requested_by)
@@ -17032,3 +17798,32 @@ mod authority_gate_tests {
         }
     }
 }
+
+#[cfg(test)]
+mod backup_output_tests {
+    use super::backup_output_path;
+    use std::path::PathBuf;
+
+    #[test]
+    fn a_confined_server_writes_named_bundles_inside_its_backup_directory() {
+        let root = Some("/data/backups".into());
+        assert_eq!(
+            backup_output_path("nightly-1", root.clone()).unwrap(),
+            PathBuf::from("/data/backups/nightly-1")
+        );
+        for refused in ["/data/state.log", "../state.log", "a/b", "..", ".", "", "/"] {
+            assert!(
+                backup_output_path(refused, root.clone()).is_err(),
+                "{refused:?} must be refused"
+            );
+        }
+    }
+
+    #[test]
+    fn a_self_hosted_server_uses_the_operator_path() {
+        assert_eq!(
+            backup_output_path("/srv/backups/x", None).unwrap(),
+            PathBuf::from("/srv/backups/x")
+        );
+    }
+}