@feltdb/core 0.12.2 → 0.15.0

package/dist/create/server-source/crates/feltdb-server/src/main.rs

@@ -67,6 +67,7 @@ use feltdb::{
     PeerAdvertisement, PeerId, RecordPrecondition, StateRevision, StateTriggerStore, StoredRow,
 };
 use feltdb_server::{
+    agent_execution::{self, CompletionCriteria, RunFailure, RunKind, RunPrincipal},
     app_state::{AppState, BoundedQueryCursor},
     application_contract::{ApplicationContractStore, ContractPatch},
     artifacts::{ArtifactKind, ArtifactLifecycle, ArtifactMetadata, ArtifactStore, ProducerRef},
@@ -87,9 +88,11 @@ use feltdb_server::{
     cluster::{ClusterConfig, ClusterStore, MembershipProposal, ProposalPhase},
     connections::{ConnectionProvider, ConnectionStatus, ConnectionStore, SecretReference},
     content::ContentStore,
+    control_authorization::{self, PrincipalClass},
+    console_accounts::{self, ConsoleAccountStore},
     control_plane,
     identity::IdentityStore,
-    key_management::{create_key, list_keys, revoke_key},
+    key_management::{create_key, list_keys, revoke_key, rotate_key},
     key_provider::provider_from_environment,
     leases::LeaseStore,
     metrics::Metrics,
@@ -199,11 +202,13 @@ struct HealthResponse<'a> {
     public_contract: &'a str,
     capability_fingerprint: String,
     runtime: &'a str,
+    /// Replay outcome of the durable log at open (`clean` or
+    /// `recovered-incomplete-write`). Liveness only: it is not a probe of the
+    /// volume now and not a durability claim — see `/ready` and `/v1/status`.
     storage: &'a str,
-    fabric: &'a str,
+    /// Configuration, not a probe: whether this process runs the autonomous
+    /// worker.
     execution: &'a str,
-    workflows: &'a str,
-    agents: &'a str,
     membership: &'a str,
 }
 
@@ -1465,6 +1470,10 @@ struct WorkloadFailRequest {
     fence: WorkloadFenceRequest,
     kind: String,
     error: String,
+    /// `none` (failed before any external effect) or `unknown` (an effect may
+    /// have happened). Omitted: the policy decides by `kind`.
+    #[serde(default)]
+    effect: Option<String>,
 }
 #[derive(Deserialize)]
 struct WorkloadContextRequest {
@@ -1476,6 +1485,9 @@ struct WorkloadContextRequest {
 #[derive(Deserialize)]
 struct WorkloadControlRequest {
     grant: feltdb::authorization::SignedGrant,
+    /// Why an operator retried the workload; recorded in its history.
+    #[serde(default)]
+    reason: Option<String>,
 }
 #[derive(Deserialize)]
 struct WorkerHeartbeatRequest {
@@ -1492,13 +1504,57 @@ struct WorkerReconcileRequest {
     reconciliation: WorkerReconciliation,
     grant: feltdb::authorization::SignedGrant,
 }
+/// Whether this node commits Durable Work.
+///
+/// Durable Work's guarantees rest on one serialization point for each
+/// workload: the guarded commit of one store. Peers replicate by causal
+/// order with last-writer-wins for concurrent writes, so two nodes that both
+/// committed work would each pass their own guard and silently diverge. A node
+/// without peers is therefore always the authority (the single-node default);
+/// in a peered cluster exactly one node, started with `--work-authority`,
+/// commits Durable Work, and every other node serves it read-only from its
+/// replicated store. A node that learns a peer is also the authority stops
+/// being one.
+fn is_work_authority(state: &AppState) -> bool {
+    if state
+        .work_authority_conflict
+        .load(std::sync::atomic::Ordering::SeqCst)
+    {
+        return false;
+    }
+    state.work_authority || state.cluster.get().peers.is_empty()
+}
+/// Refuse a Durable Work write on a node that is not the authority.
+fn require_work_authority(state: &AppState) -> Result<(), ApiError> {
+    if is_work_authority(state) {
+        return Ok(());
+    }
+    let conflict = state
+        .work_authority_conflict
+        .load(std::sync::atomic::Ordering::SeqCst);
+    Err(ApiError::structured(
+        StatusCode::MISDIRECTED_REQUEST,
+        json!({"code": "WORKLOAD_NOT_AUTHORITY",
+            "message": if conflict {
+                "more than one node is configured as the Durable Work authority; Durable Work writes are refused until that is corrected"
+            } else {
+                "this node replicates Durable Work read-only; send the request to the node started with --work-authority"
+            },
+            "details": {"reason": if conflict { "authority_conflict" } else { "replica" }, "retryable": false}}),
+    ))
+}
 fn workload_error(error: feltdb::workload::WorkloadError) -> ApiError {
     let status = match error.code.as_str() {
         "WORKLOAD_NOT_FOUND" => StatusCode::NOT_FOUND,
         "WORKLOAD_STALE_FENCE"
         | "WORKLOAD_TRANSITION_DENIED"
         | "WORKLOAD_NOT_READY"
-        | "WORKLOAD_NOT_YET_ELIGIBLE" => StatusCode::CONFLICT,
+        | "WORKLOAD_NOT_YET_ELIGIBLE"
+        | "WORKLOAD_ATTEMPTS_EXHAUSTED"
+        | "WORKLOAD_RETRY_EXHAUSTED"
+        | "WORKLOAD_RESULT_CONFLICT"
+        | "WORKLOAD_TIMED_OUT"
+        | "WORKLOAD_CONFLICT" => StatusCode::CONFLICT,
         "WORKLOAD_UNAUTHORIZED" | "WORKLOAD_CAPABILITY_DENIED" => StatusCode::FORBIDDEN,
         "WORKLOAD_STORAGE_FAILURE" => StatusCode::SERVICE_UNAVAILABLE,
         _ => StatusCode::UNPROCESSABLE_ENTITY,
@@ -1589,7 +1645,8 @@ async fn create_workload(
     State(state): State<AppState>,
     Extension(principal): Extension<Principal>,
     Json(mut input): Json<CreateWorkload>,
-) -> Result<(StatusCode, Json<Value>), ApiError> {
+) -> Result<(StatusCode, HeaderMap, Json<Value>), ApiError> {
+    require_work_authority(&state)?;
     let subject = grant_subject(&principal);
     input.requested_by = subject.clone();
     input.session_id = principal.session_id.clone();
@@ -1603,7 +1660,7 @@ async fn create_workload(
         "workload:create",
         &input.authorization_snapshot,
     )?;
-    let workload = state
+    let (workload, created) = state
         .workloads
         .lock()
         .map_err(|_| {
@@ -1612,17 +1669,42 @@ async fn create_workload(
                 "workload store unavailable".into(),
             )
         })?
-        .create(input)
-        .map_err(workload_error)?;
+        .admit(input)
+        .map_err(|error| {
+            if error.code == "IDEMPOTENCY_KEY_REUSED" {
+                // AppPort's contract for a reused key (§13.1): CONFLICT, with
+                // the reason, not retryable.
+                // `fields` names the execution-affecting request fields that
+                // differ from the admitted workload.
+                let fields = error
+                    .details
+                    .as_ref()
+                    .and_then(|details| details.get("fields"))
+                    .cloned()
+                    .unwrap_or_else(|| json!([]));
+                ApiError::structured(
+                    StatusCode::CONFLICT,
+                    json!({"code": "CONFLICT", "message": error.message,
+                        "details": {"reason": "IDEMPOTENCY_KEY_REUSED", "retryable": false, "fields": fields}}),
+                )
+            } else {
+                workload_error(error)
+            }
+        })?;
     record_authorization_audit(
         &state,
         &principal.key_id,
-        "workload.created",
+        if created { "workload.created" } else { "workload.replayed" },
         &workload.workload_id,
         "ready",
         201,
     );
-    Ok((StatusCode::CREATED, Json(json!(workload))))
+    // A replay answers with the stored response, marked as a replay.
+    let mut headers = HeaderMap::new();
+    if !created {
+        headers.insert("idempotent-replay", axum::http::HeaderValue::from_static("true"));
+    }
+    Ok((StatusCode::CREATED, headers, Json(json!(workload))))
 }
 async fn list_workloads(
     State(state): State<AppState>,
@@ -1644,7 +1726,8 @@ async fn list_workloads(
                 "workload store unavailable".into(),
             )
         })?
-        .list(&tenant, &target.application_id);
+        .fresh_list(&tenant, &target.application_id)
+        .map_err(workload_error)?;
     Ok(Json(json!({"workloads":values,"grant_id":target.grant_id})))
 }
 async fn get_workload(
@@ -1661,28 +1744,42 @@ async fn get_workload(
                 "workload store unavailable".into(),
             )
         })?
-        .get(&id)
+        .fresh(&id)
         .map_err(workload_error)?
         .clone();
-    application_scope(
-        &state,
-        &principal.key_id,
-        &workload.application_id,
-        "workload:read",
-    )?;
+    workload_read_scope(&state, &principal, &workload.application_id)?;
     Ok(Json(json!(workload)))
 }
+/// Authorize reading one workload. A caller who may not read the
+/// application's work gets the same answer as for a workload that does not
+/// exist, so workload ids cannot be probed.
+fn workload_read_scope(
+    state: &AppState,
+    principal: &Principal,
+    application_id: &str,
+) -> Result<(), ApiError> {
+    application_scope(state, &principal.key_id, application_id, "workload:read")
+        .map(|_| ())
+        .map_err(|_| {
+            // Exactly the body `workload_error` gives a missing workload.
+            ApiError::structured(
+                StatusCode::NOT_FOUND,
+                json!({"code": "WORKLOAD_NOT_FOUND", "message": "workload not found"}),
+            )
+        })
+}
 async fn claim_workload(
     State(state): State<AppState>,
     Extension(principal): Extension<Principal>,
     Path(id): Path<String>,
     Json(input): Json<WorkloadClaimRequest>,
 ) -> Result<Json<Value>, ApiError> {
+    require_work_authority(&state)?;
     let workload = state
         .workloads
         .lock()
         .unwrap()
-        .get(&id)
+        .fresh(&id)
         .map_err(workload_error)?
         .clone();
     let subject = grant_subject(&principal);
@@ -1736,7 +1833,7 @@ fn authorize_fence(
         .workloads
         .lock()
         .unwrap()
-        .get(id)
+        .fresh(id)
         .map_err(workload_error)?
         .clone();
     let subject = grant_subject(principal);
@@ -1763,6 +1860,7 @@ async fn start_workload(
     Path(id): Path<String>,
     Json(input): Json<WorkloadFenceRequest>,
 ) -> Result<StatusCode, ApiError> {
+    require_work_authority(&state)?;
     authorize_fence(&state, &principal, &id, &input, "workload:execute")?;
     state
         .workloads
@@ -1784,6 +1882,7 @@ async fn heartbeat_workload(
     Path(id): Path<String>,
     Json(input): Json<WorkloadHeartbeatRequest>,
 ) -> Result<Json<Value>, ApiError> {
+    require_work_authority(&state)?;
     authorize_fence(&state, &principal, &id, &input.fence, "workload:execute")?;
     let claim = state
         .workloads
@@ -1806,6 +1905,7 @@ async fn checkpoint_workload(
     Path(id): Path<String>,
     Json(input): Json<WorkloadCheckpointRequest>,
 ) -> Result<Json<Value>, ApiError> {
+    require_work_authority(&state)?;
     authorize_fence(&state, &principal, &id, &input.fence, "workload:execute")?;
     let checkpoint = state
         .workloads
@@ -1828,6 +1928,7 @@ async fn complete_workload(
     Path(id): Path<String>,
     Json(input): Json<WorkloadCompleteRequest>,
 ) -> Result<Json<Value>, ApiError> {
+    require_work_authority(&state)?;
     authorize_fence(&state, &principal, &id, &input.fence, "workload:execute")?;
     let result = state
         .workloads
@@ -1854,18 +1955,20 @@ async fn fail_workload(
     Path(id): Path<String>,
     Json(input): Json<WorkloadFailRequest>,
 ) -> Result<Json<Value>, ApiError> {
+    require_work_authority(&state)?;
     authorize_fence(&state, &principal, &id, &input.fence, "workload:execute")?;
     let status = state
         .workloads
         .lock()
         .unwrap()
-        .fail(
+        .fail_with_effect(
             &id,
             &input.fence.claim_id,
             &input.fence.worker_id,
             input.fence.fencing_token,
             &input.kind,
             &input.error,
+            input.effect.as_deref(),
             unix_seconds_i64(),
         )
         .map_err(workload_error)?;
@@ -1878,6 +1981,7 @@ async fn confirm_workload_cancelled(
     Path(id): Path<String>,
     Json(input): Json<WorkloadFenceRequest>,
 ) -> Result<StatusCode, ApiError> {
+    require_work_authority(&state)?;
     authorize_fence(&state, &principal, &id, &input, "workload:execute")?;
     state
         .workloads
@@ -1899,6 +2003,7 @@ async fn workload_execution_context(
     Path(id): Path<String>,
     Json(input): Json<WorkloadContextRequest>,
 ) -> Result<Json<Value>, ApiError> {
+    require_work_authority(&state)?;
     authorize_fence(&state, &principal, &id, &input.fence, "workload:execute")?;
     let context = state
         .workloads
@@ -1921,11 +2026,12 @@ async fn cancel_workload(
     Path(id): Path<String>,
     Json(input): Json<WorkloadControlRequest>,
 ) -> Result<Json<Value>, ApiError> {
+    require_work_authority(&state)?;
     let workload = state
         .workloads
         .lock()
         .unwrap()
-        .get(&id)
+        .fresh(&id)
         .map_err(workload_error)?
         .clone();
     workload_decision(
@@ -1951,11 +2057,12 @@ async fn retry_workload(
     Path(id): Path<String>,
     Json(input): Json<WorkloadControlRequest>,
 ) -> Result<StatusCode, ApiError> {
+    require_work_authority(&state)?;
     let workload = state
         .workloads
         .lock()
         .unwrap()
-        .get(&id)
+        .fresh(&id)
         .map_err(workload_error)?
         .clone();
     workload_decision(
@@ -1971,7 +2078,7 @@ async fn retry_workload(
         .workloads
         .lock()
         .unwrap()
-        .retry(&id, &principal.key_id, unix_seconds_i64())
+        .retry_with_reason(&id, &principal.key_id, unix_seconds_i64(), input.reason.as_deref())
         .map_err(workload_error)?;
     Ok(StatusCode::NO_CONTENT)
 }
@@ -1984,15 +2091,10 @@ async fn workload_history(
         .workloads
         .lock()
         .unwrap()
-        .get(&id)
+        .fresh(&id)
         .map_err(workload_error)?
         .clone();
-    application_scope(
-        &state,
-        &principal.key_id,
-        &workload.application_id,
-        "workload:read",
-    )?;
+    workload_read_scope(&state, &principal, &workload.application_id)?;
     Ok(Json(
         json!({"events":workload.execution_history,"attempts":workload.attempts,"checkpoints":workload.checkpoints}),
     ))
@@ -2006,15 +2108,10 @@ async fn workload_result(
         .workloads
         .lock()
         .unwrap()
-        .get(&id)
+        .fresh(&id)
         .map_err(workload_error)?
         .clone();
-    application_scope(
-        &state,
-        &principal.key_id,
-        &workload.application_id,
-        "workload:read",
-    )?;
+    workload_read_scope(&state, &principal, &workload.application_id)?;
     Ok(Json(json!(workload.result)))
 }
 
@@ -2196,7 +2293,8 @@ async fn reconcile_worker(
 ) -> Result<Json<Value>, ApiError> {
     authorize_worker_action(&state, &principal, &id, &input.grant, "worker:heartbeat")?;
     input.reconciliation.worker_id = id;
-    let workloads = state.workloads.lock().unwrap();
+    let mut workloads = state.workloads.lock().unwrap();
+    workloads.refresh().map_err(workload_error)?;
     let result = state
         .mesh
         .lock()
@@ -2287,15 +2385,10 @@ async fn eligible_workers(
         .workloads
         .lock()
         .unwrap()
-        .get(&id)
+        .fresh(&id)
         .map_err(workload_error)?
         .clone();
-    application_scope(
-        &state,
-        &principal.key_id,
-        &workload.application_id,
-        "workload:read",
-    )?;
+    workload_read_scope(&state, &principal, &workload.application_id)?;
     let workers = state
         .mesh
         .lock()
@@ -6935,6 +7028,139 @@ async fn auth_sign_up(
         .map_err(control_error)?;
     Ok((StatusCode::CREATED, auth_response(identity, session, token)))
 }
+#[derive(Deserialize)]
+struct ConsolePasswordRequest {
+    email: String,
+    password: String,
+}
+#[derive(Deserialize)]
+struct ConsoleGithubRequest {
+    github_id: Value,
+    login: String,
+}
+
+fn console_account_error(error: console_accounts::AccountError) -> ApiError {
+    ApiError::structured(
+        StatusCode::from_u16(error.status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR),
+        json!({ "code": error.code, "message": error.message }),
+    )
+}
+
+/// Authenticate a website call to `/v1/console/accounts/*` and parse its body.
+fn console_request<T: serde::de::DeserializeOwned>(
+    state: &AppState,
+    method: &Method,
+    uri: &axum::http::Uri,
+    headers: &HeaderMap,
+    body: &[u8],
+) -> Result<T, ApiError> {
+    let Some(secret) = state.console_accounts.secret() else {
+        return Err(ApiError::structured(
+            StatusCode::SERVICE_UNAVAILABLE,
+            json!({
+                "code": "CONSOLE_ACCOUNTS_UNCONFIGURED",
+                "message": "console accounts require FELTDB_SESSION_SECRET (at least 32 bytes) on the control plane",
+            }),
+        ));
+    };
+    let header = |name: &str| headers.get(name).and_then(|value| value.to_str().ok());
+    let now = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_secs();
+    if let Err(error) = console_accounts::verify_console_request(
+        secret,
+        method.as_str(),
+        uri.path(),
+        header(console_accounts::TIMESTAMP_HEADER),
+        header(console_accounts::SIGNATURE_HEADER),
+        body,
+        now,
+    ) {
+        state.metrics.authentication_failure();
+        audit(state, "anonymous", "console.accounts", uri.path(), "denied", 401);
+        return Err(console_account_error(error));
+    }
+    serde_json::from_slice(body)
+        .map_err(|error| ApiError(StatusCode::BAD_REQUEST, format!("invalid request body: {error}")))
+}
+
+/// Argon2 work and the account file write run off the request worker.
+async fn console_blocking<F>(work: F) -> Result<Json<Value>, ApiError>
+where
+    F: FnOnce() -> Result<console_accounts::ResolvedAccount, console_accounts::AccountError>
+        + Send
+        + 'static,
+{
+    match tokio::task::spawn_blocking(work).await {
+        Ok(Ok(account)) => Ok(Json(json!({
+            "user_id": account.user_id,
+            "identity": account.identity,
+            "display_identity": account.display_identity,
+        }))),
+        Ok(Err(error)) => Err(console_account_error(error)),
+        Err(_) => Err(ApiError(
+            StatusCode::INTERNAL_SERVER_ERROR,
+            "console account operation failed".into(),
+        )),
+    }
+}
+
+async fn console_password_signup(
+    State(state): State<AppState>,
+    method: Method,
+    uri: axum::http::Uri,
+    headers: HeaderMap,
+    body: axum::body::Bytes,
+) -> Result<(StatusCode, Json<Value>), ApiError> {
+    let input: ConsolePasswordRequest = console_request(&state, &method, &uri, &headers, &body)?;
+    let created = console_blocking(move || {
+        state
+            .console_accounts
+            .password_signup(&state.tenancy, &input.email, &input.password)
+    })
+    .await?;
+    Ok((StatusCode::CREATED, created))
+}
+
+async fn console_password_signin(
+    State(state): State<AppState>,
+    method: Method,
+    uri: axum::http::Uri,
+    headers: HeaderMap,
+    body: axum::body::Bytes,
+) -> Result<Json<Value>, ApiError> {
+    let input: ConsolePasswordRequest = console_request(&state, &method, &uri, &headers, &body)?;
+    console_blocking(move || {
+        state
+            .console_accounts
+            .password_signin(&input.email, &input.password)
+    })
+    .await
+}
+
+async fn console_github_resolve(
+    State(state): State<AppState>,
+    method: Method,
+    uri: axum::http::Uri,
+    headers: HeaderMap,
+    body: axum::body::Bytes,
+) -> Result<Json<Value>, ApiError> {
+    let input: ConsoleGithubRequest = console_request(&state, &method, &uri, &headers, &body)?;
+    // GitHub user ids are numbers; accept the number or its decimal string.
+    let github_id = match &input.github_id {
+        Value::Number(number) if number.is_u64() => number.to_string(),
+        Value::String(text) => text.clone(),
+        _ => String::new(),
+    };
+    console_blocking(move || {
+        state
+            .console_accounts
+            .github_resolve(&state.tenancy, &github_id, &input.login)
+    })
+    .await
+}
+
 async fn auth_sign_in(
     State(state): State<AppState>,
     Json(input): Json<SignInRequest>,
@@ -7573,7 +7799,9 @@ fn state_contract_error(error: feltdb::state_contract::StateFailure) -> ApiError
         "PRECONDITION_FAILED" | "CONFLICT" | "IDEMPOTENCY_CONFLICT" => StatusCode::CONFLICT,
         "UNKNOWN_COLLECTION" => StatusCode::NOT_FOUND,
         "SEMANTIC_UNSUPPORTED" => StatusCode::NOT_IMPLEMENTED,
-        "STORAGE_FAILURE" | "STATE_UNAVAILABLE" => StatusCode::SERVICE_UNAVAILABLE,
+        "STORAGE_FAILURE" | "STATE_UNAVAILABLE" | "DURABILITY_FAILURE" => {
+            StatusCode::SERVICE_UNAVAILABLE
+        }
         _ => StatusCode::UNPROCESSABLE_ENTITY,
     };
     ApiError(
@@ -8210,6 +8438,10 @@ struct SyncPullResponse {
     operations: Vec<Operation>,
     versions: HashMap<String, u64>,
     catchup: CatchupStatus,
+    /// Whether the answering node is configured as the Durable Work authority
+    /// (`--work-authority`). Absent from older peers.
+    #[serde(default)]
+    durable_work_authority: bool,
 }
 
 #[derive(Serialize, Deserialize)]
@@ -8418,6 +8650,10 @@ struct StartAgentRun {
     goal: String,
     #[serde(default)]
     input: Value,
+    /// What the runtime must observe for the run to complete. Without it the
+    /// run can execute but never completes: the goal text is not evaluated.
+    #[serde(default)]
+    completion: Option<CompletionCriteria>,
 }
 
 #[derive(Deserialize)]
@@ -8454,6 +8690,7 @@ struct Config {
     audit: PathBuf,
     allowed_origins: Vec<axum::http::HeaderValue>,
     worker_enabled: bool,
+    work_authority: bool,
     advertise_url: Option<String>,
     maximum_clock_drift_ms: u64,
 }
@@ -8502,6 +8739,7 @@ impl Config {
             .transpose()?
             .unwrap_or_default();
         let mut worker_enabled = std::env::var("FELTDB_WORKER_ENABLED").as_deref() == Ok("true");
+        let mut work_authority = std::env::var("FELTDB_WORK_AUTHORITY").as_deref() == Ok("true");
         let mut advertise_url = std::env::var("FELTDB_ADVERTISE_URL")
             .ok()
             .map(|value| value.trim_end_matches('/').to_string());
@@ -8554,6 +8792,7 @@ impl Config {
                         .map_err(|_| "invalid --allow-origin".to_string())?,
                 ),
                 "--worker" => worker_enabled = true,
+                "--work-authority" => work_authority = true,
                 "--advertise-url" => {
                     advertise_url = Some(value()?.trim_end_matches('/').to_string())
                 }
@@ -8568,7 +8807,7 @@ impl Config {
                     }
                 }
                 "--help" | "-h" => {
-                    println!("feltdb-server [--host 127.0.0.1] [--port 7700] [--advertise-url https://node.example] [--namespace default] [--data ./data/feltdb.log] [--auth] [--keys ./data/api-keys.json] [--audit ./data/feltdb.audit.log] [--allow-origin https://app.example] [--peer https://peer] [--peer-token-env FELTDB_PEER_TOKEN] [--sync-interval-ms 2000] [--worker]");
+                    println!("feltdb-server [--host 127.0.0.1] [--port 7700] [--advertise-url https://node.example] [--namespace default] [--data ./data/feltdb.log] [--auth] [--keys ./data/api-keys.json] [--audit ./data/feltdb.audit.log] [--allow-origin https://app.example] [--peer https://peer] [--peer-token-env FELTDB_PEER_TOKEN] [--sync-interval-ms 2000] [--worker] [--work-authority]");
                     std::process::exit(0);
                 }
                 _ => return Err(format!("unknown argument: {argument}")),
@@ -8591,6 +8830,7 @@ impl Config {
             audit,
             allowed_origins,
             worker_enabled,
+            work_authority,
             advertise_url,
             maximum_clock_drift_ms,
         })
@@ -8661,10 +8901,28 @@ fn validate_production_configuration(config: &Config) -> Result<(), Box<dyn std:
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
     tracing_subscriber::fmt()
         .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
+        // Every log line passes through credential redaction, whatever wrote it.
+        .with_writer(feltdb_server::redaction::RedactingStdout)
         .init();
+    if matches!(
+        std::env::args().nth(1).as_deref(),
+        Some("version" | "--version" | "-V")
+    ) {
+        // One line an operator script can print instead of a hard-coded banner.
+        println!(
+            "feltdb-server {} (release {}, commit {})",
+            env!("CARGO_PKG_VERSION"),
+            option_env!("FELTDB_RELEASE_ID").unwrap_or("unknown"),
+            option_env!("FELTDB_GIT_COMMIT").unwrap_or("unknown")
+        );
+        return Ok(());
+    }
     if manage_keys()? {
         return Ok(());
     }
+    if manage_console_accounts()? {
+        return Ok(());
+    }
     if manage_authorization()? {
         return Ok(());
     }
@@ -8801,6 +9059,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         lifecycle_lock: Arc::new(tokio::sync::Mutex::new(())),
         state_transaction_admission: Arc::new(tokio::sync::Semaphore::new(2)),
         worker_enabled: config.worker_enabled,
+        work_authority: config.work_authority,
+        work_authority_conflict: Arc::new(std::sync::atomic::AtomicBool::new(false)),
         leases: LeaseStore::load(config.data.with_extension("leases.json"))?,
         content: content_store,
         artifacts: artifact_store,
@@ -8822,6 +9082,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                 .into_bytes(),
             config.data.with_extension("revoked-sessions.json"),
         )?,
+        console_accounts: ConsoleAccountStore::load(
+            config.data.with_extension("console-accounts.json"),
+            console_accounts::configured_secret(std::env::var("FELTDB_SESSION_SECRET").ok()),
+        )?,
         principals: principal_authority,
         connections: ConnectionStore::load_with_provider(
             config.data.with_extension("connections.json"),
@@ -8846,6 +9110,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     };
     start_peer_sessions(&state, &config)?;
     start_membership_recovery(&state);
+    start_workload_recovery(&state);
     start_autonomous_worker(&state);
 
     let protected = Router::new()
@@ -8909,6 +9174,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         .route("/api/tenants/{tenant_id}", get(get_tenant))
         .route("/api/keys", get(list_keys).post(create_key))
         .route("/api/keys/{id}", axum::routing::delete(revoke_key))
+        .route("/api/keys/{id}/rotate", axum::routing::post(rotate_key))
+        // Operator observability: authenticated, `operator:read`.
+        .route("/metrics", get(network_metrics))
+        .route("/v1/status", get(operator_status))
         .route(
             "/api/tenants/{tenant_id}/applications",
             get(list_applications).post(create_application_control),
@@ -9569,23 +9838,23 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         .route("/v1/openapi.json", get(application_openapi))
         .route("/v1/auth/signup", axum::routing::post(auth_sign_up))
         .route("/v1/auth/signin", axum::routing::post(auth_sign_in))
-        .route("/internal/intents/by-hash", get(internal_find_intent))
+        // Console accounts: callable only by the website, which signs each
+        // request with FELTDB_SESSION_SECRET (see console_accounts.rs).
         .route(
-            "/internal/intents",
-            axum::routing::post(internal_create_intent),
+            "/v1/console/accounts/password/signup",
+            axum::routing::post(console_password_signup),
         )
-        .route("/internal/intents/{id}", put(internal_replace_intent))
         .route(
-            "/internal/intent-events",
-            axum::routing::post(internal_create_intent_event),
+            "/v1/console/accounts/password/signin",
+            axum::routing::post(console_password_signin),
         )
-        .route("/ready", get(readiness))
-        .route("/runtime", get(runtime))
-        .route("/metrics", get(network_metrics))
         .route(
-            "/api/certification/fixtures/{application_id}",
-            axum::routing::delete(delete_certification_fixture),
+            "/v1/console/accounts/github/resolve",
+            axum::routing::post(console_github_resolve),
         )
+        .route("/ready", get(readiness))
+        .route("/runtime", get(runtime))
+        .merge(service_secret_routes())
         .merge(protected)
         .merge(query_execution_diagnostics_router())
         .layer(middleware::from_fn_with_state(state.clone(), admit_request))
@@ -9593,7 +9862,17 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         .layer(middleware::from_fn(attribute_request))
         .layer(middleware::from_fn_with_state(state.clone(), count_request))
         .layer(RequestBodyLimitLayer::new(1024 * 1024))
-        .layer(TraceLayer::new_for_http())
+        .layer(middleware::from_fn(redact_error_responses))
+        // Spans carry the path, never the query string, where clients put tokens.
+        .layer(TraceLayer::new_for_http().make_span_with(
+            |request: &Request<axum::body::Body>| {
+                tracing::debug_span!(
+                    "request",
+                    method = %request.method(),
+                    path = %request.uri().path(),
+                )
+            },
+        ))
         .with_state(state);
     if !config.allowed_origins.is_empty() {
         app = app.layer(
@@ -9637,6 +9916,32 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     Ok(())
 }
 
+/// Error bodies can quote client input (axum's JSON rejections echo the
+/// offending value); a credential sent by mistake must not come back out, or
+/// land in whatever logs the response. Success bodies are untouched: a key's
+/// secret is returned exactly once, at creation or rotation.
+async fn redact_error_responses(request: Request<axum::body::Body>, next: Next) -> Response {
+    let response = next.run(request).await;
+    if response.status().as_u16() < 400 {
+        return response;
+    }
+    let (parts, body) = response.into_parts();
+    let bytes = match axum::body::to_bytes(body, 1024 * 1024).await {
+        Ok(bytes) => bytes,
+        Err(_) => return Response::from_parts(parts, axum::body::Body::empty()),
+    };
+    let body = match std::str::from_utf8(&bytes) {
+        Ok(text) => match feltdb_server::redaction::redact(text) {
+            std::borrow::Cow::Owned(redacted) => axum::body::Body::from(redacted),
+            std::borrow::Cow::Borrowed(_) => axum::body::Body::from(bytes),
+        },
+        Err(_) => axum::body::Body::from(bytes),
+    };
+    let mut parts = parts;
+    parts.headers.remove(axum::http::header::CONTENT_LENGTH);
+    Response::from_parts(parts, body)
+}
+
 async fn count_request(
     State(state): State<AppState>,
     request: Request<axum::body::Body>,
@@ -9669,7 +9974,7 @@ async fn authenticate(
     // an answer to PR36's question — which part of the request does is.
     let authorization =
         feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::Authorization);
-    let principal = if state.auth_enabled {
+    let (principal, class) = if state.auth_enabled {
         let token = request
             .headers()
             .get(AUTHORIZATION)
@@ -9732,8 +10037,8 @@ async fn authenticate(
             })
             .and_then(|token| state.sessions.verify(token));
         match (machine, actor, human) {
-            (Some(principal), _, _) => principal,
-            (None, Some(session), _) => Principal {
+            (Some(principal), _, _) => (principal, PrincipalClass::Machine),
+            (None, Some(session), _) => (Principal {
                 key_id: session.subject_id,
                 scopes: vec![
                     "application:read".into(),
@@ -9748,24 +10053,55 @@ async fn authenticate(
                 subject_type: "human".into(),
                 identity_id: Some(session.identity_id),
                 session_id: Some(session.session_id),
-            },
+            }, PrincipalClass::EndUser),
             (None, None, Some(session)) => {
-                if state
-                    .tenancy
-                    .register_platform_user(&session.user_id, &session.identity)
-                    .is_err()
-                    || !state.tenancy.platform_user_active(&session.user_id)
+                // A signed cookie is not enough: its user id must belong to a
+                // console account (created through the verified account
+                // routes, or bound/claimed for a pre-0.15.0 user), and its
+                // identity must be that account's. Sessions the website
+                // minted before accounts existed are refused here.
+                let account = match state.console_accounts.account_for_user(&session.user_id) {
+                    Some(account) if account.display_identity == session.identity.trim().to_lowercase() => account,
+                    _ => {
+                        state.metrics.authentication_failure();
+                        audit(
+                            &state,
+                            &session.user_id,
+                            "authenticate",
+                            request.uri().path(),
+                            "denied",
+                            401,
+                        );
+                        return ApiError::structured(
+                            StatusCode::UNAUTHORIZED,
+                            json!({
+                                "code": "CONSOLE_ACCOUNT_REQUIRED",
+                                "message": "this console session is not bound to a console account; sign in again",
+                            }),
+                        )
+                        .into_response();
+                    }
+                };
+                if (!state.tenancy.platform_user_exists(&account.user_id)
+                    && state
+                        .tenancy
+                        .register_platform_user(&account.user_id, &account.display_identity)
+                        .is_err())
+                    || !state.tenancy.platform_user_active(&account.user_id)
                 {
                     return ApiError(StatusCode::FORBIDDEN, "platform user is suspended".into())
                         .into_response();
                 }
-                Principal {
-                    key_id: session.user_id,
-                    scopes: vec![],
-                    subject_type: "human".into(),
-                    identity_id: Some(session.identity),
-                    session_id: Some(session.id),
-                }
+                (
+                    Principal {
+                        key_id: account.user_id,
+                        scopes: vec![],
+                        subject_type: "human".into(),
+                        identity_id: Some(account.display_identity),
+                        session_id: Some(session.id),
+                    },
+                    PrincipalClass::ConsoleUser,
+                )
             }
             (None, None, None) => {
                 state.metrics.authentication_failure();
@@ -9785,17 +10121,43 @@ async fn authenticate(
             }
         }
     } else {
-        Principal {
-            key_id: "development".to_string(),
-            scopes: vec!["*".to_string()],
-            subject_type: "service".into(),
-            identity_id: None,
-            session_id: None,
-        }
+        (
+            Principal {
+                key_id: "development".to_string(),
+                scopes: vec!["*".to_string()],
+                subject_type: "service".into(),
+                identity_id: None,
+                session_id: None,
+            },
+            PrincipalClass::Development,
+        )
     };
 
     let path = request.uri().path();
-    let required = if path == "/events" {
+    // The control plane (`/api/*`) is decided per route, before any handler:
+    // machine keys need the route's capability, console users are held to
+    // tenant/application membership by the handler, and application end-user
+    // sessions are refused. There is no path family without a decision.
+    let control = control_authorization::control_plane_requirement(request.method(), path);
+    if let Some(requirement) = control {
+        if let Err(denial) = control_authorization::decide(class, &principal.scopes, requirement) {
+            state.metrics.authentication_failure();
+            let action = match requirement {
+                control_authorization::Requirement::Capability(scope) => scope,
+                _ => "control-plane",
+            };
+            audit(&state, &principal.key_id, action, path, "denied", 403);
+            return ApiError(StatusCode::FORBIDDEN, denial.body()).into_response();
+        }
+    }
+    let required = if let Some(requirement) = control {
+        match requirement {
+            control_authorization::Requirement::Capability(scope) => scope,
+            _ => "session:self",
+        }
+    } else if path == "/metrics" || path == "/v1/status" {
+        control_authorization::OPERATOR_READ
+    } else if path == "/events" {
         "events:read"
     } else if path == "/v1/state/diff"
         || path == "/v1/state/conflicts"
@@ -9855,6 +10217,24 @@ async fn authenticate(
     } else {
         "state:write"
     };
+    // An end-user session belongs to one application (self-serve sign-up
+    // registers it there). Its state scopes are meaningful only on the
+    // application API, which checks that membership per request. The legacy
+    // routes (`/collections`, `/transactions`, `/cas`, `/events`, ...) are
+    // scoped to the whole server namespace, so a session there would read and
+    // write every application's records.
+    if class == PrincipalClass::EndUser && !path.starts_with("/v1/") {
+        state.metrics.authentication_failure();
+        audit(&state, &principal.key_id, required, path, "denied", 403);
+        let denial = control_authorization::Denial::new(
+            control_authorization::END_USER_SESSION_FORBIDDEN,
+            format!(
+                "end-user sessions can only use the application API (/v1/...) of their own application; {path} is a namespace-wide route — use an application-scoped service key on the server side"
+            ),
+        )
+        .with_scope(required);
+        return ApiError(StatusCode::FORBIDDEN, denial.body()).into_response();
+    }
     let subject = if principal.subject_type == "human" {
         Subject::User(principal.key_id.clone())
     } else {
@@ -9872,16 +10252,26 @@ async fn authenticate(
     let human_contract = principal.subject_type == "human"
         && path.starts_with("/v1/applications/")
         && path.split('/').any(|segment| segment == "contract");
+    // Two non-control routes are decided by who is asking rather than by a
+    // scope, and are listed here rather than exempted: a user's own contract
+    // drafts (keyed by that user) and the authenticated diagnostics summary.
     let authenticated_diagnostics = path == "/v1/diagnostics";
-    if !path.starts_with("/api/")
-        && !human_contract
-        && !authenticated_diagnostics
-        && !decision.allowed
-    {
+    let decided_by_route = control.is_some() || human_contract || authenticated_diagnostics;
+    if !decided_by_route && !decision.allowed {
         state.metrics.authentication_failure();
         audit(&state, &principal.key_id, required, path, "denied", 403);
-        return ApiError(StatusCode::FORBIDDEN, format!("missing scope: {required}"))
-            .into_response();
+        let denial = if class == PrincipalClass::EndUser
+            && (path == "/metrics" || path == "/v1/status" || path.starts_with("/admin/"))
+        {
+            control_authorization::Denial::new(
+                control_authorization::END_USER_SESSION_FORBIDDEN,
+                format!("missing scope {required} — end-user sessions cannot use operator routes; use an operator key"),
+            )
+            .with_scope(required)
+        } else {
+            control_authorization::Denial::missing_scope(required)
+        };
+        return ApiError(StatusCode::FORBIDDEN, denial.body()).into_response();
     }
     if path == "/events" {
         state.metrics.event_connection();
@@ -9900,6 +10290,7 @@ async fn authenticate(
 
     request.extensions_mut().insert(principal.clone());
     request.extensions_mut().insert(authenticated.clone());
+    request.extensions_mut().insert(class);
 
     let principal_id = request
         .extensions()
@@ -10245,14 +10636,14 @@ async fn health(State(state): State<AppState>) -> Json<HealthResponse<'static>>
         capability_fingerprint: control_plane::public_contract_capability_fingerprint(),
         runtime: "self-hosted",
         storage: database.storage.label(),
-        fabric: "healthy",
+        // `fabric`, `workflows` and `agents` were constant "healthy" /
+        // "available" here whatever their state (GAP-10). Nothing checks them,
+        // so nothing reports them.
         execution: if state.worker_enabled {
             "autonomous"
         } else {
             "state-coordinated"
         },
-        workflows: "available",
-        agents: "available",
         membership: if clock {
             membership
         } else {
@@ -10390,10 +10781,19 @@ fn manage_keys() -> Result<bool, Box<dyn std::error::Error>> {
                 values("--namespace", "default"),
                 option("--worker"),
             )?;
-            println!(
-                "API key created (id: {}). This secret will not be shown again:\n{}",
-                key.id, key.secret
-            );
+            // `--secret-file <path>` writes the secret to an owner-only file
+            // and prints only the id, so a bootstrap script never has to put a
+            // credential on stdout (and from there into platform logs).
+            match option("--secret-file") {
+                Some(file) => {
+                    write_secret_file(std::path::Path::new(&file), &key.secret)?;
+                    println!("API key created (id: {}). Secret written to {file} (mode 0600).", key.id);
+                }
+                None => println!(
+                    "API key created (id: {}). This secret will not be shown again:\n{}",
+                    key.id, key.secret
+                ),
+            }
         }
         "list" => println!("{}", serde_json::to_string_pretty(&store.list()?)?),
         "revoke" => {
@@ -10403,11 +10803,93 @@ fn manage_keys() -> Result<bool, Box<dyn std::error::Error>> {
             }
             println!("API key revoked: {id}");
         }
-        _ => return Err("usage: feltdb-server keys <create|list|revoke> [--keys path] [--name name] [--scope scopes] [--namespace names] [--worker worker-id] [--id key-id]".into()),
+        _ => return Err("usage: feltdb-server keys <create|list|revoke> [--keys path] [--name name] [--scope scopes] [--namespace names] [--worker worker-id] [--secret-file path] [--id key-id]".into()),
+    }
+    Ok(true)
+}
+
+/// `feltdb-server console-accounts <claim|list> --data <state.log> ...`
+///
+/// `claim --email <email> --password-file <file>` binds a console password
+/// account to the existing pre-0.15.0 platform user registered under that
+/// e-mail, so its owner keeps their tenants. Before 0.15.0 the website did not
+/// verify passwords, so such a user cannot be claimed by self-service sign-up;
+/// the operator verifies the person out of band and runs this. The password
+/// is read from a file (first line) so it never appears in the process list.
+fn manage_console_accounts() -> Result<bool, Box<dyn std::error::Error>> {
+    let arguments: Vec<String> = std::env::args().collect();
+    if arguments.get(1).map(String::as_str) != Some("console-accounts") {
+        return Ok(false);
+    }
+    let usage = "usage: feltdb-server console-accounts <claim|list> --data <state.log path> [--email email --password-file path]";
+    let option = |name: &str| {
+        arguments
+            .iter()
+            .position(|argument| argument == name)
+            .and_then(|index| arguments.get(index + 1))
+            .cloned()
+    };
+    let data = PathBuf::from(
+        option("--data").unwrap_or_else(|| "./data/feltdb.log".to_string()),
+    );
+    let accounts = ConsoleAccountStore::load(data.with_extension("console-accounts.json"), None)?;
+    let tenancy = TenancyStore::load(data.with_extension("platform.json"))?;
+    match arguments.get(2).map(String::as_str).unwrap_or("") {
+        "claim" => {
+            let email = option("--email").ok_or("console-accounts claim requires --email")?;
+            let file = option("--password-file")
+                .ok_or("console-accounts claim requires --password-file")?;
+            let password = std::fs::read_to_string(&file)?
+                .lines()
+                .next()
+                .unwrap_or_default()
+                .to_string();
+            let claimed = accounts
+                .claim_password(&tenancy, &email, &password)
+                .map_err(|error| format!("{}: {}", error.code, error.message))?;
+            println!(
+                "Console account claimed: {} is bound to existing platform user {}.",
+                claimed.identity, claimed.user_id
+            );
+        }
+        "list" => {
+            let listed: Vec<Value> = accounts
+                .list()
+                .into_iter()
+                .map(|account| {
+                    json!({
+                        "identity": account.identity,
+                        "user_id": account.user_id,
+                        "provider": account.provider,
+                        "display_identity": account.display_identity,
+                        "origin": account.origin,
+                        "created_at": account.created_at,
+                    })
+                })
+                .collect();
+            println!("{}", serde_json::to_string_pretty(&listed)?);
+        }
+        _ => return Err(usage.into()),
     }
     Ok(true)
 }
 
+/// Create `path` owner-only (0600) and write `secret` to it; refuses to
+/// overwrite, so a second bootstrap never silently replaces a delivered secret.
+fn write_secret_file(path: &std::path::Path, secret: &str) -> std::io::Result<()> {
+    let mut options = OpenOptions::new();
+    options.write(true).create_new(true);
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::OpenOptionsExt;
+        options.mode(0o600);
+    }
+    let mut file = options.open(path)?;
+    file.write_all(secret.as_bytes())?;
+    file.write_all(b"\n")?;
+    file.sync_all()
+}
+
 fn manage_authorization() -> Result<bool, Box<dyn std::error::Error>> {
     let arguments: Vec<String> = std::env::args().collect();
     let family = arguments.get(1).map(String::as_str).unwrap_or("");
@@ -10934,6 +11416,37 @@ fn manage_application_runtime() -> Result<bool, Box<dyn std::error::Error>> {
     Ok(true)
 }
 
+/// Where an online backup bundle may be written.
+///
+/// With `FELTDB_BACKUP_DIR` set (the managed deployments set it), the request
+/// names a bundle, not a path: a single relative component placed inside that
+/// directory. Without it the server is self-hosted and the operator's path is
+/// used as given, as before.
+fn backup_output_path(
+    requested: &str,
+    backup_dir: Option<std::ffi::OsString>,
+) -> Result<PathBuf, ApiError> {
+    let Some(root) = backup_dir.filter(|value| !value.is_empty()) else {
+        return Ok(PathBuf::from(requested));
+    };
+    let name = std::path::Path::new(requested);
+    let mut components = name.components();
+    let single = matches!(
+        (components.next(), components.next()),
+        (Some(std::path::Component::Normal(_)), None)
+    );
+    if !single {
+        return Err(ApiError(
+            StatusCode::BAD_REQUEST,
+            format!(
+                "backup output must be a bundle name, not a path: this server writes bundles inside FELTDB_BACKUP_DIR ({}); send e.g. {{\"output\":\"backup-2026-10-28\"}}",
+                std::path::Path::new(&root).display()
+            ),
+        ));
+    }
+    Ok(PathBuf::from(root).join(name))
+}
+
 async fn create_online_backup(
     State(state): State<AppState>,
     Json(request): Json<OnlineBackupRequest>,
@@ -10963,7 +11476,7 @@ async fn create_online_backup(
         })
         .cloned()
         .collect();
-    let output = PathBuf::from(request.output);
+    let output = backup_output_path(&request.output, std::env::var_os("FELTDB_BACKUP_DIR"))?;
     let source = feltdb_server::backup::BackupSource {
         data: state
             .backup_artifacts
@@ -10975,14 +11488,22 @@ async fn create_online_backup(
         server_version: Some(env!("CARGO_PKG_VERSION").into()),
         durable_format: Some(feltdb::DURABLE_FORMAT_VERSION),
     };
+    let records = operator_record_path(&state, BACKUP_RECORDS);
     let manifest = tokio::task::spawn_blocking(move || {
-        let _guard = guard;
-        feltdb_server::backup::create_bundle_with_source(
-            &output,
-            &artifacts,
-            &[("database.snapshot.json".to_string(), snapshot_bytes)],
-            source,
-        )
+        let manifest = {
+            let _guard = guard;
+            feltdb_server::backup::create_bundle_with_source(
+                &output,
+                &artifacts,
+                &[("database.snapshot.json".to_string(), snapshot_bytes)],
+                source,
+            )?
+        };
+        // Verified after the barrier is released: restoring into scratch and
+        // opening the result is what turns "a bundle exists" into "a bundle
+        // restores", and the record says which of the two is true.
+        record_online_backup(&records, &output, &manifest);
+        Ok::<_, std::io::Error>(manifest)
     })
     .await
     .map_err(|error| {
@@ -11006,28 +11527,258 @@ async fn create_online_backup(
 /// finding was that losing security evidence was invisible, and a signal an
 /// operator has to know to go and look for is not much better than no signal.
 /// It rides with the metrics they already read.
-async fn network_metrics(State(state): State<AppState>) -> Json<Value> {
-    let mut body = serde_json::to_value(state.metrics.snapshot()).unwrap_or_else(|_| json!({}));
-    if let Some(fields) = body.as_object_mut() {
-        fields.insert(
-            "audit".to_string(),
-            serde_json::to_value(state.audit.health()).unwrap_or(Value::Null),
-        );
-    }
-    Json(body)
+const BACKUP_RECORDS: &str = "backups.json";
+
+/// Operator records live beside the durable log (`<data>.<name>`), on the
+/// same volume, like the readiness probe they are derived from.
+fn operator_record_path(state: &AppState, name: &str) -> PathBuf {
+    state.readiness_probe.with_extension(name)
 }
 
-async fn sync_pull(
-    State(state): State<AppState>,
-    Json(request): Json<SyncPullRequest>,
-) -> Result<Json<SyncPullResponse>, ApiError> {
-    authorize_sync_member(&state, &request.requester, request.membership_epoch)?;
-    let peer_position = request.versions;
-    let authority_position = state.db.operation_versions()?;
-    let retained_from = state.db.retained_operation_floors()?;
-    let catchup = catchup_status(
-        request.requester,
-        peer_position.clone(),
+fn read_records(path: &std::path::Path) -> Vec<Value> {
+    std::fs::read(path)
+        .ok()
+        .and_then(|bytes| serde_json::from_slice::<Vec<Value>>(&bytes).ok())
+        .unwrap_or_default()
+}
+
+/// Append one record; bounded so the file cannot grow without limit.
+fn append_record(path: &std::path::Path, record: Value) -> std::io::Result<()> {
+    let mut records = read_records(path);
+    records.push(record);
+    let excess = records.len().saturating_sub(100);
+    records.drain(..excess);
+    let temporary = path.with_extension("json.tmp");
+    let mut file = std::fs::File::create(&temporary)?;
+    file.write_all(&serde_json::to_vec_pretty(&records).map_err(std::io::Error::other)?)?;
+    file.sync_all()?;
+    std::fs::rename(&temporary, path)
+}
+
+fn record_online_backup(
+    records: &std::path::Path,
+    output: &std::path::Path,
+    manifest: &feltdb_server::backup::BackupManifest,
+) {
+    let started = Instant::now();
+    let restore = verify_restorable(output).map_err(|error| error.to_string());
+    let record = json!({
+        "kind": "online_backup",
+        "bundle_id": feltdb_server::backup::backup_id(output).ok(),
+        "output": output.display().to_string(),
+        "created_ms": manifest.created_ms,
+        "files": manifest.entries.len(),
+        "verified": restore.is_ok(),
+        "restore_verification": match &restore {
+            Ok(detail) => json!({
+                "method": "bundle verified, restored into a scratch directory on this host and opened",
+                "verified_at_ms": unix_millis(),
+                "duration_ms": started.elapsed().as_millis() as u64,
+                "detail": detail,
+            }),
+            Err(error) => json!({ "method": "scratch restore", "error": error }),
+        },
+    });
+    if let Err(error) = append_record(records, record) {
+        tracing::error!(%error, "failed to record online backup");
+    }
+}
+
+fn unix_millis() -> u64 {
+    std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_millis() as u64
+}
+
+/// The routes authenticated by a shared service secret rather than an API
+/// key. They are routed only when deliberately configured, so an authority
+/// that is not part of a managed Intent Engine deployment or a certification
+/// run has no such surface at all.
+fn service_secret_routes() -> Router<AppState> {
+    let mut routes = Router::new();
+    let intent_secret = std::env::var("FELTDB_INTENT_SERVICE_SECRET").unwrap_or_default();
+    if intent_secret.len() >= 32 {
+        routes = routes
+            .route("/internal/intents/by-hash", get(internal_find_intent))
+            .route(
+                "/internal/intents",
+                axum::routing::post(internal_create_intent),
+            )
+            .route("/internal/intents/{id}", put(internal_replace_intent))
+            .route(
+                "/internal/intent-events",
+                axum::routing::post(internal_create_intent_event),
+            );
+    } else if !intent_secret.is_empty() {
+        tracing::warn!("FELTDB_INTENT_SERVICE_SECRET is shorter than 32 bytes; /internal/intents* is not routed");
+    }
+    let certification = matches!(
+        std::env::var("FELTDB_CERTIFICATION_MODE").as_deref(),
+        Ok("1" | "true")
+    );
+    if certification {
+        tracing::warn!("FELTDB_CERTIFICATION_MODE is set: certification fixture deletion is routed");
+        routes = routes.route(
+            "/api/certification/fixtures/{application_id}",
+            axum::routing::delete(delete_certification_fixture),
+        );
+    }
+    routes
+}
+
+/// A real write → fsync → read-back probe of the storage volume.
+fn storage_probe(path: &std::path::Path) -> Value {
+    let started = Instant::now();
+    let token = format!("feltdb-status-probe-{}\n", uuid::Uuid::new_v4().simple());
+    let mut wrote = false;
+    let mut synced = false;
+    let result = (|| -> std::io::Result<bool> {
+        if let Some(parent) = path.parent() {
+            std::fs::create_dir_all(parent)?;
+        }
+        let mut file = OpenOptions::new()
+            .create(true)
+            .write(true)
+            .truncate(true)
+            .open(path)?;
+        file.write_all(token.as_bytes())?;
+        wrote = true;
+        file.sync_all()?;
+        synced = true;
+        Ok(std::fs::read(path)? == token.as_bytes())
+    })();
+    let read_back = matches!(result, Ok(true));
+    json!({
+        "ok": read_back,
+        "write": wrote,
+        "fsync": synced,
+        "read_back": read_back,
+        "checked_at_ms": unix_millis(),
+        "latency_us": started.elapsed().as_micros() as u64,
+        "error": result.err().map(|error| error.to_string()),
+    })
+}
+
+/// `GET /v1/status` — the operator's view, needs `operator:read`.
+///
+/// Every field is observed by this request or read from a record this server
+/// wrote when the event happened; nothing is asserted. Availability (this
+/// process answers and its volume accepts a synced write) is reported apart
+/// from verified durability (a backup of this state was restored and opened).
+async fn operator_status(State(state): State<AppState>) -> Result<Json<Value>, ApiError> {
+    let probe_path = operator_record_path(&state, "status-probe");
+    let backups_path = operator_record_path(&state, BACKUP_RECORDS);
+    let (probe, backups) = tokio::task::spawn_blocking(move || {
+        (storage_probe(&probe_path), read_records(&backups_path))
+    })
+    .await
+    .map_err(|error| {
+        tracing::error!(%error, "status worker failed");
+        ApiError(StatusCode::SERVICE_UNAVAILABLE, "status check temporarily unavailable".into())
+    })?;
+    let database = state.db.health();
+    let last_backup = backups
+        .iter()
+        .rev()
+        .find(|record| record["kind"] == "online_backup")
+        .map(|record| {
+            json!({
+                "bundle_id": record["bundle_id"],
+                "created_ms": record["created_ms"],
+                "output": record["output"],
+                "verified": record["verified"],
+            })
+        });
+    let last_verified_restore = backups
+        .iter()
+        .rev()
+        .find(|record| record["verified"] == true)
+        .map(|record| {
+            json!({
+                "bundle_id": record["bundle_id"],
+                "verified_at_ms": record["restore_verification"]["verified_at_ms"],
+                "method": record["restore_verification"]["method"],
+            })
+        });
+    let clock = state.lease_clock.is_healthy();
+    let membership_healthy = !matches!(
+        state.cluster.proposal().map(|value| value.phase),
+        Some(ProposalPhase::Preparing | ProposalPhase::Prepared | ProposalPhase::Aborted)
+    );
+    let available = probe["ok"] == true;
+    Ok(Json(json!({
+        "status": if available && overall_status(membership_healthy, clock, &database.storage) == "healthy" { "available" } else { "degraded" },
+        "release": {
+            "version": env!("CARGO_PKG_VERSION"),
+            "release_id": option_env!("FELTDB_RELEASE_ID").unwrap_or("unknown"),
+            "git_commit": option_env!("FELTDB_GIT_COMMIT").unwrap_or("unknown"),
+            "build_timestamp": option_env!("FELTDB_BUILD_TIMESTAMP").unwrap_or("unknown"),
+            "deployment_id": std::env::var("FELTDB_DEPLOYMENT_ID").or_else(|_| std::env::var("FLY_IMAGE_REF")).ok(),
+            "promoted_revision": std::env::var("FELTDB_PROMOTED_REVISION").ok(),
+        },
+        "capabilities": {
+            "public_contract": "feltdb.public.substrate@1.0",
+            "capability_fingerprint": control_plane::public_contract_capability_fingerprint(),
+            "protocol_version": PROTOCOL_VERSION,
+            "authentication": state.auth_enabled,
+            "autonomous_workers": state.worker_enabled,
+            "configured_peers": state.cluster.get().peers.len(),
+            "durable_work_authority": is_work_authority(&state),
+        },
+        "availability": {
+            "serving": true,
+            "storage_writable": available,
+            "clock": if clock { "healthy" } else { "unhealthy" },
+            "membership": if membership_healthy { "healthy" } else { "recovering-or-attention-required" },
+        },
+        "storage": {
+            "replay": database.storage.label(),
+            "durable_format": database.durable_format.to_string(),
+            "probe": probe,
+        },
+        "backups": {
+            "last": last_backup,
+            "recorded": backups.len(),
+        },
+        "restore": {
+            "last_verified": last_verified_restore,
+            "status": if last_verified_restore.is_some() { "verified" } else { "never_verified" },
+            "off_host": "not observable by this server",
+        },
+        "durability": {
+            "verified": last_verified_restore.is_some(),
+            "basis": if last_verified_restore.is_some() {
+                "a backup of this authority was restored into scratch on this host and opened; off-host recovery and power-loss durability are not proven by this endpoint"
+            } else {
+                "no backup of this authority has been restored and verified; availability is not durability"
+            },
+        },
+    })))
+}
+
+async fn network_metrics(State(state): State<AppState>) -> Json<Value> {
+    let mut body = serde_json::to_value(state.metrics.snapshot()).unwrap_or_else(|_| json!({}));
+    if let Some(fields) = body.as_object_mut() {
+        fields.insert(
+            "audit".to_string(),
+            serde_json::to_value(state.audit.health()).unwrap_or(Value::Null),
+        );
+    }
+    Json(body)
+}
+
+async fn sync_pull(
+    State(state): State<AppState>,
+    Json(request): Json<SyncPullRequest>,
+) -> Result<Json<SyncPullResponse>, ApiError> {
+    authorize_sync_member(&state, &request.requester, request.membership_epoch)?;
+    let peer_position = request.versions;
+    let authority_position = state.db.operation_versions()?;
+    let retained_from = state.db.retained_operation_floors()?;
+    let catchup = catchup_status(
+        request.requester,
+        peer_position.clone(),
         authority_position.clone(),
         retained_from,
     );
@@ -11048,6 +11799,7 @@ async fn sync_pull(
         operations,
         versions: authority_position,
         catchup,
+        durable_work_authority: state.work_authority,
     }))
 }
 
@@ -11198,6 +11950,40 @@ fn start_autonomous_worker(state: &AppState) {
     });
 }
 
+/// Recover Durable Work whose lease expired, on every node, whether or not it
+/// runs the autonomous worker: external workers depend on it too.
+///
+/// It is the mesh's claim recovery (`recover_expired_work`, the part of
+/// `detect_failures` that recovers work): expired work goes to
+/// `WorkloadStore::recover_expired`, bounded by each workload's attempt budget,
+/// and its claim is released from the worker that held it. Worker liveness —
+/// marking silent workers offline — is deliberately not run here; it would lock
+/// out registered workers that do not heartbeat on the mesh's schedule. It takes
+/// the mesh lock before the workload lock, the order the claim route uses.
+fn start_workload_recovery(state: &AppState) {
+    let state = state.clone();
+    tokio::spawn(async move {
+        loop {
+            if state.cluster.get().is_member && is_work_authority(&state) {
+                let outcome = match (state.mesh.lock(), state.workloads.lock()) {
+                    // Deadlines first: work past one is timed out under its
+                    // own decision, not recovered as a lost worker.
+                    (Ok(mut mesh), Ok(mut workloads)) => mesh
+                        .enforce_deadlines(unix_seconds_i64(), &mut workloads)
+                        .and_then(|_| mesh.recover_expired_work(unix_seconds_i64(), &mut workloads))
+                        .map(|_| ())
+                        .map_err(|error| error.message),
+                    _ => Err("worker mesh or workload store unavailable".to_string()),
+                };
+                if let Err(error) = outcome {
+                    tracing::warn!(%error, "durable work recovery pass incomplete");
+                }
+            }
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        }
+    });
+}
+
 fn start_membership_recovery(state: &AppState) {
     let state = state.clone();
     tokio::spawn(async move {
@@ -11218,13 +12004,18 @@ async fn run_worker_pass(state: &AppState) -> Result<(), ApiError> {
         return Ok(());
     }
     let worker = state.db.instance_id()?;
+    // One run's failure never stops the pass: each is recorded on its own run,
+    // and an infrastructure fault is logged and retried on the next pass.
     for row in state.db.list_collection("_flow_workflow_runs")? {
+        if row.value["status"] != "running" {
+            continue;
+        }
         let Some(steps) = row.value["steps"].as_array() else {
             continue;
         };
         let now = lease_now_ms(state)?;
         let Some(step) = steps.iter().find(|step| {
-            step["status"] == "ready"
+            (step["status"] == "ready" && step["not_before_ms"].as_u64().unwrap_or(0) <= now)
                 || (step["status"] == "running"
                     && step["lease_expires_ms"].as_u64().unwrap_or(0) <= now)
         }) else {
@@ -11233,46 +12024,41 @@ async fn run_worker_pass(state: &AppState) -> Result<(), ApiError> {
         let Some(step_id) = step["id"].as_str() else {
             continue;
         };
+        // A step that names no capability here is an external step, claimed
+        // by an external worker through the claim route.
         if !is_executable_capability(state, step_id)? {
             continue;
         }
         let run_id = row.value["id"].as_str().unwrap_or_default().to_string();
-        let input = row.value["input"].clone();
-        let claimed = transition_workflow_step(
-            state,
-            &run_id,
-            step_id,
-            Some((worker.clone(), 30_000)),
-            None,
-        )
-        .await?;
-        let claim_id = claimed.0.value["steps"]
-            .as_array()
-            .and_then(|steps| steps.iter().find(|step| step["id"] == step_id))
-            .and_then(|step| step["claim_id"].as_str())
-            .ok_or_else(|| {
-                ApiError(
-                    StatusCode::CONFLICT,
-                    "worker claim was not materialized".to_string(),
-                )
-            })?
-            .to_string();
-        let result = execute_named_capability(state, step_id, &input)?;
-        let _ = transition_workflow_step(state, &run_id, step_id, None, Some((claim_id, result)))
-            .await?;
+        let attempts = step["attempt"].as_u64().unwrap_or(0);
+        if let Err(error) =
+            execute_workflow_step(state, &row.value, &run_id, step_id, attempts, &worker).await
+        {
+            tracing::warn!(run = %run_id, step = %step_id, status = error.0.as_u16(), "workflow step attempt did not finish");
+        }
     }
 
     for row in state.db.list_collection("_flow_agent_runs")? {
-        let reclaimable = row.value["status"] == "running"
-            && row.value["lease_expires_ms"].as_u64().unwrap_or(0) <= lease_now_ms(state)?;
-        if row.value["status"] != "ready" && !reclaimable {
+        let now = lease_now_ms(state)?;
+        let status = row.value["status"].as_str().unwrap_or_default();
+        let lease_expired = row.value["lease_expires_ms"].as_u64().unwrap_or(0) <= now;
+        if !agent_execution::agent_run_claimable(status, lease_expired)
+            || row.value["not_before_ms"].as_u64().unwrap_or(0) > now
+        {
             continue;
         }
-        execute_agent_run(state, &row.value, &worker).await?;
+        if let Err(error) = execute_agent_run(state, &row.value, &worker).await {
+            let run = row.value["id"].as_str().unwrap_or_default();
+            tracing::warn!(%run, status = error.0.as_u16(), "agent run attempt did not finish");
+        }
     }
 
-    evaluate_state_triggers(state)?;
-    execute_eligible_workloads(state, &worker)?;
+    // Triggers admit work and the pass claims it: Durable Work writes, so
+    // only on the authority.
+    if is_work_authority(state) {
+        evaluate_state_triggers(state)?;
+        execute_eligible_workloads(state, &worker)?;
+    }
     Ok(())
 }
 
@@ -11372,7 +12158,7 @@ fn worker_capabilities(state: &AppState) -> Result<BTreeSet<String>, ApiError> {
 fn execute_eligible_workloads(state: &AppState, worker: &str) -> Result<(), ApiError> {
     let capabilities = worker_capabilities(state)?;
     let now = unix_seconds_i64();
-    let candidates: Vec<(String, String, Value)> = {
+    let candidates: Vec<(String, String, Value, feltdb::workload::Workload)> = {
         let store = workload_store(state)?;
         store
             .workloads
@@ -11391,12 +12177,13 @@ fn execute_eligible_workloads(state: &AppState, worker: &str) -> Result<(), ApiE
                     workload.workload_id.clone(),
                     workload.definition_id.clone(),
                     workload.input.clone(),
+                    workload.clone(),
                 )
             })
             .collect()
     };
 
-    for (id, definition, input) in candidates {
+    for (id, definition, input, workload) in candidates {
         // Unlike the filter above, this one is enforcement: nothing downstream
         // knows whether `definition_id` names something runnable. Claiming a
         // workload only to dead-letter it would consume work another node could
@@ -11427,7 +12214,13 @@ fn execute_eligible_workloads(state: &AppState, worker: &str) -> Result<(), ApiE
             )
             .map_err(workload_error)?;
 
-        match execute_named_capability(state, &definition, &input) {
+        let access = CapabilityAccess::Workload {
+            requested_by: &workload.requested_by,
+            tenant_id: &workload.tenant_id,
+            application_id: &workload.application_id,
+            grant: &workload.authorization_snapshot,
+        };
+        match execute_named_capability(state, &definition, &input, &access) {
             Ok(output) => {
                 workload_store(state)?
                     .complete(
@@ -11464,6 +12257,157 @@ fn execute_eligible_workloads(state: &AppState, worker: &str) -> Result<(), ApiE
     Ok(())
 }
 
+/// Debug builds only: `FELTDB_TEST_TRANSIENT_CAPABILITY=<name>[:<count>]` makes
+/// executing `<name>` fail with a server error — every time, or for its first
+/// `<count>` executions — so retry bounds can be exercised at the HTTP
+/// boundary. Compiled out of release builds, like `FELTDB_TEST_CLOCK_OFFSET_MS`.
+#[cfg(debug_assertions)]
+fn injected_capability_fault(name: &str) -> Option<ApiError> {
+    static INJECTED: AtomicU64 = AtomicU64::new(0);
+    let setting = std::env::var("FELTDB_TEST_TRANSIENT_CAPABILITY").ok()?;
+    let (target, limit) = match setting.split_once(':') {
+        Some((target, count)) => (target.to_string(), count.parse::<u64>().ok()),
+        None => (setting, None),
+    };
+    if target != name {
+        return None;
+    }
+    let injected = INJECTED.fetch_add(1, Ordering::SeqCst);
+    if limit.is_some_and(|limit| injected >= limit) {
+        return None;
+    }
+    Some(ApiError(
+        StatusCode::SERVICE_UNAVAILABLE,
+        "injected transient capability failure".into(),
+    ))
+}
+
+/// The authority a capability reads under.
+///
+/// Every collection a capability touches is authorized against it in
+/// [`authorize_capability_read`], the single point all capability reads pass
+/// through, so no route, program step, workflow, or agent reaches state with
+/// more authority than the principal it acts for.
+enum CapabilityAccess<'a> {
+    /// The request's own principal, on the direct capability routes.
+    Principal(&'a Principal),
+    /// An application actor on the `/v1` Service API.
+    Application {
+        actor: &'a str,
+        application_id: &'a str,
+    },
+    /// A run's verified delegated authority, on the autonomous worker.
+    Delegated(&'a RunPrincipal),
+    /// A durable workload's signed authorization snapshot. Reads are allowed
+    /// only when that grant, re-verified at execution, authorizes `state:read`
+    /// for the workload's requester — so a workload never reads with the
+    /// node's authority, and a row injected without a server-signed grant
+    /// reads nothing.
+    Workload {
+        requested_by: &'a GrantSubject,
+        tenant_id: &'a str,
+        application_id: &'a str,
+        grant: &'a feltdb::authorization::SignedGrant,
+    },
+}
+
+/// Whether a principal, described the way the authenticator describes it, holds
+/// `capability`. The same decision the request middleware makes.
+fn principal_grants(
+    state: &AppState,
+    kind: &str,
+    id: &str,
+    scopes: &[String],
+    capability: &str,
+) -> bool {
+    let subject = if kind == "human" {
+        Subject::User(id.to_string())
+    } else {
+        Subject::Service(id.to_string())
+    };
+    authorize(&AuthorizationRequest {
+        subject: &subject,
+        tenant_id: &state.namespace,
+        application_id: Some(&state.namespace),
+        capability,
+        direct_grants: scopes,
+        memberships: &[],
+        roles: &[],
+    })
+    .allowed
+}
+
+fn request_principal_grants(state: &AppState, principal: &Principal, capability: &str) -> bool {
+    principal_grants(
+        state,
+        &principal.subject_type,
+        &principal.key_id,
+        &principal.scopes,
+        capability,
+    )
+}
+
+fn delegated_grants(state: &AppState, principal: &RunPrincipal, capability: &str) -> bool {
+    let scopes: Vec<String> = principal.scopes.iter().cloned().collect();
+    principal_grants(state, &principal.kind, &principal.id, &scopes, capability)
+}
+
+fn capability_read_denied(message: String) -> ApiError {
+    ApiError::structured(
+        StatusCode::FORBIDDEN,
+        json!({"code": agent_execution::AUTHORIZATION_DENIED, "message": message}),
+    )
+}
+
+/// Authorize one collection read made by a capability.
+///
+/// Reading records through a capability is reading state, so it needs the
+/// `state:read` a direct read needs. System collections (`_`-prefixed) hold
+/// runs, definitions and other server records and are never readable this way.
+fn authorize_capability_read(
+    state: &AppState,
+    access: &CapabilityAccess<'_>,
+    collection: &str,
+) -> Result<(), ApiError> {
+    if agent_execution::is_system_collection(collection) {
+        return Err(capability_read_denied(format!(
+            "capabilities cannot read system collection {collection}"
+        )));
+    }
+    let allowed = match access {
+        CapabilityAccess::Principal(principal) => {
+            request_principal_grants(state, principal, "state:read")
+        }
+        CapabilityAccess::Application {
+            actor,
+            application_id,
+        } => application_scope(state, actor, application_id, "state:read").is_ok(),
+        CapabilityAccess::Delegated(principal) => delegated_grants(state, principal, "state:read"),
+        CapabilityAccess::Workload {
+            requested_by,
+            tenant_id,
+            application_id,
+            grant,
+        } => workload_decision(
+            state,
+            (*requested_by).clone(),
+            tenant_id,
+            application_id,
+            format!("flow://{application_id}/state/{collection}"),
+            "state:read",
+            grant,
+        )
+        .is_ok(),
+    };
+    if allowed {
+        Ok(())
+    } else {
+        Err(capability_read_denied(format!(
+            "the caller is not authorized to read {collection}"
+        )))
+    }
+}
+
 fn is_builtin_capability(name: &str) -> bool {
     matches!(name, "search" | "identity")
 }
@@ -11480,6 +12424,7 @@ fn execute_builtin_capability(
     state: &AppState,
     name: &str,
     input: &Value,
+    access: &CapabilityAccess<'_>,
 ) -> Result<Value, ApiError> {
     match name {
         "identity" => Ok(input.clone()),
@@ -11491,6 +12436,7 @@ fn execute_builtin_capability(
                 )
             })?;
             validate_segment(collection)?;
+            authorize_capability_read(state, access, collection)?;
             let needle = input["query"].as_str().unwrap_or_default().to_lowercase();
             let limit = input["limit"].as_u64().unwrap_or(50).min(500) as usize;
             Ok(Value::Array(
@@ -11511,80 +12457,369 @@ fn execute_builtin_capability(
     }
 }
 
+/// The text of an [`ApiError`] without its wire envelope: the `message` of a
+/// structured error, or the plain message.
+fn api_error_message(error: &ApiError) -> String {
+    serde_json::from_str::<Value>(&error.1)
+        .ok()
+        .and_then(|value| value["message"].as_str().map(str::to_string))
+        .unwrap_or_else(|| error.1.clone())
+}
+
+/// Everything an agent run needs before the worker may lease it.
+///
+/// Checked on every attempt, so authority revoked after the run was created,
+/// a definition removed, or a capability unregistered ends the run here,
+/// before anything executes.
+fn prepare_agent_run(
+    state: &AppState,
+    run: &Value,
+    id: &str,
+    attempt: u64,
+) -> Result<(String, RunPrincipal, Option<CompletionCriteria>), RunFailure> {
+    if attempt > agent_execution::max_run_attempts() {
+        return Err(RunFailure::permanent(
+            agent_execution::RETRY_EXHAUSTED,
+            format!(
+                "run was claimed {} times without finishing",
+                attempt - 1
+            ),
+        ));
+    }
+    let principal = resolve_run_principal(state, run, RunKind::Agent, id)?;
+    if !delegated_grants(state, &principal, "agents:run") {
+        return Err(RunFailure::permanent(
+            agent_execution::AUTHORIZATION_DENIED,
+            "the run's principal may not run agents",
+        ));
+    }
+    let agent = run["agent"].as_str().unwrap_or_default();
+    let definition = match validate_segment(agent)
+        .and_then(|_| required_value(state, "_flow_agents", agent, "agent not found"))
+    {
+        Ok(definition) => definition,
+        Err(_) => {
+            return Err(RunFailure::permanent(
+                agent_execution::AGENT_NOT_FOUND,
+                "the run's agent is not defined",
+            ))
+        }
+    };
+    let capability = agent_capability(state, &definition)?;
+    let criteria = match &run["completion"] {
+        Value::Null => None,
+        value => Some(
+            serde_json::from_value::<CompletionCriteria>(value.clone())
+                .ok()
+                .filter(|criteria| criteria.validate().is_ok())
+                .ok_or_else(|| {
+                    RunFailure::permanent(
+                        agent_execution::RUN_INVALID,
+                        "the run's completion criteria are invalid",
+                    )
+                })?,
+        ),
+    };
+    Ok((capability, principal, criteria))
+}
+
+fn mark_agent_run_failed(run: &mut Value, failure: &RunFailure, attempt: u64) {
+    run["status"] = json!(agent_execution::STATUS_FAILED);
+    run["failure"] = failure.to_record(attempt);
+    run["attempt"] = json!(attempt);
+}
+
+/// Execute one attempt of an agent run.
+///
+/// The run completes only when its declared criteria hold for the result this
+/// worker observed. Every other outcome is explicit: a permanent error fails
+/// the run, a transient one returns it to `ready` after a backoff until the
+/// attempt bound, and a run without criteria fails as unverifiable. An `Err`
+/// is an infrastructure fault that left the run untouched.
 async fn execute_agent_run(state: &AppState, run: &Value, worker: &str) -> Result<(), ApiError> {
     let id = run["id"]
         .as_str()
         .ok_or_else(|| ApiError(StatusCode::CONFLICT, "agent run has no id".to_string()))?;
-    let agent_name = run["agent"].as_str().ok_or_else(|| {
-        ApiError(
-            StatusCode::CONFLICT,
-            "agent run has no definition".to_string(),
-        )
-    })?;
-    let definition = required_value(state, "_flow_agents", agent_name, "agent not found")?;
-    let capability = definition["capabilities"].as_array().and_then(|values| {
-        values
-            .iter()
-            .filter_map(Value::as_str)
-            .find(|name| is_executable_capability(state, name).unwrap_or(false))
-    });
-    let Some(capability) = capability else {
-        return Ok(());
-    };
-    let claim_id = next_id(state, "claim");
-    let expires_ms = lease_now_ms(state)?.saturating_add(30_000);
-    let lease = LeaseRequest {
-        claim_id: claim_id.clone(),
-        owner: worker.to_string(),
-        expires_ms,
-        membership_epoch: state.cluster.get().epoch,
-    };
-    {
+    let (capability, principal, criteria, claim_id, attempt, input) = {
         let _guard = state.lifecycle_lock.lock().await;
-        reserve_quorum_lease(state, &format!("agent-{id}"), &lease).await?;
         let mut current = required_value(state, "_flow_agent_runs", id, "agent run not found")?;
-        let reclaimable = current["status"] == "running"
-            && current["lease_expires_ms"].as_u64().unwrap_or(0) <= lease_now_ms(state)?;
-        if current["status"] != "ready" && !reclaimable {
+        let now = lease_now_ms(state)?;
+        let status = current["status"].as_str().unwrap_or_default();
+        let lease_expired = current["lease_expires_ms"].as_u64().unwrap_or(0) <= now;
+        if !agent_execution::agent_run_claimable(status, lease_expired)
+            || current["not_before_ms"].as_u64().unwrap_or(0) > now
+        {
             return Ok(());
         }
-        current["status"] = json!("running");
+        let attempt = current["attempt"].as_u64().unwrap_or(0) + 1;
+        let (capability, principal, criteria) =
+            match prepare_agent_run(state, &current, id, attempt) {
+                Ok(prepared) => prepared,
+                Err(failure) => {
+                    mark_agent_run_failed(&mut current, &failure, attempt);
+                    put_canonical(state, "_flow_agent_runs", id, current)?;
+                    return Ok(());
+                }
+            };
+        let claim_id = next_id(state, "claim");
+        let expires_ms = now.saturating_add(30_000);
+        let lease = LeaseRequest {
+            claim_id: claim_id.clone(),
+            owner: worker.to_string(),
+            expires_ms,
+            membership_epoch: state.cluster.get().epoch,
+        };
+        reserve_quorum_lease(state, &format!("agent-{id}"), &lease).await?;
+        current["status"] = json!(agent_execution::STATUS_RUNNING);
         current["worker"] = json!(worker);
         current["claim_id"] = json!(claim_id);
         current["lease_expires_ms"] = json!(expires_ms);
-        let _ = put_canonical(state, "_flow_agent_runs", id, current)?;
-    }
-    let result = execute_named_capability(state, capability, &run["input"])?;
+        current["attempt"] = json!(attempt);
+        let input = current["input"].clone();
+        put_canonical(state, "_flow_agent_runs", id, current)?;
+        (capability, principal, criteria, claim_id, attempt, input)
+    };
+
+    let outcome = execute_named_capability(
+        state,
+        &capability,
+        &input,
+        &CapabilityAccess::Delegated(&principal),
+    );
+
     let _guard = state.lifecycle_lock.lock().await;
     let mut current = required_value(state, "_flow_agent_runs", id, "agent run not found")?;
-    if current["claim_id"] != claim_id
-        || current["lease_expires_ms"].as_u64().unwrap_or(0) <= lease_now_ms(state)?
+    let now = lease_now_ms(state)?;
+    // Publish only while this claim still owns the run. A claim that lost its
+    // lease publishes nothing; the next claim counts as another attempt.
+    if current["claim_id"] != claim_id.as_str()
+        || current["status"] != agent_execution::STATUS_RUNNING
+        || current["lease_expires_ms"].as_u64().unwrap_or(0) <= now
     {
-        return Err(ApiError(
-            StatusCode::CONFLICT,
-            "agent ownership expired before result publication".to_string(),
-        ));
+        return Ok(());
+    }
+    match outcome {
+        Ok(result) => {
+            let satisfied = criteria
+                .as_ref()
+                .map(|criteria| criteria.satisfied_by(&result));
+            let digest = format!(
+                "sha256:{:x}",
+                Sha256::digest(serde_json::to_vec(&result).unwrap_or_default())
+            );
+            let evidence = json!({
+                "source": "runtime", "attempt": attempt, "claim_id": claim_id,
+                "capability": capability, "result_items": result.as_array().map(Vec::len),
+                "result_digest": digest, "criteria": criteria, "satisfied": satisfied,
+                "observed_at_ms": now,
+            });
+            for (field, value) in [
+                ("observations", result.clone()),
+                (
+                    "decisions",
+                    json!({"capability": capability, "result": result, "source": "runtime"}),
+                ),
+                ("evidence", evidence),
+            ] {
+                if !current[field].is_array() {
+                    current[field] = json!([]);
+                }
+                current[field]
+                    .as_array_mut()
+                    .expect("normalized to an array")
+                    .push(value);
+            }
+            match satisfied {
+                Some(true) => {
+                    current["status"] = json!(agent_execution::STATUS_COMPLETED);
+                    current["completed_at_ms"] = json!(now);
+                }
+                Some(false) => mark_agent_run_failed(
+                    &mut current,
+                    &RunFailure::permanent(
+                        agent_execution::COMPLETION_CRITERIA_UNMET,
+                        "the observed result does not satisfy the run's completion criteria",
+                    ),
+                    attempt,
+                ),
+                None => mark_agent_run_failed(
+                    &mut current,
+                    &RunFailure::permanent(
+                        agent_execution::COMPLETION_UNVERIFIABLE,
+                        "the run declares no completion criteria the runtime can verify; its goal text is not evaluated",
+                    ),
+                    attempt,
+                ),
+            }
+        }
+        Err(error) => {
+            let failure = agent_execution::classify_capability_error(
+                error.0.as_u16(),
+                &api_error_message(&error),
+            );
+            if failure.retryable && attempt < agent_execution::max_run_attempts() {
+                // The failed attempt's lease stays reserved until it lapses, so
+                // the retry waits for that as well as for the backoff: a new
+                // claim before then would be refused by the lease quorum.
+                let lease_expires = current["lease_expires_ms"].as_u64().unwrap_or(0);
+                current["status"] = json!(agent_execution::STATUS_READY);
+                current["not_before_ms"] = json!(now
+                    .saturating_add(agent_execution::retry_delay_ms(attempt + 1))
+                    .max(lease_expires));
+                current["last_failure"] = failure.to_record(attempt);
+            } else if failure.retryable {
+                mark_agent_run_failed(
+                    &mut current,
+                    &RunFailure::permanent(
+                        agent_execution::RETRY_EXHAUSTED,
+                        format!("{} after {attempt} attempts", failure.message),
+                    ),
+                    attempt,
+                );
+            } else {
+                mark_agent_run_failed(&mut current, &failure, attempt);
+            }
+        }
     }
-    current["observations"]
+    put_canonical(state, "_flow_agent_runs", id, current)?;
+    Ok(())
+}
+
+/// Fail a workflow step and its run. `claim_id`, when given, must still own
+/// the step; otherwise another claim has taken over and nothing changes.
+async fn fail_workflow_step(
+    state: &AppState,
+    run_id: &str,
+    step_id: &str,
+    claim_id: Option<&str>,
+    failure: &RunFailure,
+) -> Result<(), ApiError> {
+    let _guard = state.lifecycle_lock.lock().await;
+    let mut run = required_value(state, "_flow_workflow_runs", run_id, "workflow run not found")?;
+    if run["status"] != "running" {
+        return Ok(());
+    }
+    let Some(step) = run["steps"]
         .as_array_mut()
-        .ok_or_else(|| {
-            ApiError(
-                StatusCode::CONFLICT,
-                "agent observations are invalid".to_string(),
-            )
-        })?
-        .push(result.clone());
-    current["decisions"]
+        .and_then(|steps| steps.iter_mut().find(|step| step["id"] == step_id))
+    else {
+        return Ok(());
+    };
+    if claim_id.is_some_and(|claim| step["claim_id"] != claim) {
+        return Ok(());
+    }
+    let attempt = step["attempt"].as_u64().unwrap_or(0);
+    step["status"] = json!(agent_execution::STATUS_FAILED);
+    step["failure"] = failure.to_record(attempt);
+    run["status"] = json!(agent_execution::STATUS_FAILED);
+    run["failure"] = json!({"step": step_id, "code": failure.code, "message": failure.message});
+    put_canonical(state, "_flow_workflow_runs", run_id, run)?;
+    Ok(())
+}
+
+/// Return a step that failed transiently to `ready` after a backoff.
+async fn release_workflow_step(
+    state: &AppState,
+    run_id: &str,
+    step_id: &str,
+    claim_id: &str,
+    failure: &RunFailure,
+) -> Result<(), ApiError> {
+    let _guard = state.lifecycle_lock.lock().await;
+    let mut run = required_value(state, "_flow_workflow_runs", run_id, "workflow run not found")?;
+    let now = lease_now_ms(state)?;
+    let Some(step) = run["steps"]
         .as_array_mut()
+        .and_then(|steps| steps.iter_mut().find(|step| step["id"] == step_id))
+    else {
+        return Ok(());
+    };
+    if step["claim_id"] != claim_id || step["status"] != "running" {
+        return Ok(());
+    }
+    let attempt = step["attempt"].as_u64().unwrap_or(0);
+    let lease_expires = step["lease_expires_ms"].as_u64().unwrap_or(0);
+    step["status"] = json!(agent_execution::STATUS_READY);
+    // Not before the failed attempt's lease lapses: until then the lease
+    // quorum refuses any other claim.
+    step["not_before_ms"] = json!(now
+        .saturating_add(agent_execution::retry_delay_ms(attempt + 1))
+        .max(lease_expires));
+    step["last_failure"] = failure.to_record(attempt);
+    put_canonical(state, "_flow_workflow_runs", run_id, run)?;
+    Ok(())
+}
+
+/// Execute one autonomous workflow step under the run's delegated authority.
+async fn execute_workflow_step(
+    state: &AppState,
+    run: &Value,
+    run_id: &str,
+    step_id: &str,
+    attempts: u64,
+    worker: &str,
+) -> Result<(), ApiError> {
+    if attempts >= agent_execution::max_run_attempts() {
+        let failure = RunFailure::permanent(
+            agent_execution::RETRY_EXHAUSTED,
+            format!("step was claimed {attempts} times without finishing"),
+        );
+        return fail_workflow_step(state, run_id, step_id, None, &failure).await;
+    }
+    let principal = match resolve_run_principal(state, run, RunKind::Workflow, run_id) {
+        Ok(principal) if delegated_grants(state, &principal, "workflows:run") => principal,
+        Ok(_) => {
+            let failure = RunFailure::permanent(
+                agent_execution::AUTHORIZATION_DENIED,
+                "the run's principal may not run workflows",
+            );
+            return fail_workflow_step(state, run_id, step_id, None, &failure).await;
+        }
+        Err(failure) => return fail_workflow_step(state, run_id, step_id, None, &failure).await,
+    };
+    let claimed = transition_workflow_step(
+        state,
+        run_id,
+        step_id,
+        Some((worker.to_string(), 30_000)),
+        None,
+    )
+    .await?;
+    let claim_id = claimed.0.value["steps"]
+        .as_array()
+        .and_then(|steps| steps.iter().find(|step| step["id"] == step_id))
+        .and_then(|step| step["claim_id"].as_str())
         .ok_or_else(|| {
             ApiError(
                 StatusCode::CONFLICT,
-                "agent decisions are invalid".to_string(),
+                "worker claim was not materialized".to_string(),
             )
         })?
-        .push(json!({"capability": capability, "result": result}));
-    current["status"] = json!("completed");
-    let _ = put_canonical(state, "_flow_agent_runs", id, current)?;
+        .to_string();
+    let input = run["input"].clone();
+    match execute_named_capability(state, step_id, &input, &CapabilityAccess::Delegated(&principal))
+    {
+        Ok(result) => {
+            transition_workflow_step(state, run_id, step_id, None, Some((claim_id, result))).await?;
+        }
+        Err(error) => {
+            let failure = agent_execution::classify_capability_error(
+                error.0.as_u16(),
+                &api_error_message(&error),
+            );
+            if failure.retryable && attempts + 1 < agent_execution::max_run_attempts() {
+                release_workflow_step(state, run_id, step_id, &claim_id, &failure).await?;
+            } else if failure.retryable {
+                let exhausted = RunFailure::permanent(
+                    agent_execution::RETRY_EXHAUSTED,
+                    format!("{} after {} attempts", failure.message, attempts + 1),
+                );
+                fail_workflow_step(state, run_id, step_id, Some(&claim_id), &exhausted).await?;
+            } else {
+                fail_workflow_step(state, run_id, step_id, Some(&claim_id), &failure).await?;
+            }
+        }
+    }
     Ok(())
 }
 
@@ -11638,6 +12873,17 @@ async fn sync_with_peer(
         .json::<SyncPullResponse>()
         .await?;
 
+    // Two nodes configured as the Durable Work authority would each commit
+    // work the other cannot see in time; both refuse until one is restarted
+    // without `--work-authority`.
+    if state.work_authority && pull.durable_work_authority {
+        if !state
+            .work_authority_conflict
+            .swap(true, std::sync::atomic::Ordering::SeqCst)
+        {
+            tracing::error!(%peer, "two nodes are configured as the Durable Work authority; refusing Durable Work writes");
+        }
+    }
     let mut incoming = pull.operations;
     incoming.sort_by(|left, right| {
         left.instance_id
@@ -12563,10 +13809,12 @@ async fn get_provenance(
 
 async fn search_collection(
     State(state): State<AppState>,
+    Extension(principal): Extension<Principal>,
     Path(collection): Path<String>,
     Json(request): Json<SearchRequest>,
 ) -> Result<Json<Vec<RecordResponse>>, ApiError> {
     validate_segment(&collection)?;
+    authorize_capability_read(&state, &CapabilityAccess::Principal(&principal), &collection)?;
     if request.query.len() > 512 || request.limit == 0 || request.limit > 500 {
         return Err(ApiError(
             StatusCode::BAD_REQUEST,
@@ -12616,7 +13864,12 @@ async fn execute_capability(
         ));
     }
     validate_segment(&name)?;
-    Ok(Json(execute_named_capability(&state, &name, &input)?))
+    Ok(Json(execute_named_capability(
+        &state,
+        &name,
+        &input,
+        &CapabilityAccess::Principal(&principal),
+    )?))
 }
 
 async fn execute_application_capability(
@@ -12663,7 +13916,15 @@ async fn execute_application_capability(
             "capability is not externally invocable".into(),
         ));
     }
-    Ok(Json(execute_named_capability(&state, &name, &payload)?))
+    Ok(Json(execute_named_capability(
+        &state,
+        &name,
+        &payload,
+        &CapabilityAccess::Application {
+            actor: &principal.key_id,
+            application_id: &input.application_id,
+        },
+    )?))
 }
 
 fn validate_capability_program(program: &CapabilityProgram) -> Result<(), ApiError> {
@@ -12722,9 +13983,14 @@ fn execute_named_capability(
     state: &AppState,
     name: &str,
     input: &Value,
+    access: &CapabilityAccess<'_>,
 ) -> Result<Value, ApiError> {
+    #[cfg(debug_assertions)]
+    if let Some(fault) = injected_capability_fault(name) {
+        return Err(fault);
+    }
     if is_builtin_capability(name) {
-        return execute_builtin_capability(state, name, input);
+        return execute_builtin_capability(state, name, input, access);
     }
     let value = required_value(state, "_flow_capabilities", name, "capability not found")?;
     let program: CapabilityProgram = serde_json::from_value(value).map_err(|error| {
@@ -12758,6 +14024,7 @@ fn execute_named_capability(
                     state,
                     "search",
                     &json!({"collection": collection, "query": query, "limit": limit.unwrap_or(50)}),
+                    access,
                 )?
             }
             CapabilityInstruction::FilterEq {
@@ -12873,12 +14140,14 @@ async fn start_workflow_run(
     let step_states: Vec<Value> = steps.iter().enumerate().map(|(index, step)| json!({
         "id": step, "status": if index == 0 { "ready" } else { "pending" }, "worker": null, "result": null
     })).collect();
+    let (authority, attribution) = run_authority(&state, RunKind::Workflow, &id, &principal)?;
     let response = insert_canonical(
         &state,
         "_flow_workflow_runs",
         &id,
         json!({
-            "id": id, "workflow": name, "status": "running", "input": request.input, "steps": step_states
+            "id": id, "workflow": name, "status": "running", "input": request.input, "steps": step_states,
+            "authority": authority, "principal": attribution,
         }),
     )?;
     Ok((StatusCode::CREATED, Json(response)))
@@ -12886,9 +14155,12 @@ async fn start_workflow_run(
 
 async fn get_workflow_run(
     State(state): State<AppState>,
+    Extension(principal): Extension<Principal>,
     Path(id): Path<String>,
 ) -> Result<Json<RecordResponse>, ApiError> {
-    get_canonical(&state, "_flow_workflow_runs", &id, "workflow run not found")
+    let mut response = get_canonical(&state, "_flow_workflow_runs", &id, "workflow run not found")?;
+    redact_run_results(&state, &principal, &mut response.0.value);
+    Ok(response)
 }
 
 async fn claim_workflow_step(
@@ -12940,6 +14212,12 @@ async fn transition_workflow_step(
     validate_segment(step)?;
     let _guard = state.lifecycle_lock.lock().await;
     let mut run = required_value(state, "_flow_workflow_runs", id, "workflow run not found")?;
+    if run["status"] == agent_execution::STATUS_FAILED {
+        return Err(ApiError::structured(
+            StatusCode::CONFLICT,
+            json!({"code": "WORKFLOW_RUN_TERMINAL", "message": "workflow run has failed"}),
+        ));
+    }
     let steps = run["steps"]
         .as_array_mut()
         .ok_or_else(|| ApiError(StatusCode::CONFLICT, "workflow run is invalid".to_string()))?;
@@ -12968,10 +14246,12 @@ async fn transition_workflow_step(
             membership_epoch: state.cluster.get().epoch,
         };
         reserve_quorum_lease(state, &resource, &lease).await?;
+        let attempt = steps[index]["attempt"].as_u64().unwrap_or(0) + 1;
         steps[index]["status"] = json!("running");
         steps[index]["worker"] = json!(worker);
         steps[index]["claim_id"] = json!(claim_id);
         steps[index]["lease_expires_ms"] = json!(expires_ms);
+        steps[index]["attempt"] = json!(attempt);
     } else {
         if status != "running" {
             return Err(ApiError(
@@ -13029,31 +14309,169 @@ async fn start_agent_run(
         ));
     }
     validate_segment(&name)?;
-    required_value(&state, "_flow_agents", &name, "agent not found")?;
+    let definition = required_value(&state, "_flow_agents", &name, "agent not found")?;
     if request.goal.trim().is_empty() {
         return Err(ApiError(
             StatusCode::BAD_REQUEST,
             "agent goal is required".to_string(),
         ));
     }
+    if let Some(criteria) = &request.completion {
+        criteria.validate().map_err(|message| {
+            ApiError::structured(
+                StatusCode::BAD_REQUEST,
+                json!({"code": "COMPLETION_CRITERIA_INVALID", "message": message}),
+            )
+        })?;
+    }
+    // Refuse a run that could only ever fail, instead of creating one.
+    agent_capability(&state, &definition).map_err(|failure| {
+        ApiError::structured(
+            StatusCode::UNPROCESSABLE_ENTITY,
+            json!({"code": failure.code, "message": failure.message}),
+        )
+    })?;
     let id = next_id(&state, "agent");
+    let (authority, attribution) = run_authority(&state, RunKind::Agent, &id, &principal)?;
     let response = insert_canonical(
         &state,
         "_flow_agent_runs",
         &id,
         json!({
             "id": id, "agent": name, "goal": request.goal, "input": request.input,
-            "status": "ready", "observations": [], "decisions": []
+            "status": agent_execution::STATUS_READY, "observations": [], "decisions": [],
+            "completion": request.completion, "evidence": [], "attempt": 0,
+            "authority": authority, "principal": attribution,
         }),
     )?;
     Ok((StatusCode::CREATED, Json(response)))
 }
 
+/// Sign the requesting principal's authority into a new run, and the
+/// attribution recorded beside it.
+fn run_authority(
+    state: &AppState,
+    kind: RunKind,
+    run_id: &str,
+    principal: &Principal,
+) -> Result<(Value, Value), ApiError> {
+    let run_principal = RunPrincipal {
+        id: principal.key_id.clone(),
+        kind: principal.subject_type.clone(),
+        scopes: principal.scopes.iter().cloned().collect(),
+    };
+    let grant = agent_execution::issue_run_authority(
+        &state.grant_signer,
+        kind,
+        run_id,
+        &state.namespace,
+        &run_principal,
+        unix_seconds_i64(),
+    )
+    .map_err(|message| ApiError(StatusCode::INTERNAL_SERVER_ERROR, message))?;
+    let authority = serde_json::to_value(grant)
+        .map_err(|error| ApiError(StatusCode::INTERNAL_SERVER_ERROR, error.to_string()))?;
+    Ok((
+        authority,
+        json!({"id": principal.key_id, "kind": principal.subject_type}),
+    ))
+}
+
+/// The capability an agent run executes: the first one its definition
+/// declares. It must be executable now; one that is not is a permanent error,
+/// never a run left waiting for a capability that does not exist.
+fn agent_capability(state: &AppState, definition: &Value) -> Result<String, RunFailure> {
+    let Some(name) = definition["capabilities"]
+        .as_array()
+        .and_then(|values| values.first())
+        .and_then(Value::as_str)
+    else {
+        return Err(RunFailure::permanent(
+            agent_execution::CAPABILITY_UNKNOWN,
+            "agent declares no capability",
+        ));
+    };
+    match is_executable_capability(state, name) {
+        Ok(true) => Ok(name.to_string()),
+        _ => Err(RunFailure::permanent(
+            agent_execution::CAPABILITY_UNKNOWN,
+            format!("capability {name} is not defined on this node"),
+        )),
+    }
+}
+
+/// The principal a run may act for at execution time.
+///
+/// The signed authority binds scopes to the run. When authentication is on,
+/// the key is also re-read, and only scopes it still holds remain: a revoked
+/// key acts for nothing, and a narrowed key never acts with its old breadth.
+fn resolve_run_principal(
+    state: &AppState,
+    run: &Value,
+    kind: RunKind,
+    run_id: &str,
+) -> Result<RunPrincipal, RunFailure> {
+    let mut principal = agent_execution::verify_run_authority(
+        &state.grant_signer,
+        &run["authority"],
+        kind,
+        run_id,
+        &state.namespace,
+        unix_seconds_i64(),
+    )?;
+    if state.auth_enabled && principal.kind != "human" {
+        let Some((current, current_kind)) = state.keys.active_scopes(&principal.id, &state.namespace)
+        else {
+            return Err(RunFailure::permanent(
+                agent_execution::AUTHORIZATION_DENIED,
+                "the run's principal is no longer active",
+            ));
+        };
+        if current_kind != principal.kind {
+            return Err(RunFailure::permanent(
+                agent_execution::AUTHORIZATION_DENIED,
+                "the run's principal changed type",
+            ));
+        }
+        principal.scopes =
+            agent_execution::effective_scopes(&principal.scopes, &current.into_iter().collect());
+    }
+    Ok(principal)
+}
+
 async fn get_agent_run(
     State(state): State<AppState>,
+    Extension(principal): Extension<Principal>,
     Path(id): Path<String>,
 ) -> Result<Json<RecordResponse>, ApiError> {
-    get_canonical(&state, "_flow_agent_runs", &id, "agent run not found")
+    let mut response = get_canonical(&state, "_flow_agent_runs", &id, "agent run not found")?;
+    redact_run_results(&state, &principal, &mut response.0.value);
+    Ok(response)
+}
+
+/// Remove the state a run read from a response to a reader who may not read
+/// state. A run's observations hold records read under its caller's
+/// authority; `agents:read` alone must not disclose them.
+fn redact_run_results(state: &AppState, principal: &Principal, run: &mut Value) {
+    if request_principal_grants(state, principal, "state:read") {
+        return;
+    }
+    let Some(object) = run.as_object_mut() else {
+        return;
+    };
+    for field in ["observations", "decisions"] {
+        if object.contains_key(field) {
+            object.insert(field.into(), json!([]));
+        }
+    }
+    if let Some(steps) = object.get_mut("steps").and_then(Value::as_array_mut) {
+        for step in steps {
+            if step.get("result").is_some_and(|result| !result.is_null()) {
+                step["result"] = Value::Null;
+            }
+        }
+    }
+    object.insert("results_redacted".into(), json!(true));
 }
 
 async fn add_agent_observation(
@@ -13061,12 +14479,11 @@ async fn add_agent_observation(
     Path(id): Path<String>,
     Json(request): Json<Observation>,
 ) -> Result<Json<RecordResponse>, ApiError> {
-    mutate_agent_run(&state, &id, |run| {
+    mutate_agent_run(&state, &id, agent_execution::STATUS_DELIBERATING, |run| {
         run["observations"]
             .as_array_mut()
             .expect("validated agent run")
             .push(request.observation);
-        run["status"] = json!("deliberating");
     })
     .await
 }
@@ -13076,32 +14493,44 @@ async fn add_agent_decision(
     Path(id): Path<String>,
     Json(request): Json<Decision>,
 ) -> Result<Json<RecordResponse>, ApiError> {
-    mutate_agent_run(&state, &id, |run| {
+    // Completion is decided by the runtime from criteria it verifies, never by
+    // a participant saying so — whatever evidence the request carries.
+    if request.completed {
+        return Err(ApiError::structured(
+            StatusCode::CONFLICT,
+            json!({
+                "code": "AGENT_COMPLETION_NOT_CLIENT_CONTROLLED",
+                "message": "a run completes only when the runtime verifies its declared completion criteria",
+            }),
+        ));
+    }
+    mutate_agent_run(&state, &id, agent_execution::STATUS_ACTING, |run| {
         run["decisions"]
             .as_array_mut()
             .expect("validated agent run")
-            .push(json!({"decision": request.decision, "action": request.action}));
-        run["status"] = json!(if request.completed {
-            "completed"
-        } else {
-            "acting"
-        });
+            .push(json!({"decision": request.decision, "action": request.action, "source": "client"}));
     })
     .await
 }
 
+/// Append a participant's annotation to a run that is still open.
+///
+/// Terminal outcomes are never rewritten. While the worker holds the run, the
+/// annotation is kept but the status is the worker's.
 async fn mutate_agent_run<F: FnOnce(&mut Value)>(
     state: &AppState,
     id: &str,
+    annotation: &str,
     change: F,
 ) -> Result<Json<RecordResponse>, ApiError> {
     validate_segment(id)?;
     let _guard = state.lifecycle_lock.lock().await;
     let mut run = required_value(state, "_flow_agent_runs", id, "agent run not found")?;
-    if run["status"] == "completed" {
-        return Err(ApiError(
+    let status = run["status"].as_str().unwrap_or_default().to_string();
+    if agent_execution::is_terminal(&status) {
+        return Err(ApiError::structured(
             StatusCode::CONFLICT,
-            "agent run is already completed".to_string(),
+            json!({"code": "AGENT_RUN_TERMINAL", "message": format!("agent run is already {status}")}),
         ));
     }
     if !run["observations"].is_array() || !run["decisions"].is_array() {
@@ -13111,6 +14540,9 @@ async fn mutate_agent_run<F: FnOnce(&mut Value)>(
         ));
     }
     change(&mut run);
+    if status != agent_execution::STATUS_RUNNING {
+        run["status"] = json!(annotation);
+    }
     put_canonical(state, "_flow_agent_runs", id, run)
 }
 
@@ -14214,6 +15646,7 @@ async fn commit_transaction(
         record_preconditions.push(precondition);
     }
     for operation in request.operations {
+        ensure_not_runtime_owned(&operation.collection)?;
         let key = record_key(&operation.collection, &operation.id)?;
         if !seen.insert(key.clone()) {
             return Err(ApiError(
@@ -14287,7 +15720,7 @@ async fn commit_transaction(
                         "committed": false,
                         "code": "PRECONDITION_FAILED",
                         "error": failure.to_string(),
-                        "failure": *failure,
+                        "failure": failure.public_json(&feltdb::ReadLayout::Direct),
                     }),
                 ),
                 FlowError::CapabilityError(message) if message.starts_with("INVALID_READ_SET:") => {
@@ -14304,7 +15737,29 @@ async fn commit_transaction(
                         }),
                     )
                 }
-                other => ApiError(StatusCode::CONFLICT, other.to_string()),
+                // The commit could not be made durable. It is neither a lost
+                // race nor a success, and answering 409 here once let a caller
+                // read a storage failure as "someone else won". The outcome is
+                // to be treated as unknown and reconciled by retrying with the
+                // same transaction id.
+                storage @ (FlowError::Io(_) | FlowError::Serde(_)) => ApiError::structured(
+                    StatusCode::SERVICE_UNAVAILABLE,
+                    json!({
+                        "committed": "unknown",
+                        "code": "DURABILITY_FAILURE",
+                        "message": storage.to_string(),
+                        "transaction_id": request.transaction_id,
+                    }),
+                ),
+                other => ApiError::structured(
+                    StatusCode::INTERNAL_SERVER_ERROR,
+                    json!({
+                        "committed": "unknown",
+                        "code": "EXECUTION_FAILED",
+                        "message": other.to_string(),
+                        "transaction_id": request.transaction_id,
+                    }),
+                ),
             }
         })?;
     let operation_ids = state
@@ -14334,6 +15789,21 @@ async fn commit_transaction(
     ))
 }
 
+/// Refuse a raw write to a collection only the runtime writes.
+///
+/// A run's status, retry state, evidence and authority decide what the worker
+/// does and what the run reports. Letting `/collections` or `/transactions`
+/// write them would let any `state:write` caller declare a run completed.
+fn ensure_not_runtime_owned(collection: &str) -> Result<(), ApiError> {
+    if agent_execution::is_runtime_owned_collection(collection) {
+        return Err(ApiError::structured(
+            StatusCode::FORBIDDEN,
+            json!({"code": "RUNTIME_OWNED_COLLECTION", "message": "this collection is written only by the runtime"}),
+        ));
+    }
+    Ok(())
+}
+
 async fn create_record(
     State(state): State<AppState>,
     Path(collection): Path<String>,
@@ -14341,6 +15811,7 @@ async fn create_record(
 ) -> Result<(StatusCode, Json<RecordResponse>), ApiError> {
     let _handler =
         feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::HandlerBody);
+    ensure_not_runtime_owned(&collection)?;
     if collection == PROPOSAL_COLLECTION
         || collection == PROPOSAL_EVENT_COLLECTION
         || collection == CONVERGENCE_COLLECTION
@@ -14389,6 +15860,7 @@ async fn update_record(
 ) -> Result<Json<RecordResponse>, ApiError> {
     let _handler =
         feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::HandlerBody);
+    ensure_not_runtime_owned(&collection)?;
     if collection == PROPOSAL_COLLECTION
         || collection == PROPOSAL_EVENT_COLLECTION
         || collection == CONVERGENCE_COLLECTION
@@ -14428,6 +15900,7 @@ async fn compare_and_set_record(
 ) -> Result<Response, ApiError> {
     let _handler =
         feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::HandlerBody);
+    ensure_not_runtime_owned(&collection)?;
     let key = record_key(&collection, &id)?;
     let result = state.db.compare_and_set_json(
         &key,
@@ -14519,6 +15992,7 @@ async fn delete_record(
 ) -> Result<StatusCode, ApiError> {
     let _handler =
         feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::HandlerBody);
+    ensure_not_runtime_owned(&collection)?;
     if collection == PROPOSAL_COLLECTION
         || collection == PROPOSAL_EVENT_COLLECTION
         || collection == CONVERGENCE_COLLECTION
@@ -15728,7 +17202,8 @@ async fn control_plane_agent(
                 "workload store unavailable".into(),
             )
         })?
-        .list(&tenant, &application_id)
+        .fresh_list(&tenant, &application_id)
+        .map_err(workload_error)?
         .into_iter()
         .filter(|workload| {
             serde_json::to_value(&workload.requested_by)
@@ -16323,3 +17798,32 @@ mod authority_gate_tests {
         }
     }
 }
+
+#[cfg(test)]
+mod backup_output_tests {
+    use super::backup_output_path;
+    use std::path::PathBuf;
+
+    #[test]
+    fn a_confined_server_writes_named_bundles_inside_its_backup_directory() {
+        let root = Some("/data/backups".into());
+        assert_eq!(
+            backup_output_path("nightly-1", root.clone()).unwrap(),
+            PathBuf::from("/data/backups/nightly-1")
+        );
+        for refused in ["/data/state.log", "../state.log", "a/b", "..", ".", "", "/"] {
+            assert!(
+                backup_output_path(refused, root.clone()).is_err(),
+                "{refused:?} must be refused"
+            );
+        }
+    }
+
+    #[test]
+    fn a_self_hosted_server_uses_the_operator_path() {
+        assert_eq!(
+            backup_output_path("/srv/backups/x", None).unwrap(),
+            PathBuf::from("/srv/backups/x")
+        );
+    }
+}