@feltdb/core 0.12.1 → 0.13.0

package/dist/create/server-source/crates/feltdb-server/src/main.rs

@@ -67,6 +67,7 @@ use feltdb::{
     PeerAdvertisement, PeerId, RecordPrecondition, StateRevision, StateTriggerStore, StoredRow,
 };
 use feltdb_server::{
+    agent_execution::{self, CompletionCriteria, RunFailure, RunKind, RunPrincipal},
     app_state::{AppState, BoundedQueryCursor},
     application_contract::{ApplicationContractStore, ContractPatch},
     artifacts::{ArtifactKind, ArtifactLifecycle, ArtifactMetadata, ArtifactStore, ProducerRef},
@@ -1476,6 +1477,9 @@ struct WorkloadContextRequest {
 #[derive(Deserialize)]
 struct WorkloadControlRequest {
     grant: feltdb::authorization::SignedGrant,
+    /// Why an operator retried the workload; recorded in its history.
+    #[serde(default)]
+    reason: Option<String>,
 }
 #[derive(Deserialize)]
 struct WorkerHeartbeatRequest {
@@ -1498,7 +1502,10 @@ fn workload_error(error: feltdb::workload::WorkloadError) -> ApiError {
         "WORKLOAD_STALE_FENCE"
         | "WORKLOAD_TRANSITION_DENIED"
         | "WORKLOAD_NOT_READY"
-        | "WORKLOAD_NOT_YET_ELIGIBLE" => StatusCode::CONFLICT,
+        | "WORKLOAD_NOT_YET_ELIGIBLE"
+        | "WORKLOAD_ATTEMPTS_EXHAUSTED"
+        | "WORKLOAD_RETRY_EXHAUSTED"
+        | "WORKLOAD_CONFLICT" => StatusCode::CONFLICT,
         "WORKLOAD_UNAUTHORIZED" | "WORKLOAD_CAPABILITY_DENIED" => StatusCode::FORBIDDEN,
         "WORKLOAD_STORAGE_FAILURE" => StatusCode::SERVICE_UNAVAILABLE,
         _ => StatusCode::UNPROCESSABLE_ENTITY,
@@ -1589,7 +1596,7 @@ async fn create_workload(
     State(state): State<AppState>,
     Extension(principal): Extension<Principal>,
     Json(mut input): Json<CreateWorkload>,
-) -> Result<(StatusCode, Json<Value>), ApiError> {
+) -> Result<(StatusCode, HeaderMap, Json<Value>), ApiError> {
     let subject = grant_subject(&principal);
     input.requested_by = subject.clone();
     input.session_id = principal.session_id.clone();
@@ -1603,7 +1610,7 @@ async fn create_workload(
         "workload:create",
         &input.authorization_snapshot,
     )?;
-    let workload = state
+    let (workload, created) = state
         .workloads
         .lock()
         .map_err(|_| {
@@ -1612,17 +1619,34 @@ async fn create_workload(
                 "workload store unavailable".into(),
             )
         })?
-        .create(input)
-        .map_err(workload_error)?;
+        .admit(input)
+        .map_err(|error| {
+            if error.code == "IDEMPOTENCY_KEY_REUSED" {
+                // AppPort's contract for a reused key (§13.1): CONFLICT, with
+                // the reason, not retryable.
+                ApiError::structured(
+                    StatusCode::CONFLICT,
+                    json!({"code": "CONFLICT", "message": error.message,
+                        "details": {"reason": "IDEMPOTENCY_KEY_REUSED", "retryable": false}}),
+                )
+            } else {
+                workload_error(error)
+            }
+        })?;
     record_authorization_audit(
         &state,
         &principal.key_id,
-        "workload.created",
+        if created { "workload.created" } else { "workload.replayed" },
         &workload.workload_id,
         "ready",
         201,
     );
-    Ok((StatusCode::CREATED, Json(json!(workload))))
+    // A replay answers with the stored response, marked as a replay.
+    let mut headers = HeaderMap::new();
+    if !created {
+        headers.insert("idempotent-replay", axum::http::HeaderValue::from_static("true"));
+    }
+    Ok((StatusCode::CREATED, headers, Json(json!(workload))))
 }
 async fn list_workloads(
     State(state): State<AppState>,
@@ -1664,14 +1688,27 @@ async fn get_workload(
         .get(&id)
         .map_err(workload_error)?
         .clone();
-    application_scope(
-        &state,
-        &principal.key_id,
-        &workload.application_id,
-        "workload:read",
-    )?;
+    workload_read_scope(&state, &principal, &workload.application_id)?;
     Ok(Json(json!(workload)))
 }
+/// Authorize reading one workload. A caller who may not read the
+/// application's work gets the same answer as for a workload that does not
+/// exist, so workload ids cannot be probed.
+fn workload_read_scope(
+    state: &AppState,
+    principal: &Principal,
+    application_id: &str,
+) -> Result<(), ApiError> {
+    application_scope(state, &principal.key_id, application_id, "workload:read")
+        .map(|_| ())
+        .map_err(|_| {
+            // Exactly the body `workload_error` gives a missing workload.
+            ApiError::structured(
+                StatusCode::NOT_FOUND,
+                json!({"code": "WORKLOAD_NOT_FOUND", "message": "workload not found"}),
+            )
+        })
+}
 async fn claim_workload(
     State(state): State<AppState>,
     Extension(principal): Extension<Principal>,
@@ -1971,7 +2008,7 @@ async fn retry_workload(
         .workloads
         .lock()
         .unwrap()
-        .retry(&id, &principal.key_id, unix_seconds_i64())
+        .retry_with_reason(&id, &principal.key_id, unix_seconds_i64(), input.reason.as_deref())
         .map_err(workload_error)?;
     Ok(StatusCode::NO_CONTENT)
 }
@@ -1987,12 +2024,7 @@ async fn workload_history(
         .get(&id)
         .map_err(workload_error)?
         .clone();
-    application_scope(
-        &state,
-        &principal.key_id,
-        &workload.application_id,
-        "workload:read",
-    )?;
+    workload_read_scope(&state, &principal, &workload.application_id)?;
     Ok(Json(
         json!({"events":workload.execution_history,"attempts":workload.attempts,"checkpoints":workload.checkpoints}),
     ))
@@ -2009,12 +2041,7 @@ async fn workload_result(
         .get(&id)
         .map_err(workload_error)?
         .clone();
-    application_scope(
-        &state,
-        &principal.key_id,
-        &workload.application_id,
-        "workload:read",
-    )?;
+    workload_read_scope(&state, &principal, &workload.application_id)?;
     Ok(Json(json!(workload.result)))
 }
 
@@ -2290,12 +2317,7 @@ async fn eligible_workers(
         .get(&id)
         .map_err(workload_error)?
         .clone();
-    application_scope(
-        &state,
-        &principal.key_id,
-        &workload.application_id,
-        "workload:read",
-    )?;
+    workload_read_scope(&state, &principal, &workload.application_id)?;
     let workers = state
         .mesh
         .lock()
@@ -8418,6 +8440,10 @@ struct StartAgentRun {
     goal: String,
     #[serde(default)]
     input: Value,
+    /// What the runtime must observe for the run to complete. Without it the
+    /// run can execute but never completes: the goal text is not evaluated.
+    #[serde(default)]
+    completion: Option<CompletionCriteria>,
 }
 
 #[derive(Deserialize)]
@@ -8846,6 +8872,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     };
     start_peer_sessions(&state, &config)?;
     start_membership_recovery(&state);
+    start_workload_recovery(&state);
     start_autonomous_worker(&state);
 
     let protected = Router::new()
@@ -11198,6 +11225,37 @@ fn start_autonomous_worker(state: &AppState) {
     });
 }
 
+/// Recover Durable Work whose lease expired, on every node, whether or not it
+/// runs the autonomous worker: external workers depend on it too.
+///
+/// It is the mesh's claim recovery (`recover_expired_work`, the part of
+/// `detect_failures` that recovers work): expired work goes to
+/// `WorkloadStore::recover_expired`, bounded by each workload's attempt budget,
+/// and its claim is released from the worker that held it. Worker liveness —
+/// marking silent workers offline — is deliberately not run here; it would lock
+/// out registered workers that do not heartbeat on the mesh's schedule. It takes
+/// the mesh lock before the workload lock, the order the claim route uses.
+fn start_workload_recovery(state: &AppState) {
+    let state = state.clone();
+    tokio::spawn(async move {
+        loop {
+            if state.cluster.get().is_member {
+                let outcome = match (state.mesh.lock(), state.workloads.lock()) {
+                    (Ok(mut mesh), Ok(mut workloads)) => mesh
+                        .recover_expired_work(unix_seconds_i64(), &mut workloads)
+                        .map(|_| ())
+                        .map_err(|error| error.message),
+                    _ => Err("worker mesh or workload store unavailable".to_string()),
+                };
+                if let Err(error) = outcome {
+                    tracing::warn!(%error, "durable work recovery pass incomplete");
+                }
+            }
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        }
+    });
+}
+
 fn start_membership_recovery(state: &AppState) {
     let state = state.clone();
     tokio::spawn(async move {
@@ -11218,13 +11276,18 @@ async fn run_worker_pass(state: &AppState) -> Result<(), ApiError> {
         return Ok(());
     }
     let worker = state.db.instance_id()?;
+    // One run's failure never stops the pass: each is recorded on its own run,
+    // and an infrastructure fault is logged and retried on the next pass.
     for row in state.db.list_collection("_flow_workflow_runs")? {
+        if row.value["status"] != "running" {
+            continue;
+        }
         let Some(steps) = row.value["steps"].as_array() else {
             continue;
         };
         let now = lease_now_ms(state)?;
         let Some(step) = steps.iter().find(|step| {
-            step["status"] == "ready"
+            (step["status"] == "ready" && step["not_before_ms"].as_u64().unwrap_or(0) <= now)
                 || (step["status"] == "running"
                     && step["lease_expires_ms"].as_u64().unwrap_or(0) <= now)
         }) else {
@@ -11233,42 +11296,33 @@ async fn run_worker_pass(state: &AppState) -> Result<(), ApiError> {
         let Some(step_id) = step["id"].as_str() else {
             continue;
         };
+        // A step that names no capability here is an external step, claimed
+        // by an external worker through the claim route.
         if !is_executable_capability(state, step_id)? {
             continue;
         }
         let run_id = row.value["id"].as_str().unwrap_or_default().to_string();
-        let input = row.value["input"].clone();
-        let claimed = transition_workflow_step(
-            state,
-            &run_id,
-            step_id,
-            Some((worker.clone(), 30_000)),
-            None,
-        )
-        .await?;
-        let claim_id = claimed.0.value["steps"]
-            .as_array()
-            .and_then(|steps| steps.iter().find(|step| step["id"] == step_id))
-            .and_then(|step| step["claim_id"].as_str())
-            .ok_or_else(|| {
-                ApiError(
-                    StatusCode::CONFLICT,
-                    "worker claim was not materialized".to_string(),
-                )
-            })?
-            .to_string();
-        let result = execute_named_capability(state, step_id, &input)?;
-        let _ = transition_workflow_step(state, &run_id, step_id, None, Some((claim_id, result)))
-            .await?;
+        let attempts = step["attempt"].as_u64().unwrap_or(0);
+        if let Err(error) =
+            execute_workflow_step(state, &row.value, &run_id, step_id, attempts, &worker).await
+        {
+            tracing::warn!(run = %run_id, step = %step_id, status = error.0.as_u16(), "workflow step attempt did not finish");
+        }
     }
 
     for row in state.db.list_collection("_flow_agent_runs")? {
-        let reclaimable = row.value["status"] == "running"
-            && row.value["lease_expires_ms"].as_u64().unwrap_or(0) <= lease_now_ms(state)?;
-        if row.value["status"] != "ready" && !reclaimable {
+        let now = lease_now_ms(state)?;
+        let status = row.value["status"].as_str().unwrap_or_default();
+        let lease_expired = row.value["lease_expires_ms"].as_u64().unwrap_or(0) <= now;
+        if !agent_execution::agent_run_claimable(status, lease_expired)
+            || row.value["not_before_ms"].as_u64().unwrap_or(0) > now
+        {
             continue;
         }
-        execute_agent_run(state, &row.value, &worker).await?;
+        if let Err(error) = execute_agent_run(state, &row.value, &worker).await {
+            let run = row.value["id"].as_str().unwrap_or_default();
+            tracing::warn!(%run, status = error.0.as_u16(), "agent run attempt did not finish");
+        }
     }
 
     evaluate_state_triggers(state)?;
@@ -11372,7 +11426,7 @@ fn worker_capabilities(state: &AppState) -> Result<BTreeSet<String>, ApiError> {
 fn execute_eligible_workloads(state: &AppState, worker: &str) -> Result<(), ApiError> {
     let capabilities = worker_capabilities(state)?;
     let now = unix_seconds_i64();
-    let candidates: Vec<(String, String, Value)> = {
+    let candidates: Vec<(String, String, Value, feltdb::workload::Workload)> = {
         let store = workload_store(state)?;
         store
             .workloads
@@ -11391,12 +11445,13 @@ fn execute_eligible_workloads(state: &AppState, worker: &str) -> Result<(), ApiE
                     workload.workload_id.clone(),
                     workload.definition_id.clone(),
                     workload.input.clone(),
+                    workload.clone(),
                 )
             })
             .collect()
     };
 
-    for (id, definition, input) in candidates {
+    for (id, definition, input, workload) in candidates {
         // Unlike the filter above, this one is enforcement: nothing downstream
         // knows whether `definition_id` names something runnable. Claiming a
         // workload only to dead-letter it would consume work another node could
@@ -11427,7 +11482,13 @@ fn execute_eligible_workloads(state: &AppState, worker: &str) -> Result<(), ApiE
             )
             .map_err(workload_error)?;
 
-        match execute_named_capability(state, &definition, &input) {
+        let access = CapabilityAccess::Workload {
+            requested_by: &workload.requested_by,
+            tenant_id: &workload.tenant_id,
+            application_id: &workload.application_id,
+            grant: &workload.authorization_snapshot,
+        };
+        match execute_named_capability(state, &definition, &input, &access) {
             Ok(output) => {
                 workload_store(state)?
                     .complete(
@@ -11464,6 +11525,157 @@ fn execute_eligible_workloads(state: &AppState, worker: &str) -> Result<(), ApiE
     Ok(())
 }
 
+/// Debug builds only: `FELTDB_TEST_TRANSIENT_CAPABILITY=<name>[:<count>]` makes
+/// executing `<name>` fail with a server error — every time, or for its first
+/// `<count>` executions — so retry bounds can be exercised at the HTTP
+/// boundary. Compiled out of release builds, like `FELTDB_TEST_CLOCK_OFFSET_MS`.
+#[cfg(debug_assertions)]
+fn injected_capability_fault(name: &str) -> Option<ApiError> {
+    static INJECTED: AtomicU64 = AtomicU64::new(0);
+    let setting = std::env::var("FELTDB_TEST_TRANSIENT_CAPABILITY").ok()?;
+    let (target, limit) = match setting.split_once(':') {
+        Some((target, count)) => (target.to_string(), count.parse::<u64>().ok()),
+        None => (setting, None),
+    };
+    if target != name {
+        return None;
+    }
+    let injected = INJECTED.fetch_add(1, Ordering::SeqCst);
+    if limit.is_some_and(|limit| injected >= limit) {
+        return None;
+    }
+    Some(ApiError(
+        StatusCode::SERVICE_UNAVAILABLE,
+        "injected transient capability failure".into(),
+    ))
+}
+
+/// The authority a capability reads under.
+///
+/// Every collection a capability touches is authorized against it in
+/// [`authorize_capability_read`], the single point all capability reads pass
+/// through, so no route, program step, workflow, or agent reaches state with
+/// more authority than the principal it acts for.
+enum CapabilityAccess<'a> {
+    /// The request's own principal, on the direct capability routes.
+    Principal(&'a Principal),
+    /// An application actor on the `/v1` Service API.
+    Application {
+        actor: &'a str,
+        application_id: &'a str,
+    },
+    /// A run's verified delegated authority, on the autonomous worker.
+    Delegated(&'a RunPrincipal),
+    /// A durable workload's signed authorization snapshot. Reads are allowed
+    /// only when that grant, re-verified at execution, authorizes `state:read`
+    /// for the workload's requester — so a workload never reads with the
+    /// node's authority, and a row injected without a server-signed grant
+    /// reads nothing.
+    Workload {
+        requested_by: &'a GrantSubject,
+        tenant_id: &'a str,
+        application_id: &'a str,
+        grant: &'a feltdb::authorization::SignedGrant,
+    },
+}
+
+/// Whether a principal, described the way the authenticator describes it, holds
+/// `capability`. The same decision the request middleware makes.
+fn principal_grants(
+    state: &AppState,
+    kind: &str,
+    id: &str,
+    scopes: &[String],
+    capability: &str,
+) -> bool {
+    let subject = if kind == "human" {
+        Subject::User(id.to_string())
+    } else {
+        Subject::Service(id.to_string())
+    };
+    authorize(&AuthorizationRequest {
+        subject: &subject,
+        tenant_id: &state.namespace,
+        application_id: Some(&state.namespace),
+        capability,
+        direct_grants: scopes,
+        memberships: &[],
+        roles: &[],
+    })
+    .allowed
+}
+
+fn request_principal_grants(state: &AppState, principal: &Principal, capability: &str) -> bool {
+    principal_grants(
+        state,
+        &principal.subject_type,
+        &principal.key_id,
+        &principal.scopes,
+        capability,
+    )
+}
+
+fn delegated_grants(state: &AppState, principal: &RunPrincipal, capability: &str) -> bool {
+    let scopes: Vec<String> = principal.scopes.iter().cloned().collect();
+    principal_grants(state, &principal.kind, &principal.id, &scopes, capability)
+}
+
+fn capability_read_denied(message: String) -> ApiError {
+    ApiError::structured(
+        StatusCode::FORBIDDEN,
+        json!({"code": agent_execution::AUTHORIZATION_DENIED, "message": message}),
+    )
+}
+
+/// Authorize one collection read made by a capability.
+///
+/// Reading records through a capability is reading state, so it needs the
+/// `state:read` a direct read needs. System collections (`_`-prefixed) hold
+/// runs, definitions and other server records and are never readable this way.
+fn authorize_capability_read(
+    state: &AppState,
+    access: &CapabilityAccess<'_>,
+    collection: &str,
+) -> Result<(), ApiError> {
+    if agent_execution::is_system_collection(collection) {
+        return Err(capability_read_denied(format!(
+            "capabilities cannot read system collection {collection}"
+        )));
+    }
+    let allowed = match access {
+        CapabilityAccess::Principal(principal) => {
+            request_principal_grants(state, principal, "state:read")
+        }
+        CapabilityAccess::Application {
+            actor,
+            application_id,
+        } => application_scope(state, actor, application_id, "state:read").is_ok(),
+        CapabilityAccess::Delegated(principal) => delegated_grants(state, principal, "state:read"),
+        CapabilityAccess::Workload {
+            requested_by,
+            tenant_id,
+            application_id,
+            grant,
+        } => workload_decision(
+            state,
+            (*requested_by).clone(),
+            tenant_id,
+            application_id,
+            format!("flow://{application_id}/state/{collection}"),
+            "state:read",
+            grant,
+        )
+        .is_ok(),
+    };
+    if allowed {
+        Ok(())
+    } else {
+        Err(capability_read_denied(format!(
+            "the caller is not authorized to read {collection}"
+        )))
+    }
+}
+
 fn is_builtin_capability(name: &str) -> bool {
     matches!(name, "search" | "identity")
 }
@@ -11480,6 +11692,7 @@ fn execute_builtin_capability(
     state: &AppState,
     name: &str,
     input: &Value,
+    access: &CapabilityAccess<'_>,
 ) -> Result<Value, ApiError> {
     match name {
         "identity" => Ok(input.clone()),
@@ -11491,6 +11704,7 @@ fn execute_builtin_capability(
                 )
             })?;
             validate_segment(collection)?;
+            authorize_capability_read(state, access, collection)?;
             let needle = input["query"].as_str().unwrap_or_default().to_lowercase();
             let limit = input["limit"].as_u64().unwrap_or(50).min(500) as usize;
             Ok(Value::Array(
@@ -11511,80 +11725,369 @@ fn execute_builtin_capability(
     }
 }
 
+/// The text of an [`ApiError`] without its wire envelope: the `message` of a
+/// structured error, or the plain message.
+fn api_error_message(error: &ApiError) -> String {
+    serde_json::from_str::<Value>(&error.1)
+        .ok()
+        .and_then(|value| value["message"].as_str().map(str::to_string))
+        .unwrap_or_else(|| error.1.clone())
+}
+
+/// Everything an agent run needs before the worker may lease it.
+///
+/// Checked on every attempt, so authority revoked after the run was created,
+/// a definition removed, or a capability unregistered ends the run here,
+/// before anything executes.
+fn prepare_agent_run(
+    state: &AppState,
+    run: &Value,
+    id: &str,
+    attempt: u64,
+) -> Result<(String, RunPrincipal, Option<CompletionCriteria>), RunFailure> {
+    if attempt > agent_execution::max_run_attempts() {
+        return Err(RunFailure::permanent(
+            agent_execution::RETRY_EXHAUSTED,
+            format!(
+                "run was claimed {} times without finishing",
+                attempt - 1
+            ),
+        ));
+    }
+    let principal = resolve_run_principal(state, run, RunKind::Agent, id)?;
+    if !delegated_grants(state, &principal, "agents:run") {
+        return Err(RunFailure::permanent(
+            agent_execution::AUTHORIZATION_DENIED,
+            "the run's principal may not run agents",
+        ));
+    }
+    let agent = run["agent"].as_str().unwrap_or_default();
+    let definition = match validate_segment(agent)
+        .and_then(|_| required_value(state, "_flow_agents", agent, "agent not found"))
+    {
+        Ok(definition) => definition,
+        Err(_) => {
+            return Err(RunFailure::permanent(
+                agent_execution::AGENT_NOT_FOUND,
+                "the run's agent is not defined",
+            ))
+        }
+    };
+    let capability = agent_capability(state, &definition)?;
+    let criteria = match &run["completion"] {
+        Value::Null => None,
+        value => Some(
+            serde_json::from_value::<CompletionCriteria>(value.clone())
+                .ok()
+                .filter(|criteria| criteria.validate().is_ok())
+                .ok_or_else(|| {
+                    RunFailure::permanent(
+                        agent_execution::RUN_INVALID,
+                        "the run's completion criteria are invalid",
+                    )
+                })?,
+        ),
+    };
+    Ok((capability, principal, criteria))
+}
+
+fn mark_agent_run_failed(run: &mut Value, failure: &RunFailure, attempt: u64) {
+    run["status"] = json!(agent_execution::STATUS_FAILED);
+    run["failure"] = failure.to_record(attempt);
+    run["attempt"] = json!(attempt);
+}
+
+/// Execute one attempt of an agent run.
+///
+/// The run completes only when its declared criteria hold for the result this
+/// worker observed. Every other outcome is explicit: a permanent error fails
+/// the run, a transient one returns it to `ready` after a backoff until the
+/// attempt bound, and a run without criteria fails as unverifiable. An `Err`
+/// is an infrastructure fault that left the run untouched.
 async fn execute_agent_run(state: &AppState, run: &Value, worker: &str) -> Result<(), ApiError> {
     let id = run["id"]
         .as_str()
         .ok_or_else(|| ApiError(StatusCode::CONFLICT, "agent run has no id".to_string()))?;
-    let agent_name = run["agent"].as_str().ok_or_else(|| {
-        ApiError(
-            StatusCode::CONFLICT,
-            "agent run has no definition".to_string(),
-        )
-    })?;
-    let definition = required_value(state, "_flow_agents", agent_name, "agent not found")?;
-    let capability = definition["capabilities"].as_array().and_then(|values| {
-        values
-            .iter()
-            .filter_map(Value::as_str)
-            .find(|name| is_executable_capability(state, name).unwrap_or(false))
-    });
-    let Some(capability) = capability else {
-        return Ok(());
-    };
-    let claim_id = next_id(state, "claim");
-    let expires_ms = lease_now_ms(state)?.saturating_add(30_000);
-    let lease = LeaseRequest {
-        claim_id: claim_id.clone(),
-        owner: worker.to_string(),
-        expires_ms,
-        membership_epoch: state.cluster.get().epoch,
-    };
-    {
+    let (capability, principal, criteria, claim_id, attempt, input) = {
         let _guard = state.lifecycle_lock.lock().await;
-        reserve_quorum_lease(state, &format!("agent-{id}"), &lease).await?;
         let mut current = required_value(state, "_flow_agent_runs", id, "agent run not found")?;
-        let reclaimable = current["status"] == "running"
-            && current["lease_expires_ms"].as_u64().unwrap_or(0) <= lease_now_ms(state)?;
-        if current["status"] != "ready" && !reclaimable {
+        let now = lease_now_ms(state)?;
+        let status = current["status"].as_str().unwrap_or_default();
+        let lease_expired = current["lease_expires_ms"].as_u64().unwrap_or(0) <= now;
+        if !agent_execution::agent_run_claimable(status, lease_expired)
+            || current["not_before_ms"].as_u64().unwrap_or(0) > now
+        {
             return Ok(());
         }
-        current["status"] = json!("running");
+        let attempt = current["attempt"].as_u64().unwrap_or(0) + 1;
+        let (capability, principal, criteria) =
+            match prepare_agent_run(state, &current, id, attempt) {
+                Ok(prepared) => prepared,
+                Err(failure) => {
+                    mark_agent_run_failed(&mut current, &failure, attempt);
+                    put_canonical(state, "_flow_agent_runs", id, current)?;
+                    return Ok(());
+                }
+            };
+        let claim_id = next_id(state, "claim");
+        let expires_ms = now.saturating_add(30_000);
+        let lease = LeaseRequest {
+            claim_id: claim_id.clone(),
+            owner: worker.to_string(),
+            expires_ms,
+            membership_epoch: state.cluster.get().epoch,
+        };
+        reserve_quorum_lease(state, &format!("agent-{id}"), &lease).await?;
+        current["status"] = json!(agent_execution::STATUS_RUNNING);
         current["worker"] = json!(worker);
         current["claim_id"] = json!(claim_id);
         current["lease_expires_ms"] = json!(expires_ms);
-        let _ = put_canonical(state, "_flow_agent_runs", id, current)?;
-    }
-    let result = execute_named_capability(state, capability, &run["input"])?;
+        current["attempt"] = json!(attempt);
+        let input = current["input"].clone();
+        put_canonical(state, "_flow_agent_runs", id, current)?;
+        (capability, principal, criteria, claim_id, attempt, input)
+    };
+
+    let outcome = execute_named_capability(
+        state,
+        &capability,
+        &input,
+        &CapabilityAccess::Delegated(&principal),
+    );
+
     let _guard = state.lifecycle_lock.lock().await;
     let mut current = required_value(state, "_flow_agent_runs", id, "agent run not found")?;
-    if current["claim_id"] != claim_id
-        || current["lease_expires_ms"].as_u64().unwrap_or(0) <= lease_now_ms(state)?
+    let now = lease_now_ms(state)?;
+    // Publish only while this claim still owns the run. A claim that lost its
+    // lease publishes nothing; the next claim counts as another attempt.
+    if current["claim_id"] != claim_id.as_str()
+        || current["status"] != agent_execution::STATUS_RUNNING
+        || current["lease_expires_ms"].as_u64().unwrap_or(0) <= now
     {
-        return Err(ApiError(
-            StatusCode::CONFLICT,
-            "agent ownership expired before result publication".to_string(),
-        ));
+        return Ok(());
+    }
+    match outcome {
+        Ok(result) => {
+            let satisfied = criteria
+                .as_ref()
+                .map(|criteria| criteria.satisfied_by(&result));
+            let digest = format!(
+                "sha256:{:x}",
+                Sha256::digest(serde_json::to_vec(&result).unwrap_or_default())
+            );
+            let evidence = json!({
+                "source": "runtime", "attempt": attempt, "claim_id": claim_id,
+                "capability": capability, "result_items": result.as_array().map(Vec::len),
+                "result_digest": digest, "criteria": criteria, "satisfied": satisfied,
+                "observed_at_ms": now,
+            });
+            for (field, value) in [
+                ("observations", result.clone()),
+                (
+                    "decisions",
+                    json!({"capability": capability, "result": result, "source": "runtime"}),
+                ),
+                ("evidence", evidence),
+            ] {
+                if !current[field].is_array() {
+                    current[field] = json!([]);
+                }
+                current[field]
+                    .as_array_mut()
+                    .expect("normalized to an array")
+                    .push(value);
+            }
+            match satisfied {
+                Some(true) => {
+                    current["status"] = json!(agent_execution::STATUS_COMPLETED);
+                    current["completed_at_ms"] = json!(now);
+                }
+                Some(false) => mark_agent_run_failed(
+                    &mut current,
+                    &RunFailure::permanent(
+                        agent_execution::COMPLETION_CRITERIA_UNMET,
+                        "the observed result does not satisfy the run's completion criteria",
+                    ),
+                    attempt,
+                ),
+                None => mark_agent_run_failed(
+                    &mut current,
+                    &RunFailure::permanent(
+                        agent_execution::COMPLETION_UNVERIFIABLE,
+                        "the run declares no completion criteria the runtime can verify; its goal text is not evaluated",
+                    ),
+                    attempt,
+                ),
+            }
+        }
+        Err(error) => {
+            let failure = agent_execution::classify_capability_error(
+                error.0.as_u16(),
+                &api_error_message(&error),
+            );
+            if failure.retryable && attempt < agent_execution::max_run_attempts() {
+                // The failed attempt's lease stays reserved until it lapses, so
+                // the retry waits for that as well as for the backoff: a new
+                // claim before then would be refused by the lease quorum.
+                let lease_expires = current["lease_expires_ms"].as_u64().unwrap_or(0);
+                current["status"] = json!(agent_execution::STATUS_READY);
+                current["not_before_ms"] = json!(now
+                    .saturating_add(agent_execution::retry_delay_ms(attempt + 1))
+                    .max(lease_expires));
+                current["last_failure"] = failure.to_record(attempt);
+            } else if failure.retryable {
+                mark_agent_run_failed(
+                    &mut current,
+                    &RunFailure::permanent(
+                        agent_execution::RETRY_EXHAUSTED,
+                        format!("{} after {attempt} attempts", failure.message),
+                    ),
+                    attempt,
+                );
+            } else {
+                mark_agent_run_failed(&mut current, &failure, attempt);
+            }
+        }
+    }
+    put_canonical(state, "_flow_agent_runs", id, current)?;
+    Ok(())
+}
+
+/// Fail a workflow step and its run. `claim_id`, when given, must still own
+/// the step; otherwise another claim has taken over and nothing changes.
+async fn fail_workflow_step(
+    state: &AppState,
+    run_id: &str,
+    step_id: &str,
+    claim_id: Option<&str>,
+    failure: &RunFailure,
+) -> Result<(), ApiError> {
+    let _guard = state.lifecycle_lock.lock().await;
+    let mut run = required_value(state, "_flow_workflow_runs", run_id, "workflow run not found")?;
+    if run["status"] != "running" {
+        return Ok(());
     }
-    current["observations"]
+    let Some(step) = run["steps"]
         .as_array_mut()
-        .ok_or_else(|| {
-            ApiError(
-                StatusCode::CONFLICT,
-                "agent observations are invalid".to_string(),
-            )
-        })?
-        .push(result.clone());
-    current["decisions"]
+        .and_then(|steps| steps.iter_mut().find(|step| step["id"] == step_id))
+    else {
+        return Ok(());
+    };
+    if claim_id.is_some_and(|claim| step["claim_id"] != claim) {
+        return Ok(());
+    }
+    let attempt = step["attempt"].as_u64().unwrap_or(0);
+    step["status"] = json!(agent_execution::STATUS_FAILED);
+    step["failure"] = failure.to_record(attempt);
+    run["status"] = json!(agent_execution::STATUS_FAILED);
+    run["failure"] = json!({"step": step_id, "code": failure.code, "message": failure.message});
+    put_canonical(state, "_flow_workflow_runs", run_id, run)?;
+    Ok(())
+}
+
+/// Return a step that failed transiently to `ready` after a backoff.
+async fn release_workflow_step(
+    state: &AppState,
+    run_id: &str,
+    step_id: &str,
+    claim_id: &str,
+    failure: &RunFailure,
+) -> Result<(), ApiError> {
+    let _guard = state.lifecycle_lock.lock().await;
+    let mut run = required_value(state, "_flow_workflow_runs", run_id, "workflow run not found")?;
+    let now = lease_now_ms(state)?;
+    let Some(step) = run["steps"]
         .as_array_mut()
+        .and_then(|steps| steps.iter_mut().find(|step| step["id"] == step_id))
+    else {
+        return Ok(());
+    };
+    if step["claim_id"] != claim_id || step["status"] != "running" {
+        return Ok(());
+    }
+    let attempt = step["attempt"].as_u64().unwrap_or(0);
+    let lease_expires = step["lease_expires_ms"].as_u64().unwrap_or(0);
+    step["status"] = json!(agent_execution::STATUS_READY);
+    // Not before the failed attempt's lease lapses: until then the lease
+    // quorum refuses any other claim.
+    step["not_before_ms"] = json!(now
+        .saturating_add(agent_execution::retry_delay_ms(attempt + 1))
+        .max(lease_expires));
+    step["last_failure"] = failure.to_record(attempt);
+    put_canonical(state, "_flow_workflow_runs", run_id, run)?;
+    Ok(())
+}
+
+/// Execute one autonomous workflow step under the run's delegated authority.
+async fn execute_workflow_step(
+    state: &AppState,
+    run: &Value,
+    run_id: &str,
+    step_id: &str,
+    attempts: u64,
+    worker: &str,
+) -> Result<(), ApiError> {
+    if attempts >= agent_execution::max_run_attempts() {
+        let failure = RunFailure::permanent(
+            agent_execution::RETRY_EXHAUSTED,
+            format!("step was claimed {attempts} times without finishing"),
+        );
+        return fail_workflow_step(state, run_id, step_id, None, &failure).await;
+    }
+    let principal = match resolve_run_principal(state, run, RunKind::Workflow, run_id) {
+        Ok(principal) if delegated_grants(state, &principal, "workflows:run") => principal,
+        Ok(_) => {
+            let failure = RunFailure::permanent(
+                agent_execution::AUTHORIZATION_DENIED,
+                "the run's principal may not run workflows",
+            );
+            return fail_workflow_step(state, run_id, step_id, None, &failure).await;
+        }
+        Err(failure) => return fail_workflow_step(state, run_id, step_id, None, &failure).await,
+    };
+    let claimed = transition_workflow_step(
+        state,
+        run_id,
+        step_id,
+        Some((worker.to_string(), 30_000)),
+        None,
+    )
+    .await?;
+    let claim_id = claimed.0.value["steps"]
+        .as_array()
+        .and_then(|steps| steps.iter().find(|step| step["id"] == step_id))
+        .and_then(|step| step["claim_id"].as_str())
         .ok_or_else(|| {
             ApiError(
                 StatusCode::CONFLICT,
-                "agent decisions are invalid".to_string(),
+                "worker claim was not materialized".to_string(),
             )
         })?
-        .push(json!({"capability": capability, "result": result}));
-    current["status"] = json!("completed");
-    let _ = put_canonical(state, "_flow_agent_runs", id, current)?;
+        .to_string();
+    let input = run["input"].clone();
+    match execute_named_capability(state, step_id, &input, &CapabilityAccess::Delegated(&principal))
+    {
+        Ok(result) => {
+            transition_workflow_step(state, run_id, step_id, None, Some((claim_id, result))).await?;
+        }
+        Err(error) => {
+            let failure = agent_execution::classify_capability_error(
+                error.0.as_u16(),
+                &api_error_message(&error),
+            );
+            if failure.retryable && attempts + 1 < agent_execution::max_run_attempts() {
+                release_workflow_step(state, run_id, step_id, &claim_id, &failure).await?;
+            } else if failure.retryable {
+                let exhausted = RunFailure::permanent(
+                    agent_execution::RETRY_EXHAUSTED,
+                    format!("{} after {} attempts", failure.message, attempts + 1),
+                );
+                fail_workflow_step(state, run_id, step_id, Some(&claim_id), &exhausted).await?;
+            } else {
+                fail_workflow_step(state, run_id, step_id, Some(&claim_id), &failure).await?;
+            }
+        }
+    }
     Ok(())
 }
 
@@ -12563,10 +13066,12 @@ async fn get_provenance(
 
 async fn search_collection(
     State(state): State<AppState>,
+    Extension(principal): Extension<Principal>,
     Path(collection): Path<String>,
     Json(request): Json<SearchRequest>,
 ) -> Result<Json<Vec<RecordResponse>>, ApiError> {
     validate_segment(&collection)?;
+    authorize_capability_read(&state, &CapabilityAccess::Principal(&principal), &collection)?;
     if request.query.len() > 512 || request.limit == 0 || request.limit > 500 {
         return Err(ApiError(
             StatusCode::BAD_REQUEST,
@@ -12616,7 +13121,12 @@ async fn execute_capability(
         ));
     }
     validate_segment(&name)?;
-    Ok(Json(execute_named_capability(&state, &name, &input)?))
+    Ok(Json(execute_named_capability(
+        &state,
+        &name,
+        &input,
+        &CapabilityAccess::Principal(&principal),
+    )?))
 }
 
 async fn execute_application_capability(
@@ -12663,7 +13173,15 @@ async fn execute_application_capability(
             "capability is not externally invocable".into(),
         ));
     }
-    Ok(Json(execute_named_capability(&state, &name, &payload)?))
+    Ok(Json(execute_named_capability(
+        &state,
+        &name,
+        &payload,
+        &CapabilityAccess::Application {
+            actor: &principal.key_id,
+            application_id: &input.application_id,
+        },
+    )?))
 }
 
 fn validate_capability_program(program: &CapabilityProgram) -> Result<(), ApiError> {
@@ -12722,9 +13240,14 @@ fn execute_named_capability(
     state: &AppState,
     name: &str,
     input: &Value,
+    access: &CapabilityAccess<'_>,
 ) -> Result<Value, ApiError> {
+    #[cfg(debug_assertions)]
+    if let Some(fault) = injected_capability_fault(name) {
+        return Err(fault);
+    }
     if is_builtin_capability(name) {
-        return execute_builtin_capability(state, name, input);
+        return execute_builtin_capability(state, name, input, access);
     }
     let value = required_value(state, "_flow_capabilities", name, "capability not found")?;
     let program: CapabilityProgram = serde_json::from_value(value).map_err(|error| {
@@ -12758,6 +13281,7 @@ fn execute_named_capability(
                     state,
                     "search",
                     &json!({"collection": collection, "query": query, "limit": limit.unwrap_or(50)}),
+                    access,
                 )?
             }
             CapabilityInstruction::FilterEq {
@@ -12873,12 +13397,14 @@ async fn start_workflow_run(
     let step_states: Vec<Value> = steps.iter().enumerate().map(|(index, step)| json!({
         "id": step, "status": if index == 0 { "ready" } else { "pending" }, "worker": null, "result": null
     })).collect();
+    let (authority, attribution) = run_authority(&state, RunKind::Workflow, &id, &principal)?;
     let response = insert_canonical(
         &state,
         "_flow_workflow_runs",
         &id,
         json!({
-            "id": id, "workflow": name, "status": "running", "input": request.input, "steps": step_states
+            "id": id, "workflow": name, "status": "running", "input": request.input, "steps": step_states,
+            "authority": authority, "principal": attribution,
         }),
     )?;
     Ok((StatusCode::CREATED, Json(response)))
@@ -12886,9 +13412,12 @@ async fn start_workflow_run(
 
 async fn get_workflow_run(
     State(state): State<AppState>,
+    Extension(principal): Extension<Principal>,
     Path(id): Path<String>,
 ) -> Result<Json<RecordResponse>, ApiError> {
-    get_canonical(&state, "_flow_workflow_runs", &id, "workflow run not found")
+    let mut response = get_canonical(&state, "_flow_workflow_runs", &id, "workflow run not found")?;
+    redact_run_results(&state, &principal, &mut response.0.value);
+    Ok(response)
 }
 
 async fn claim_workflow_step(
@@ -12940,6 +13469,12 @@ async fn transition_workflow_step(
     validate_segment(step)?;
     let _guard = state.lifecycle_lock.lock().await;
     let mut run = required_value(state, "_flow_workflow_runs", id, "workflow run not found")?;
+    if run["status"] == agent_execution::STATUS_FAILED {
+        return Err(ApiError::structured(
+            StatusCode::CONFLICT,
+            json!({"code": "WORKFLOW_RUN_TERMINAL", "message": "workflow run has failed"}),
+        ));
+    }
     let steps = run["steps"]
         .as_array_mut()
         .ok_or_else(|| ApiError(StatusCode::CONFLICT, "workflow run is invalid".to_string()))?;
@@ -12968,10 +13503,12 @@ async fn transition_workflow_step(
             membership_epoch: state.cluster.get().epoch,
         };
         reserve_quorum_lease(state, &resource, &lease).await?;
+        let attempt = steps[index]["attempt"].as_u64().unwrap_or(0) + 1;
         steps[index]["status"] = json!("running");
         steps[index]["worker"] = json!(worker);
         steps[index]["claim_id"] = json!(claim_id);
         steps[index]["lease_expires_ms"] = json!(expires_ms);
+        steps[index]["attempt"] = json!(attempt);
     } else {
         if status != "running" {
             return Err(ApiError(
@@ -13029,31 +13566,169 @@ async fn start_agent_run(
         ));
     }
     validate_segment(&name)?;
-    required_value(&state, "_flow_agents", &name, "agent not found")?;
+    let definition = required_value(&state, "_flow_agents", &name, "agent not found")?;
     if request.goal.trim().is_empty() {
         return Err(ApiError(
             StatusCode::BAD_REQUEST,
             "agent goal is required".to_string(),
         ));
     }
+    if let Some(criteria) = &request.completion {
+        criteria.validate().map_err(|message| {
+            ApiError::structured(
+                StatusCode::BAD_REQUEST,
+                json!({"code": "COMPLETION_CRITERIA_INVALID", "message": message}),
+            )
+        })?;
+    }
+    // Refuse a run that could only ever fail, instead of creating one.
+    agent_capability(&state, &definition).map_err(|failure| {
+        ApiError::structured(
+            StatusCode::UNPROCESSABLE_ENTITY,
+            json!({"code": failure.code, "message": failure.message}),
+        )
+    })?;
     let id = next_id(&state, "agent");
+    let (authority, attribution) = run_authority(&state, RunKind::Agent, &id, &principal)?;
     let response = insert_canonical(
         &state,
         "_flow_agent_runs",
         &id,
         json!({
             "id": id, "agent": name, "goal": request.goal, "input": request.input,
-            "status": "ready", "observations": [], "decisions": []
+            "status": agent_execution::STATUS_READY, "observations": [], "decisions": [],
+            "completion": request.completion, "evidence": [], "attempt": 0,
+            "authority": authority, "principal": attribution,
         }),
     )?;
     Ok((StatusCode::CREATED, Json(response)))
 }
 
+/// Sign the requesting principal's authority into a new run, and the
+/// attribution recorded beside it.
+fn run_authority(
+    state: &AppState,
+    kind: RunKind,
+    run_id: &str,
+    principal: &Principal,
+) -> Result<(Value, Value), ApiError> {
+    let run_principal = RunPrincipal {
+        id: principal.key_id.clone(),
+        kind: principal.subject_type.clone(),
+        scopes: principal.scopes.iter().cloned().collect(),
+    };
+    let grant = agent_execution::issue_run_authority(
+        &state.grant_signer,
+        kind,
+        run_id,
+        &state.namespace,
+        &run_principal,
+        unix_seconds_i64(),
+    )
+    .map_err(|message| ApiError(StatusCode::INTERNAL_SERVER_ERROR, message))?;
+    let authority = serde_json::to_value(grant)
+        .map_err(|error| ApiError(StatusCode::INTERNAL_SERVER_ERROR, error.to_string()))?;
+    Ok((
+        authority,
+        json!({"id": principal.key_id, "kind": principal.subject_type}),
+    ))
+}
+
+/// The capability an agent run executes: the first one its definition
+/// declares. It must be executable now; one that is not is a permanent error,
+/// never a run left waiting for a capability that does not exist.
+fn agent_capability(state: &AppState, definition: &Value) -> Result<String, RunFailure> {
+    let Some(name) = definition["capabilities"]
+        .as_array()
+        .and_then(|values| values.first())
+        .and_then(Value::as_str)
+    else {
+        return Err(RunFailure::permanent(
+            agent_execution::CAPABILITY_UNKNOWN,
+            "agent declares no capability",
+        ));
+    };
+    match is_executable_capability(state, name) {
+        Ok(true) => Ok(name.to_string()),
+        _ => Err(RunFailure::permanent(
+            agent_execution::CAPABILITY_UNKNOWN,
+            format!("capability {name} is not defined on this node"),
+        )),
+    }
+}
+
+/// The principal a run may act for at execution time.
+///
+/// The signed authority binds scopes to the run. When authentication is on,
+/// the key is also re-read, and only scopes it still holds remain: a revoked
+/// key acts for nothing, and a narrowed key never acts with its old breadth.
+fn resolve_run_principal(
+    state: &AppState,
+    run: &Value,
+    kind: RunKind,
+    run_id: &str,
+) -> Result<RunPrincipal, RunFailure> {
+    let mut principal = agent_execution::verify_run_authority(
+        &state.grant_signer,
+        &run["authority"],
+        kind,
+        run_id,
+        &state.namespace,
+        unix_seconds_i64(),
+    )?;
+    if state.auth_enabled && principal.kind != "human" {
+        let Some((current, current_kind)) = state.keys.active_scopes(&principal.id, &state.namespace)
+        else {
+            return Err(RunFailure::permanent(
+                agent_execution::AUTHORIZATION_DENIED,
+                "the run's principal is no longer active",
+            ));
+        };
+        if current_kind != principal.kind {
+            return Err(RunFailure::permanent(
+                agent_execution::AUTHORIZATION_DENIED,
+                "the run's principal changed type",
+            ));
+        }
+        principal.scopes =
+            agent_execution::effective_scopes(&principal.scopes, &current.into_iter().collect());
+    }
+    Ok(principal)
+}
+
 async fn get_agent_run(
     State(state): State<AppState>,
+    Extension(principal): Extension<Principal>,
     Path(id): Path<String>,
 ) -> Result<Json<RecordResponse>, ApiError> {
-    get_canonical(&state, "_flow_agent_runs", &id, "agent run not found")
+    let mut response = get_canonical(&state, "_flow_agent_runs", &id, "agent run not found")?;
+    redact_run_results(&state, &principal, &mut response.0.value);
+    Ok(response)
+}
+
+/// Remove the state a run read from a response to a reader who may not read
+/// state. A run's observations hold records read under its caller's
+/// authority; `agents:read` alone must not disclose them.
+fn redact_run_results(state: &AppState, principal: &Principal, run: &mut Value) {
+    if request_principal_grants(state, principal, "state:read") {
+        return;
+    }
+    let Some(object) = run.as_object_mut() else {
+        return;
+    };
+    for field in ["observations", "decisions"] {
+        if object.contains_key(field) {
+            object.insert(field.into(), json!([]));
+        }
+    }
+    if let Some(steps) = object.get_mut("steps").and_then(Value::as_array_mut) {
+        for step in steps {
+            if step.get("result").is_some_and(|result| !result.is_null()) {
+                step["result"] = Value::Null;
+            }
+        }
+    }
+    object.insert("results_redacted".into(), json!(true));
 }
 
 async fn add_agent_observation(
@@ -13061,12 +13736,11 @@ async fn add_agent_observation(
     Path(id): Path<String>,
     Json(request): Json<Observation>,
 ) -> Result<Json<RecordResponse>, ApiError> {
-    mutate_agent_run(&state, &id, |run| {
+    mutate_agent_run(&state, &id, agent_execution::STATUS_DELIBERATING, |run| {
         run["observations"]
             .as_array_mut()
             .expect("validated agent run")
             .push(request.observation);
-        run["status"] = json!("deliberating");
     })
     .await
 }
@@ -13076,32 +13750,44 @@ async fn add_agent_decision(
     Path(id): Path<String>,
     Json(request): Json<Decision>,
 ) -> Result<Json<RecordResponse>, ApiError> {
-    mutate_agent_run(&state, &id, |run| {
+    // Completion is decided by the runtime from criteria it verifies, never by
+    // a participant saying so — whatever evidence the request carries.
+    if request.completed {
+        return Err(ApiError::structured(
+            StatusCode::CONFLICT,
+            json!({
+                "code": "AGENT_COMPLETION_NOT_CLIENT_CONTROLLED",
+                "message": "a run completes only when the runtime verifies its declared completion criteria",
+            }),
+        ));
+    }
+    mutate_agent_run(&state, &id, agent_execution::STATUS_ACTING, |run| {
         run["decisions"]
             .as_array_mut()
             .expect("validated agent run")
-            .push(json!({"decision": request.decision, "action": request.action}));
-        run["status"] = json!(if request.completed {
-            "completed"
-        } else {
-            "acting"
-        });
+            .push(json!({"decision": request.decision, "action": request.action, "source": "client"}));
     })
     .await
 }
 
+/// Append a participant's annotation to a run that is still open.
+///
+/// Terminal outcomes are never rewritten. While the worker holds the run, the
+/// annotation is kept but the status is the worker's.
 async fn mutate_agent_run<F: FnOnce(&mut Value)>(
     state: &AppState,
     id: &str,
+    annotation: &str,
     change: F,
 ) -> Result<Json<RecordResponse>, ApiError> {
     validate_segment(id)?;
     let _guard = state.lifecycle_lock.lock().await;
     let mut run = required_value(state, "_flow_agent_runs", id, "agent run not found")?;
-    if run["status"] == "completed" {
-        return Err(ApiError(
+    let status = run["status"].as_str().unwrap_or_default().to_string();
+    if agent_execution::is_terminal(&status) {
+        return Err(ApiError::structured(
             StatusCode::CONFLICT,
-            "agent run is already completed".to_string(),
+            json!({"code": "AGENT_RUN_TERMINAL", "message": format!("agent run is already {status}")}),
         ));
     }
     if !run["observations"].is_array() || !run["decisions"].is_array() {
@@ -13111,6 +13797,9 @@ async fn mutate_agent_run<F: FnOnce(&mut Value)>(
         ));
     }
     change(&mut run);
+    if status != agent_execution::STATUS_RUNNING {
+        run["status"] = json!(annotation);
+    }
     put_canonical(state, "_flow_agent_runs", id, run)
 }
 
@@ -14214,6 +14903,7 @@ async fn commit_transaction(
         record_preconditions.push(precondition);
     }
     for operation in request.operations {
+        ensure_not_runtime_owned(&operation.collection)?;
         let key = record_key(&operation.collection, &operation.id)?;
         if !seen.insert(key.clone()) {
             return Err(ApiError(
@@ -14293,9 +14983,7 @@ async fn commit_transaction(
                 FlowError::CapabilityError(message) if message.starts_with("INVALID_READ_SET:") => {
                     ApiError(StatusCode::BAD_REQUEST, message)
                 }
-                FlowError::CapabilityError(message)
-                    if message.starts_with("TRANSACTION_ID_PAYLOAD_MISMATCH:") =>
-                {
+                FlowError::TransactionIdPayloadMismatch { .. } => {
                     ApiError::structured(
                         StatusCode::CONFLICT,
                         json!({
@@ -14336,6 +15024,21 @@ async fn commit_transaction(
     ))
 }
 
+/// Refuse a raw write to a collection only the runtime writes.
+///
+/// A run's status, retry state, evidence and authority decide what the worker
+/// does and what the run reports. Letting `/collections` or `/transactions`
+/// write them would let any `state:write` caller declare a run completed.
+fn ensure_not_runtime_owned(collection: &str) -> Result<(), ApiError> {
+    if agent_execution::is_runtime_owned_collection(collection) {
+        return Err(ApiError::structured(
+            StatusCode::FORBIDDEN,
+            json!({"code": "RUNTIME_OWNED_COLLECTION", "message": "this collection is written only by the runtime"}),
+        ));
+    }
+    Ok(())
+}
+
 async fn create_record(
     State(state): State<AppState>,
     Path(collection): Path<String>,
@@ -14343,6 +15046,7 @@ async fn create_record(
 ) -> Result<(StatusCode, Json<RecordResponse>), ApiError> {
     let _handler =
         feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::HandlerBody);
+    ensure_not_runtime_owned(&collection)?;
     if collection == PROPOSAL_COLLECTION
         || collection == PROPOSAL_EVENT_COLLECTION
         || collection == CONVERGENCE_COLLECTION
@@ -14391,6 +15095,7 @@ async fn update_record(
 ) -> Result<Json<RecordResponse>, ApiError> {
     let _handler =
         feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::HandlerBody);
+    ensure_not_runtime_owned(&collection)?;
     if collection == PROPOSAL_COLLECTION
         || collection == PROPOSAL_EVENT_COLLECTION
         || collection == CONVERGENCE_COLLECTION
@@ -14430,6 +15135,7 @@ async fn compare_and_set_record(
 ) -> Result<Response, ApiError> {
     let _handler =
         feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::HandlerBody);
+    ensure_not_runtime_owned(&collection)?;
     let key = record_key(&collection, &id)?;
     let result = state.db.compare_and_set_json(
         &key,
@@ -14521,6 +15227,7 @@ async fn delete_record(
 ) -> Result<StatusCode, ApiError> {
     let _handler =
         feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::HandlerBody);
+    ensure_not_runtime_owned(&collection)?;
     if collection == PROPOSAL_COLLECTION
         || collection == PROPOSAL_EVENT_COLLECTION
         || collection == CONVERGENCE_COLLECTION