@feltdb/core 0.12.2 → 0.15.0

package/dist/create/server-source/crates/feltdb/tests/production_readiness_contract.rs

@@ -1020,19 +1020,25 @@ fn retention_cost_grows_with_the_window_and_inflates_the_log() {
 
     // Compare the best of repeated independent runs. The workspace executes
     // other crash/soak processes concurrently, so a single wall-clock sample
-    // can measure scheduler delay rather than retention work. Taking the
-    // minimum preserves the envelope claim while filtering that one-sided
-    // noise; every sample still performs the complete durable workload.
-    fn sample(run_once: impl Fn() -> (std::time::Duration, u64)) -> (std::time::Duration, u64) {
-        (0..3)
-            .map(|_| run_once())
-            .min_by_key(|(elapsed, _)| *elapsed)
-            .unwrap()
+    // can measure scheduler delay rather than retention work. The minimum
+    // filters that one-sided noise, but only if every configuration is sampled
+    // across the same stretch of time: sampling them one after another lets a
+    // busy sibling inflate one configuration's every sample and not another's.
+    // So each round runs all three, and each keeps its own best. Every sample
+    // still performs the complete durable workload.
+    let mut none = (std::time::Duration::MAX, 0u64);
+    let mut small = (std::time::Duration::MAX, 0u64);
+    let mut large = std::time::Duration::MAX;
+    for _ in 0..9 {
+        let sample = run(None);
+        if sample.0 < none.0 { none = sample; }
+        let sample = run(Some(5));
+        if sample.0 < small.0 { small = sample; }
+        large = large.min(run(Some(50)).0);
     }
-
-    let (none_time, none_bytes) = sample(|| run(None));
-    let (small_time, small_bytes) = sample(|| run(Some(5)));
-    let (large_time, _) = sample(|| run(Some(50)));
+    let (none_time, none_bytes) = none;
+    let (small_time, small_bytes) = small;
+    let large_time = large;
 
     assert!(
         large_time > small_time,

package/dist/create/server-source/crates/feltdb/tests/workload_envelope_contract.rs

@@ -215,14 +215,26 @@ fn a_barrier_per_write_costs_more_than_none() {
     let directory = TempDir::new().unwrap();
     let workload = WORKLOADS[0];
 
-    let flushed = run(directory.path(), workload, DurabilityMode::Flushed);
+    // A stall can only invert this ordering by inflating the cheap, flushed
+    // run, so take the best of several of those against one synced run. The
+    // minimum is the cost of the mode with scheduler contention taken out;
+    // repeating the slow synced run would only add load to sibling timing
+    // tests in this concurrently-run binary.
     let synced = run(directory.path(), workload, DurabilityMode::Synced);
+    let flushed = run(directory.path(), workload, DurabilityMode::Flushed);
+    let mut best_flushed = flushed.elapsed;
+    for sample in 0..4 {
+        // A fresh log per sample: a reused path would append to its history.
+        let sample_directory = directory.path().join(format!("flushed-{sample}"));
+        std::fs::create_dir_all(&sample_directory).unwrap();
+        let repeat = run(&sample_directory, workload, DurabilityMode::Flushed);
+        best_flushed = best_flushed.min(repeat.elapsed);
+    }
 
     assert!(
-        synced.elapsed > flushed.elapsed,
-        "a barrier per write must cost something: synced {:?} vs flushed {:?}",
-        synced.elapsed,
-        flushed.elapsed
+        synced.elapsed > best_flushed,
+        "a barrier per write must cost something: synced {:?} vs flushed {best_flushed:?}",
+        synced.elapsed
     );
 
     // Durability is the only thing that changed. The database is identical.

package/dist/create/server-source/crates/feltdb-client/Cargo.toml

@@ -16,7 +16,7 @@ exclude = ["tests/", "target/"]
 # The wire types are the server's own: the request and response documents are defined once, in
 # `feltdb`, and both sides serialize the same structs. Exact, because the modules they come from are
 # not yet covered by `feltdb`'s compatibility policy; the workspace releases both at one version.
-feltdb = { version = "=0.2.1", path = "../feltdb" }
+feltdb = { version = "=0.3.0", path = "../feltdb" }
 reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"

package/dist/create/server-source/crates/feltdb-server/Cargo.toml

@@ -24,7 +24,7 @@ hmac = "0.12"
 # Exact: the server uses `unstable-internals`, which carries no semver promise, so a release of
 # this crate is built against the one `feltdb` release it was tested with. The workspace shares one
 # version, so this moves with every release (a mismatch fails to resolve, it cannot drift silently).
-feltdb = { version = "=0.2.1", path = "../feltdb", features = ["unstable-internals"] }
+feltdb = { version = "=0.3.0", path = "../feltdb", features = ["unstable-internals"] }
 futures-core = "0.3"
 rand = "0.8"
 ring = "0.17"

package/dist/create/server-source/crates/feltdb-server/README.md

@@ -53,7 +53,12 @@ Implemented in this slice:
 - collection list/create/get/update/delete endpoints
 - `GET /events` as a server-sent event stream
 - durable, hashed API keys with namespace and `state:read`, `state:write`, or `events:read` scopes
-- `GET /metrics` request, authorization, state, and event counters
+- `GET /metrics` request, authorization, state, and event counters (requires `operator:read`)
+- `GET /v1/status` operator status: release identity, a write/fsync/read-back storage probe, the
+  last recorded backup and the last verified restore (requires `operator:read`)
+- `/api/keys` listing (never secrets), issuing, `POST /api/keys/{id}/rotate` and revocation
+  (requires `keys:admin`; a caller can never delegate a scope it does not hold). See
+  [`docs/reference/managed-operator-surface.md`](../../docs/reference/managed-operator-surface.md).
 - authenticated `POST /sync/pull` and `POST /sync/push` operation exchange
 - durable, transitive operation history with content-hash verification, deduplication, and gap rejection
 

package/dist/create/server-source/crates/feltdb-server/src/agent_execution.rs

@@ -0,0 +1,511 @@
+//! Authority, completion, and failure rules for runs the autonomous worker executes.
+//!
+//! Agent and workflow runs are created by an authenticated request and executed
+//! later by the worker, which runs with the node's own authority. Without the
+//! rules here, every capability a run invokes would read with that node
+//! authority, and a run would end `completed` whenever a capability returned.
+//!
+//! - **Authority.** When a run is created, the server signs a grant that binds
+//!   the requesting principal's scopes to that one run ([`issue_run_authority`]).
+//!   The worker verifies it before executing ([`verify_run_authority`]) and
+//!   authorizes every collection read against it. The grant is signed with the
+//!   server's existing [`GrantSigner`], so a run record written or replayed by
+//!   anything other than the server — including a grant copied from another
+//!   run — is refused rather than trusted.
+//! - **Completion.** An agent run completes only when criteria declared at
+//!   creation are satisfied by the result the runtime itself observed
+//!   ([`CompletionCriteria`]). The goal text is never evaluated. A run without
+//!   verifiable criteria ends `failed` with [`COMPLETION_UNVERIFIABLE`]; nothing
+//!   a client posts can complete a run.
+//! - **Failure.** Permanent errors end the run at once; transient errors retry
+//!   at most [`max_run_attempts`] times, under the workload retry policy's bound.
+
+use std::collections::BTreeSet;
+
+use feltdb::authorization::{
+    Grant, GrantConstraints, GrantSigner, GrantStatus, SignedGrant, Subject,
+    AUTHORIZATION_CONTRACT_VERSION,
+};
+use feltdb::workload::WorkloadRetryPolicy;
+use serde::{Deserialize, Serialize};
+use serde_json::{json, Value};
+
+/// The issuer recorded on every run authority grant.
+pub const RUN_AUTHORITY_ISSUER: &str = "feltdb-run-authority";
+/// How long a run's delegated authority stays valid after the run is created.
+///
+/// A run that has not finished by then fails with [`AUTHORIZATION_DENIED`]
+/// rather than executing on a delegation its caller may no longer intend.
+pub const RUN_AUTHORITY_TTL_SECONDS: i64 = 24 * 60 * 60;
+
+/// Run statuses written by the runtime.
+pub const STATUS_READY: &str = "ready";
+pub const STATUS_RUNNING: &str = "running";
+pub const STATUS_COMPLETED: &str = "completed";
+pub const STATUS_FAILED: &str = "failed";
+/// Non-terminal annotations an external participant may set on an agent run.
+pub const STATUS_DELIBERATING: &str = "deliberating";
+pub const STATUS_ACTING: &str = "acting";
+
+/// Stable failure codes stored in a failed run's `failure.code`.
+pub const AUTHORIZATION_DENIED: &str = "AUTHORIZATION_DENIED";
+pub const AGENT_NOT_FOUND: &str = "AGENT_NOT_FOUND";
+pub const CAPABILITY_UNKNOWN: &str = "CAPABILITY_UNKNOWN";
+pub const CAPABILITY_INPUT_INVALID: &str = "CAPABILITY_INPUT_INVALID";
+pub const RETRY_EXHAUSTED: &str = "RETRY_EXHAUSTED";
+pub const COMPLETION_CRITERIA_UNMET: &str = "COMPLETION_CRITERIA_UNMET";
+pub const COMPLETION_UNVERIFIABLE: &str = "COMPLETION_UNVERIFIABLE";
+pub const RUN_INVALID: &str = "RUN_INVALID";
+
+/// Which kind of run a grant was issued for. It is part of the grant subject,
+/// so an agent run's authority cannot be presented by a workflow run.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum RunKind {
+    Agent,
+    Workflow,
+}
+
+impl RunKind {
+    fn subject(self, run_id: &str) -> Subject {
+        match self {
+            Self::Agent => Subject::Agent(format!("agent-run:{run_id}")),
+            Self::Workflow => Subject::Workload(format!("workflow-run:{run_id}")),
+        }
+    }
+}
+
+/// The principal a run acts for, as the server authenticated it.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct RunPrincipal {
+    pub id: String,
+    /// `service`, `worker`, or `human`, exactly as the authenticator decided.
+    pub kind: String,
+    pub scopes: BTreeSet<String>,
+}
+
+/// Sign the authority a run executes under.
+pub fn issue_run_authority(
+    signer: &GrantSigner,
+    kind: RunKind,
+    run_id: &str,
+    tenant_id: &str,
+    principal: &RunPrincipal,
+    issued_at: i64,
+) -> Result<SignedGrant, String> {
+    let mut constraints = GrantConstraints::default();
+    constraints
+        .subject_attributes
+        .insert("principal_id".into(), principal.id.clone());
+    constraints
+        .subject_attributes
+        .insert("principal_kind".into(), principal.kind.clone());
+    signer
+        .sign(Grant {
+            contract_version: AUTHORIZATION_CONTRACT_VERSION,
+            id: format!("run-authority:{run_id}"),
+            issuer: Subject::Service(RUN_AUTHORITY_ISSUER.into()),
+            subject: kind.subject(run_id),
+            tenant_id: tenant_id.into(),
+            application_id: tenant_id.into(),
+            capabilities: principal.scopes.clone(),
+            resources: Vec::new(),
+            constraints,
+            issued_at,
+            expires_at: issued_at.saturating_add(RUN_AUTHORITY_TTL_SECONDS),
+            parent_grant: None,
+            revision: 1,
+            status: GrantStatus::Active,
+        })
+        .map_err(|error| error.message)
+}
+
+/// Verify a run's stored authority and return the principal it delegates.
+///
+/// Every refusal is [`AUTHORIZATION_DENIED`] and permanent: retrying cannot
+/// change a signature, a binding, or an expiry.
+pub fn verify_run_authority(
+    signer: &GrantSigner,
+    stored: &Value,
+    kind: RunKind,
+    run_id: &str,
+    tenant_id: &str,
+    now: i64,
+) -> Result<RunPrincipal, RunFailure> {
+    let denied = |message: &str| RunFailure::permanent(AUTHORIZATION_DENIED, message);
+    if stored.is_null() {
+        return Err(denied("run carries no delegated authority"));
+    }
+    let signed: SignedGrant =
+        serde_json::from_value(stored.clone()).map_err(|_| denied("run authority is malformed"))?;
+    signer
+        .verify_signature(&signed)
+        .map_err(|_| denied("run authority signature is invalid"))?;
+    let grant = &signed.grant;
+    if grant.issuer != Subject::Service(RUN_AUTHORITY_ISSUER.into())
+        || grant.subject != kind.subject(run_id)
+        || grant.id != format!("run-authority:{run_id}")
+    {
+        return Err(denied("run authority was issued for a different run"));
+    }
+    if grant.tenant_id != tenant_id || grant.application_id != tenant_id {
+        return Err(denied("run authority belongs to a different tenant"));
+    }
+    if grant.status != GrantStatus::Active || now >= grant.expires_at || now < grant.issued_at {
+        return Err(denied("run authority is not current"));
+    }
+    let attribute = |name: &str| grant.constraints.subject_attributes.get(name).cloned();
+    let (Some(id), Some(kind)) = (attribute("principal_id"), attribute("principal_kind")) else {
+        return Err(denied("run authority names no principal"));
+    };
+    Ok(RunPrincipal {
+        id,
+        kind,
+        scopes: grant.capabilities.clone(),
+    })
+}
+
+/// A machine-checkable condition the runtime evaluates against the result it
+/// observed. The set is closed: anything else is refused when the run starts.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
+pub enum CompletionCriteria {
+    /// The result is an array with at least `min` (and at most `max`) items.
+    ResultCount { min: usize, max: Option<usize> },
+    /// The result is an array with an item whose `field` equals `equals`.
+    ResultContains { field: String, equals: Value },
+}
+
+impl CompletionCriteria {
+    /// Reject criteria that would be satisfied by any outcome.
+    pub fn validate(&self) -> Result<(), String> {
+        match self {
+            Self::ResultCount { min, max } => {
+                if *min == 0 {
+                    return Err("result_count requires min >= 1; a criterion every outcome satisfies verifies nothing".into());
+                }
+                if max.is_some_and(|max| max < *min) {
+                    return Err("result_count max must be >= min".into());
+                }
+            }
+            Self::ResultContains { field, .. } => {
+                if field.is_empty() || field.len() > 128 {
+                    return Err("result_contains requires a field name of 1..128 characters".into());
+                }
+            }
+        }
+        Ok(())
+    }
+
+    /// Whether `result`, as observed by the runtime, satisfies the criterion.
+    pub fn satisfied_by(&self, result: &Value) -> bool {
+        let Some(items) = result.as_array() else {
+            return false;
+        };
+        match self {
+            Self::ResultCount { min, max } => {
+                items.len() >= *min && max.is_none_or(|max| items.len() <= max)
+            }
+            Self::ResultContains { field, equals } => {
+                items.iter().any(|item| item.get(field) == Some(equals))
+            }
+        }
+    }
+}
+
+/// Why a run did not complete. `retryable` is decided by the runtime, never by
+/// the run record.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+pub struct RunFailure {
+    pub code: &'static str,
+    pub message: String,
+    pub retryable: bool,
+}
+
+impl RunFailure {
+    pub fn permanent(code: &'static str, message: impl Into<String>) -> Self {
+        Self {
+            code,
+            message: message.into(),
+            retryable: false,
+        }
+    }
+
+    pub fn to_record(&self, attempt: u64) -> Value {
+        json!({ "code": self.code, "message": self.message, "retryable": self.retryable, "attempt": attempt })
+    }
+}
+
+/// Classify a capability error by its HTTP status.
+///
+/// A capability reads committed state and nothing else, so a client error
+/// (bad input, missing record, refused authority) fails the same way on every
+/// attempt and is permanent. Only a server-side error can succeed on retry.
+pub fn classify_capability_error(status: u16, message: &str) -> RunFailure {
+    match status {
+        401 | 403 => RunFailure::permanent(AUTHORIZATION_DENIED, message),
+        404 => RunFailure::permanent(CAPABILITY_UNKNOWN, message),
+        400..=499 => RunFailure::permanent(CAPABILITY_INPUT_INVALID, message),
+        _ => RunFailure {
+            code: "CAPABILITY_FAILED",
+            message: message.into(),
+            retryable: true,
+        },
+    }
+}
+
+/// The scopes a run may use: those granted when it was created that the key
+/// still holds now. `*` on either side stands for the other side's set, so a
+/// wildcard never widens a narrowed key, and an empty set authorizes nothing.
+pub fn effective_scopes(
+    granted: &BTreeSet<String>,
+    current: &BTreeSet<String>,
+) -> BTreeSet<String> {
+    if granted.contains("*") {
+        current.clone()
+    } else if current.contains("*") {
+        granted.clone()
+    } else {
+        granted.intersection(current).cloned().collect()
+    }
+}
+
+/// The most attempts a run gets, shared with durable workloads.
+pub fn max_run_attempts() -> u64 {
+    u64::from(WorkloadRetryPolicy::default().max_attempts)
+}
+
+/// Delay before attempt `next_attempt` (2 or later), exponential from the
+/// workload policy's initial delay and capped at its maximum.
+pub fn retry_delay_ms(next_attempt: u64) -> u64 {
+    let policy = WorkloadRetryPolicy::default();
+    let exponent = next_attempt.saturating_sub(2).min(20) as u32;
+    policy
+        .initial_delay_ms
+        .saturating_mul(2u64.saturating_pow(exponent))
+        .min(policy.max_delay_ms)
+}
+
+/// Collections reserved for the server. No capability reads them.
+pub fn is_system_collection(collection: &str) -> bool {
+    collection.starts_with('_')
+}
+
+/// Collections the runtime alone writes. A raw write could otherwise set a
+/// run's status or retry state directly, or — for durable work, which the
+/// workload store loads from this collection at startup — inject a workload
+/// whose authorization snapshot was never verified.
+pub fn is_runtime_owned_collection(collection: &str) -> bool {
+    matches!(
+        collection,
+        "_flow_agent_runs" | "_flow_workflow_runs" | feltdb::workload::DURABLE_WORK_COLLECTION
+    )
+}
+
+pub fn is_terminal(status: &str) -> bool {
+    matches!(status, STATUS_COMPLETED | STATUS_FAILED)
+}
+
+/// Whether the worker may claim an agent run in `status`.
+///
+/// External annotations (`deliberating`, `acting`) leave a run eligible, so a
+/// client posting an observation cannot strand it outside the worker's reach.
+pub fn agent_run_claimable(status: &str, lease_expired: bool) -> bool {
+    match status {
+        STATUS_READY | STATUS_DELIBERATING | STATUS_ACTING => true,
+        STATUS_RUNNING => lease_expired,
+        _ => false,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn signer(seed: u8) -> GrantSigner {
+        GrantSigner::new("feltdb-primary-v1", vec![seed; 32]).unwrap()
+    }
+
+    fn principal() -> RunPrincipal {
+        RunPrincipal {
+            id: "key-1".into(),
+            kind: "service".into(),
+            scopes: ["agents:run".to_string(), "state:read".to_string()].into(),
+        }
+    }
+
+    fn issued(kind: RunKind, run: &str) -> Value {
+        serde_json::to_value(
+            issue_run_authority(&signer(7), kind, run, "tenant", &principal(), 1_000).unwrap(),
+        )
+        .unwrap()
+    }
+
+    #[test]
+    fn a_run_authority_round_trips_for_its_own_run() {
+        let verified = verify_run_authority(
+            &signer(7),
+            &issued(RunKind::Agent, "agent-1"),
+            RunKind::Agent,
+            "agent-1",
+            "tenant",
+            1_001,
+        )
+        .unwrap();
+        assert_eq!(verified, principal());
+    }
+
+    #[test]
+    fn a_run_authority_is_refused_for_any_other_binding() {
+        let grant = issued(RunKind::Agent, "agent-1");
+        let refuse = |value: &Value, kind, run: &str, tenant: &str, now| {
+            let failure =
+                verify_run_authority(&signer(7), value, kind, run, tenant, now).unwrap_err();
+            assert_eq!(failure.code, AUTHORIZATION_DENIED);
+            assert!(!failure.retryable);
+        };
+        refuse(&grant, RunKind::Agent, "agent-2", "tenant", 1_001);
+        refuse(&grant, RunKind::Workflow, "agent-1", "tenant", 1_001);
+        refuse(&grant, RunKind::Agent, "agent-1", "other-tenant", 1_001);
+        refuse(
+            &grant,
+            RunKind::Agent,
+            "agent-1",
+            "tenant",
+            1_000 + RUN_AUTHORITY_TTL_SECONDS,
+        );
+        refuse(&Value::Null, RunKind::Agent, "agent-1", "tenant", 1_001);
+        assert_eq!(
+            verify_run_authority(
+                &signer(8),
+                &grant,
+                RunKind::Agent,
+                "agent-1",
+                "tenant",
+                1_001
+            )
+            .unwrap_err()
+            .code,
+            AUTHORIZATION_DENIED,
+            "a grant from another signer is refused"
+        );
+    }
+
+    #[test]
+    fn widening_a_signed_run_authority_breaks_its_signature() {
+        let mut grant = issued(RunKind::Agent, "agent-1");
+        grant["grant"]["capabilities"] = json!(["*"]);
+        assert!(verify_run_authority(
+            &signer(7),
+            &grant,
+            RunKind::Agent,
+            "agent-1",
+            "tenant",
+            1_001
+        )
+        .is_err());
+        let mut grant = issued(RunKind::Agent, "agent-1");
+        grant["grant"]["constraints"]["subject_attributes"]["principal_id"] = json!("admin");
+        assert!(verify_run_authority(
+            &signer(7),
+            &grant,
+            RunKind::Agent,
+            "agent-1",
+            "tenant",
+            1_001
+        )
+        .is_err());
+    }
+
+    #[test]
+    fn completion_criteria_verify_the_observed_result_only() {
+        let count = CompletionCriteria::ResultCount { min: 1, max: None };
+        assert!(count.satisfied_by(&json!([{ "id": "a" }])));
+        assert!(!count.satisfied_by(&json!([])));
+        assert!(!count.satisfied_by(&json!({ "items": [1] })));
+        let bounded = CompletionCriteria::ResultCount {
+            min: 1,
+            max: Some(1),
+        };
+        assert!(!bounded.satisfied_by(&json!([1, 2])));
+        let contains = CompletionCriteria::ResultContains {
+            field: "id".into(),
+            equals: json!("doc-1"),
+        };
+        assert!(contains.satisfied_by(&json!([{ "id": "doc-1" }])));
+        assert!(!contains.satisfied_by(&json!([{ "id": "doc-2" }])));
+        assert!(CompletionCriteria::ResultCount { min: 0, max: None }
+            .validate()
+            .is_err());
+        assert!(CompletionCriteria::ResultCount {
+            min: 2,
+            max: Some(1)
+        }
+        .validate()
+        .is_err());
+        assert!(
+            serde_json::from_value::<CompletionCriteria>(json!({ "kind": "goal_achieved" }))
+                .is_err()
+        );
+        assert!(serde_json::from_value::<CompletionCriteria>(
+            json!({ "kind": "result_count", "min": 1, "trust": true })
+        )
+        .is_err());
+    }
+
+    #[test]
+    fn effective_scopes_never_exceed_either_side() {
+        let set = |values: &[&str]| {
+            values
+                .iter()
+                .map(|v| v.to_string())
+                .collect::<BTreeSet<_>>()
+        };
+        let wide = set(&["*"]);
+        let reader = set(&["agents:run", "state:read"]);
+        let runner = set(&["agents:run"]);
+        assert_eq!(
+            effective_scopes(&wide, &runner),
+            runner,
+            "a wildcard grant narrows to the current key"
+        );
+        assert_eq!(
+            effective_scopes(&runner, &wide),
+            runner,
+            "a wildcard key does not widen the grant"
+        );
+        assert_eq!(effective_scopes(&reader, &runner), runner);
+        assert_eq!(effective_scopes(&wide, &wide), wide);
+        assert!(effective_scopes(&reader, &set(&[])).is_empty());
+        assert!(effective_scopes(&set(&[]), &wide).is_empty());
+        assert!(effective_scopes(&set(&["state:read"]), &set(&["state:write"])).is_empty());
+    }
+
+    #[test]
+    fn only_server_errors_are_retryable() {
+        assert!(!classify_capability_error(403, "denied").retryable);
+        assert_eq!(
+            classify_capability_error(404, "missing").code,
+            CAPABILITY_UNKNOWN
+        );
+        assert_eq!(
+            classify_capability_error(400, "bad").code,
+            CAPABILITY_INPUT_INVALID
+        );
+        assert!(classify_capability_error(500, "storage").retryable);
+        assert_eq!(max_run_attempts(), 3);
+        assert_eq!(retry_delay_ms(2), 1_000);
+        assert_eq!(retry_delay_ms(3), 2_000);
+        assert_eq!(retry_delay_ms(99), 300_000);
+    }
+
+    #[test]
+    fn terminal_runs_are_never_claimable() {
+        for status in [STATUS_COMPLETED, STATUS_FAILED] {
+            assert!(is_terminal(status));
+            assert!(!agent_run_claimable(status, true));
+        }
+        assert!(agent_run_claimable(STATUS_DELIBERATING, false));
+        assert!(!agent_run_claimable(STATUS_RUNNING, false));
+        assert!(agent_run_claimable(STATUS_RUNNING, true));
+    }
+}

package/dist/create/server-source/crates/feltdb-server/src/app_state.rs

@@ -21,7 +21,7 @@ use crate::{
     auth::KeyStore, canonical_principals::CanonicalPrincipalAuthority,
     causal::CanonicalEventAuthority,
     clock::LeaseClock, cluster::ClusterStore,
-    connections::ConnectionStore, content::ContentStore, identity::IdentityStore,
+    connections::ConnectionStore, console_accounts::ConsoleAccountStore, content::ContentStore, identity::IdentityStore,
     leases::LeaseStore, metrics::Metrics, portable_bundle::BundleStore,
     providers::ProviderStore, releases::ReleaseStore, sessions::SessionVerifier,
     tenancy::TenancyStore,
@@ -60,6 +60,12 @@ pub struct AppState {
     /// arbiter; this semaphore only bounds pre-commit working memory.
     pub state_transaction_admission: Arc<tokio::sync::Semaphore>,
     pub worker_enabled: bool,
+    /// Started with `--work-authority`: this node commits Durable Work even
+    /// with peers. See `is_work_authority`.
+    pub work_authority: bool,
+    /// A peer also reports being the Durable Work authority. Set by peer sync;
+    /// while set, this node refuses Durable Work writes (fail closed).
+    pub work_authority_conflict: Arc<std::sync::atomic::AtomicBool>,
     pub leases: LeaseStore,
     pub content: ContentStore,
     pub artifacts: ArtifactStore,
@@ -74,6 +80,8 @@ pub struct AppState {
     pub backup_artifacts: Arc<Vec<(String, PathBuf)>>,
     pub tenancy: TenancyStore,
     pub sessions: SessionVerifier,
+    /// Console (website) accounts; the only source of console user ids.
+    pub console_accounts: ConsoleAccountStore,
     pub principals: CanonicalPrincipalAuthority,
     pub connections: ConnectionStore,
     pub identities: IdentityStore,