@feltdb/core 0.4.12 → 0.4.14

package/dist/create/server-source/crates/feltdb/src/workload.rs

@@ -0,0 +1,1310 @@
+//! Durable workload authority. Workers execute; this state machine decides.
+use crate::authorization::{SignedGrant, Subject};
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use sha2::{Digest, Sha256};
+use std::collections::{BTreeMap, BTreeSet};
+use std::fs;
+use std::path::{Path, PathBuf};
+pub const WORKLOAD_CONTRACT_VERSION: u32 = 1;
+
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
+pub enum WorkloadState {
+    Created,
+    Ready,
+    Claimed,
+    Running,
+    Checkpointed,
+    Recovering,
+    CancellationRequested,
+    Succeeded,
+    Failed,
+    Cancelled,
+    TimedOut,
+    DeadLetter,
+}
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "snake_case")]
+pub enum BackoffPolicy {
+    Fixed,
+    Exponential,
+}
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct WorkloadRetryPolicy {
+    pub max_attempts: u32,
+    pub initial_delay_ms: u64,
+    pub max_delay_ms: u64,
+    pub backoff: BackoffPolicy,
+    #[serde(default)]
+    pub retry_on: BTreeSet<String>,
+}
+impl Default for WorkloadRetryPolicy {
+    fn default() -> Self {
+        Self {
+            max_attempts: 3,
+            initial_delay_ms: 1000,
+            max_delay_ms: 300_000,
+            backoff: BackoffPolicy::Exponential,
+            retry_on: [
+                "timeout".into(),
+                "worker_failure".into(),
+                "transient_connection".into(),
+            ]
+            .into(),
+        }
+    }
+}
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct WorkloadTimeouts {
+    pub queue_ms: Option<u64>,
+    pub claim_ms: Option<u64>,
+    pub execution_ms: Option<u64>,
+    pub heartbeat_ms: Option<u64>,
+    pub overall_ms: Option<u64>,
+}
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct WorkloadDependency {
+    pub workload_id: String,
+    #[serde(default = "succeeded_state")]
+    pub required_state: WorkloadState,
+}
+fn succeeded_state() -> WorkloadState {
+    WorkloadState::Succeeded
+}
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct WorkloadClaim {
+    pub claim_id: String,
+    pub workload_id: String,
+    pub worker_id: String,
+    pub lease_id: String,
+    pub fencing_token: u64,
+    pub capability_snapshot: BTreeSet<String>,
+    pub claimed_at: i64,
+    pub expires_at: i64,
+    pub last_heartbeat: i64,
+}
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
+pub enum AttemptStatus {
+    Claimed,
+    Running,
+    Checkpointed,
+    Succeeded,
+    Failed,
+    TimedOut,
+    Cancelled,
+    WorkerLost,
+}
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct WorkloadAttempt {
+    pub attempt_id: String,
+    pub workload_id: String,
+    pub worker_id: String,
+    pub claim_id: String,
+    pub fencing_token: u64,
+    pub started_at: Option<i64>,
+    pub ended_at: Option<i64>,
+    pub input_hash: String,
+    pub result_hash: Option<String>,
+    pub error: Option<String>,
+    #[serde(default)]
+    pub resource_usage: BTreeMap<String, u64>,
+    pub status: AttemptStatus,
+}
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct WorkloadCheckpoint {
+    pub checkpoint_id: String,
+    pub workload_id: String,
+    pub attempt_id: String,
+    pub sequence: u64,
+    pub state: Value,
+    pub state_hash: String,
+    pub created_at: i64,
+    pub produced_by: String,
+}
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct WorkloadResult {
+    pub result_id: String,
+    pub workload_id: String,
+    pub attempt_id: String,
+    pub status: String,
+    pub output: Value,
+    pub output_hash: String,
+    #[serde(default)]
+    pub artifacts: Vec<String>,
+    #[serde(default)]
+    pub provenance: BTreeMap<String, String>,
+    pub produced_by: String,
+    pub created_at: i64,
+    pub schema_version: u64,
+}
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct WorkloadExecutionContext {
+    pub workload_id: String,
+    pub attempt_id: String,
+    pub worker_id: String,
+    pub capabilities: BTreeSet<String>,
+    pub connection_scopes: BTreeSet<String>,
+    pub secret_references: Vec<String>,
+    pub expires_at: i64,
+    pub fencing_token: u64,
+}
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct WorkloadEvent {
+    pub sequence: u64,
+    pub event_type: String,
+    pub from: Option<WorkloadState>,
+    pub to: WorkloadState,
+    pub actor: String,
+    pub at: i64,
+    #[serde(default)]
+    pub evidence: BTreeMap<String, String>,
+}
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Workload {
+    pub contract_version: u32,
+    pub workload_id: String,
+    pub tenant_id: String,
+    pub application_id: String,
+    pub environment: String,
+    pub definition_id: String,
+    pub definition_revision: String,
+    pub trigger: String,
+    pub input: Value,
+    pub input_hash: String,
+    pub requested_by: Subject,
+    pub authorization_snapshot: SignedGrant,
+    pub capability_snapshot: BTreeSet<String>,
+    pub connection_scopes: BTreeSet<String>,
+    pub priority: i32,
+    pub idempotency_key: String,
+    pub created_at: i64,
+    pub updated_at: i64,
+    pub state: WorkloadState,
+    pub execution_history: Vec<WorkloadEvent>,
+    pub attempts: Vec<WorkloadAttempt>,
+    pub checkpoints: Vec<WorkloadCheckpoint>,
+    pub result: Option<WorkloadResult>,
+    pub retry_policy: WorkloadRetryPolicy,
+    pub timeouts: WorkloadTimeouts,
+    pub attempt_count: u32,
+    pub retry_count: u32,
+    pub next_attempt_at: Option<i64>,
+    pub last_failure: Option<String>,
+    pub current_claim: Option<WorkloadClaim>,
+    pub fencing_token: u64,
+    #[serde(default)]
+    pub dependencies: Vec<WorkloadDependency>,
+}
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct CreateWorkload {
+    pub tenant_id: String,
+    pub application_id: String,
+    pub environment: String,
+    pub definition_id: String,
+    pub definition_revision: String,
+    pub trigger: String,
+    pub input: Value,
+    pub requested_by: Subject,
+    pub authorization_snapshot: SignedGrant,
+    pub capability_snapshot: BTreeSet<String>,
+    #[serde(default)]
+    pub connection_scopes: BTreeSet<String>,
+    #[serde(default)]
+    pub priority: i32,
+    pub idempotency_key: String,
+    #[serde(default)]
+    pub retry_policy: WorkloadRetryPolicy,
+    #[serde(default)]
+    pub timeouts: WorkloadTimeouts,
+    #[serde(default)]
+    pub dependencies: Vec<WorkloadDependency>,
+    pub created_at: i64,
+}
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct WorkloadError {
+    pub code: String,
+    pub message: String,
+}
+impl WorkloadError {
+    fn new(code: &str, message: impl Into<String>) -> Self {
+        Self {
+            code: code.into(),
+            message: message.into(),
+        }
+    }
+}
+impl std::fmt::Display for WorkloadError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}: {}", self.code, self.message)
+    }
+}
+impl std::error::Error for WorkloadError {}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct WorkloadStore {
+    #[serde(skip)]
+    path: PathBuf,
+    #[serde(default)]
+    pub workloads: BTreeMap<String, Workload>,
+    #[serde(default)]
+    idempotency: BTreeMap<String, String>,
+    #[serde(default)]
+    result_ids: BTreeMap<String, String>,
+    #[serde(default)]
+    event_sequence: u64,
+}
+impl WorkloadStore {
+    pub fn load(path: impl AsRef<Path>) -> Result<Self, WorkloadError> {
+        let path = path.as_ref().to_path_buf();
+        if !path.exists() {
+            return Ok(Self {
+                path,
+                workloads: BTreeMap::new(),
+                idempotency: BTreeMap::new(),
+                result_ids: BTreeMap::new(),
+                event_sequence: 0,
+            });
+        }
+        let mut value: Self = serde_json::from_slice(&fs::read(&path).map_err(io)?).map_err(io)?;
+        value.path = path;
+        Ok(value)
+    }
+    fn persist(&self) -> Result<(), WorkloadError> {
+        if let Some(parent) = self.path.parent() {
+            fs::create_dir_all(parent).map_err(io)?;
+        }
+        let temp = self.path.with_extension("tmp");
+        fs::write(&temp, serde_json::to_vec_pretty(self).map_err(io)?).map_err(io)?;
+        fs::rename(temp, &self.path).map_err(io)
+    }
+    pub fn create(&mut self, input: CreateWorkload) -> Result<Workload, WorkloadError> {
+        ensure_safe(&input.input)?;
+        if input.idempotency_key.trim().is_empty() {
+            return Err(WorkloadError::new(
+                "WORKLOAD_INVALID",
+                "idempotency key is required",
+            ));
+        }
+        let idempotency = format!(
+            "{}:{}:{}",
+            input.tenant_id, input.application_id, input.idempotency_key
+        );
+        if let Some(id) = self.idempotency.get(&idempotency) {
+            return Ok(self.workloads[id].clone());
+        }
+        if input.authorization_snapshot.grant.subject != input.requested_by
+            || input.authorization_snapshot.grant.tenant_id != input.tenant_id
+            || input.authorization_snapshot.grant.application_id != input.application_id
+        {
+            return Err(WorkloadError::new(
+                "WORKLOAD_UNAUTHORIZED",
+                "authorization snapshot does not bind the requester",
+            ));
+        }
+        let seed = format!(
+            "{idempotency}:{}:{}",
+            input.definition_revision,
+            hash(&input.input)
+        );
+        let id = format!("workload_{}", &hex(&Sha256::digest(seed))[..24]);
+        let input_hash = hash(&input.input);
+        let mut workload = Workload {
+            contract_version: 1,
+            workload_id: id.clone(),
+            tenant_id: input.tenant_id,
+            application_id: input.application_id,
+            environment: input.environment,
+            definition_id: input.definition_id,
+            definition_revision: input.definition_revision,
+            trigger: input.trigger,
+            input: input.input,
+            input_hash,
+            requested_by: input.requested_by,
+            authorization_snapshot: input.authorization_snapshot,
+            capability_snapshot: input.capability_snapshot,
+            connection_scopes: input.connection_scopes,
+            priority: input.priority,
+            idempotency_key: input.idempotency_key,
+            created_at: input.created_at,
+            updated_at: input.created_at,
+            state: WorkloadState::Created,
+            execution_history: vec![],
+            attempts: vec![],
+            checkpoints: vec![],
+            result: None,
+            retry_policy: input.retry_policy,
+            timeouts: input.timeouts,
+            attempt_count: 0,
+            retry_count: 0,
+            next_attempt_at: None,
+            last_failure: None,
+            current_claim: None,
+            fencing_token: 0,
+            dependencies: input.dependencies,
+        };
+        self.transition(
+            &mut workload,
+            WorkloadState::Ready,
+            "rust",
+            input.created_at,
+            BTreeMap::new(),
+        )?;
+        self.idempotency.insert(idempotency, id.clone());
+        self.workloads.insert(id, workload.clone());
+        self.persist()?;
+        Ok(workload)
+    }
+    pub fn get(&self, id: &str) -> Result<&Workload, WorkloadError> {
+        self.workloads
+            .get(id)
+            .ok_or_else(|| WorkloadError::new("WORKLOAD_NOT_FOUND", "workload not found"))
+    }
+    pub fn list(&self, tenant: &str, application: &str) -> Vec<Workload> {
+        let mut values = self
+            .workloads
+            .values()
+            .filter(|v| v.tenant_id == tenant && v.application_id == application)
+            .cloned()
+            .collect::<Vec<_>>();
+        values.sort_by_key(|v| (std::cmp::Reverse(v.priority), v.created_at));
+        values
+    }
+    fn edit<R>(
+        &mut self,
+        id: &str,
+        action: impl FnOnce(&mut Self, &mut Workload) -> Result<R, WorkloadError>,
+    ) -> Result<R, WorkloadError> {
+        let mut workload = self
+            .workloads
+            .remove(id)
+            .ok_or_else(|| WorkloadError::new("WORKLOAD_NOT_FOUND", "workload not found"))?;
+        let result = action(self, &mut workload);
+        self.workloads.insert(id.into(), workload);
+        if result.is_ok() {
+            self.persist()?;
+        }
+        result
+    }
+    fn transition(
+        &mut self,
+        w: &mut Workload,
+        to: WorkloadState,
+        actor: &str,
+        at: i64,
+        evidence: BTreeMap<String, String>,
+    ) -> Result<(), WorkloadError> {
+        if !legal(w.state, to) {
+            return Err(WorkloadError::new(
+                "WORKLOAD_TRANSITION_DENIED",
+                format!("{:?} cannot transition to {:?}", w.state, to),
+            ));
+        }
+        let from = w.state;
+        w.state = to;
+        w.updated_at = at;
+        self.event_sequence += 1;
+        w.execution_history.push(WorkloadEvent {
+            sequence: self.event_sequence,
+            event_type: event_name(to).into(),
+            from: Some(from),
+            to,
+            actor: actor.into(),
+            at,
+            evidence,
+        });
+        Ok(())
+    }
+    pub fn claim(
+        &mut self,
+        id: &str,
+        worker: &str,
+        capabilities: &BTreeSet<String>,
+        now: i64,
+        lease_ms: u64,
+    ) -> Result<WorkloadClaim, WorkloadError> {
+        self.edit(id, |store, w| {
+            if w.state != WorkloadState::Ready || w.next_attempt_at.is_some_and(|v| v > now) {
+                return Err(WorkloadError::new(
+                    "WORKLOAD_NOT_READY",
+                    "workload is not eligible",
+                ));
+            }
+            if !w.capability_snapshot.is_subset(capabilities) {
+                return Err(WorkloadError::new(
+                    "WORKLOAD_CAPABILITY_DENIED",
+                    "worker lacks required capability snapshot",
+                ));
+            }
+            w.fencing_token += 1;
+            w.attempt_count += 1;
+            let seed = format!("{}:{}:{}", w.workload_id, w.fencing_token, worker);
+            let claim = WorkloadClaim {
+                claim_id: format!("claim_{}", &hex(&Sha256::digest(seed.as_bytes()))[..20]),
+                workload_id: w.workload_id.clone(),
+                worker_id: worker.into(),
+                lease_id: format!("lease_{}", w.fencing_token),
+                fencing_token: w.fencing_token,
+                capability_snapshot: w.capability_snapshot.clone(),
+                claimed_at: now,
+                expires_at: now + (lease_ms / 1000) as i64,
+                last_heartbeat: now,
+            };
+            w.attempts.push(WorkloadAttempt {
+                attempt_id: format!("attempt_{}_{}", w.workload_id, w.attempt_count),
+                workload_id: w.workload_id.clone(),
+                worker_id: worker.into(),
+                claim_id: claim.claim_id.clone(),
+                fencing_token: claim.fencing_token,
+                started_at: None,
+                ended_at: None,
+                input_hash: w.input_hash.clone(),
+                result_hash: None,
+                error: None,
+                resource_usage: BTreeMap::new(),
+                status: AttemptStatus::Claimed,
+            });
+            w.current_claim = Some(claim.clone());
+            store.transition(
+                w,
+                WorkloadState::Claimed,
+                worker,
+                now,
+                [("fencing_token".into(), claim.fencing_token.to_string())].into(),
+            )?;
+            Ok(claim)
+        })
+    }
+    fn fence(
+        w: &Workload,
+        claim: &str,
+        worker: &str,
+        token: u64,
+        now: i64,
+    ) -> Result<(), WorkloadError> {
+        let current = w
+            .current_claim
+            .as_ref()
+            .ok_or_else(|| WorkloadError::new("WORKLOAD_CLAIM_INVALID", "no active claim"))?;
+        if current.claim_id != claim
+            || current.worker_id != worker
+            || current.fencing_token != token
+        {
+            return Err(WorkloadError::new(
+                "WORKLOAD_STALE_FENCE",
+                "claim or fencing token is stale",
+            ));
+        }
+        if current.expires_at <= now {
+            return Err(WorkloadError::new(
+                "WORKLOAD_LEASE_EXPIRED",
+                "claim lease expired",
+            ));
+        }
+        Ok(())
+    }
+    pub fn start(
+        &mut self,
+        id: &str,
+        claim: &str,
+        worker: &str,
+        token: u64,
+        now: i64,
+    ) -> Result<(), WorkloadError> {
+        self.edit(id, |store, w| {
+            Self::fence(w, claim, worker, token, now)?;
+            let attempt = w.attempts.last_mut().unwrap();
+            attempt.status = AttemptStatus::Running;
+            attempt.started_at = Some(now);
+            store.transition(w, WorkloadState::Running, worker, now, BTreeMap::new())
+        })
+    }
+    pub fn heartbeat(
+        &mut self,
+        id: &str,
+        claim: &str,
+        worker: &str,
+        token: u64,
+        now: i64,
+        extend_ms: u64,
+    ) -> Result<WorkloadClaim, WorkloadError> {
+        self.edit(id, |_, w| {
+            Self::fence(w, claim, worker, token, now)?;
+            let current = w.current_claim.as_mut().unwrap();
+            current.last_heartbeat = now;
+            current.expires_at = now + (extend_ms / 1000) as i64;
+            Ok(current.clone())
+        })
+    }
+    pub fn execution_context(
+        &self,
+        id: &str,
+        claim: &str,
+        worker: &str,
+        token: u64,
+        now: i64,
+        secret_references: Vec<String>,
+    ) -> Result<WorkloadExecutionContext, WorkloadError> {
+        let workload = self.get(id)?;
+        Self::fence(workload, claim, worker, token, now)?;
+        if secret_references
+            .iter()
+            .any(|v| !v.starts_with("secret://"))
+        {
+            return Err(WorkloadError::new(
+                "WORKLOAD_SECRET_REJECTED",
+                "execution context accepts secret references only",
+            ));
+        }
+        let current = workload.current_claim.as_ref().unwrap();
+        Ok(WorkloadExecutionContext {
+            workload_id: id.into(),
+            attempt_id: workload.attempts.last().unwrap().attempt_id.clone(),
+            worker_id: worker.into(),
+            capabilities: current.capability_snapshot.clone(),
+            connection_scopes: workload.connection_scopes.clone(),
+            secret_references,
+            expires_at: current.expires_at,
+            fencing_token: token,
+        })
+    }
+    pub fn checkpoint(
+        &mut self,
+        id: &str,
+        claim: &str,
+        worker: &str,
+        token: u64,
+        state: Value,
+        now: i64,
+    ) -> Result<WorkloadCheckpoint, WorkloadError> {
+        ensure_safe(&state)?;
+        self.edit(id, |store, w| {
+            Self::fence(w, claim, worker, token, now)?;
+            if !matches!(
+                w.state,
+                WorkloadState::Running | WorkloadState::Checkpointed
+            ) {
+                return Err(WorkloadError::new(
+                    "WORKLOAD_TRANSITION_DENIED",
+                    "checkpoint requires a running workload",
+                ));
+            }
+            let attempt = w.attempts.last_mut().unwrap();
+            attempt.status = AttemptStatus::Checkpointed;
+            let sequence = w.checkpoints.len() as u64 + 1;
+            let checkpoint = WorkloadCheckpoint {
+                checkpoint_id: format!("checkpoint_{}_{}", w.workload_id, sequence),
+                workload_id: w.workload_id.clone(),
+                attempt_id: attempt.attempt_id.clone(),
+                sequence,
+                state_hash: hash(&state),
+                state,
+                created_at: now,
+                produced_by: worker.into(),
+            };
+            w.checkpoints.push(checkpoint.clone());
+            if w.state == WorkloadState::Running {
+                store.transition(w, WorkloadState::Checkpointed, worker, now, BTreeMap::new())?;
+            }
+            Ok(checkpoint)
+        })
+    }
+    pub fn complete(
+        &mut self,
+        id: &str,
+        claim: &str,
+        worker: &str,
+        token: u64,
+        result_id: &str,
+        output: Value,
+        artifacts: Vec<String>,
+        schema_version: u64,
+        now: i64,
+    ) -> Result<WorkloadResult, WorkloadError> {
+        ensure_safe(&output)?;
+        if let Some(workload_id) = self.result_ids.get(result_id) {
+            if workload_id == id {
+                return Ok(self.get(id)?.result.clone().unwrap());
+            }
+            return Err(WorkloadError::new(
+                "WORKLOAD_RESULT_DUPLICATE",
+                "result id belongs to another workload",
+            ));
+        }
+        let result = self.edit(id, |store, w| {
+            Self::fence(w, claim, worker, token, now)?;
+            if !matches!(
+                w.state,
+                WorkloadState::Running | WorkloadState::Checkpointed
+            ) {
+                return Err(WorkloadError::new(
+                    "WORKLOAD_TRANSITION_DENIED",
+                    "completion requires running state",
+                ));
+            }
+            let attempt = w.attempts.last_mut().unwrap();
+            let output_hash = hash(&output);
+            attempt.status = AttemptStatus::Succeeded;
+            attempt.ended_at = Some(now);
+            attempt.result_hash = Some(output_hash.clone());
+            let result = WorkloadResult {
+                result_id: result_id.into(),
+                workload_id: w.workload_id.clone(),
+                attempt_id: attempt.attempt_id.clone(),
+                status: "succeeded".into(),
+                output,
+                output_hash,
+                artifacts,
+                provenance: [
+                    ("claim_id".into(), claim.into()),
+                    ("fencing_token".into(), token.to_string()),
+                ]
+                .into(),
+                produced_by: worker.into(),
+                created_at: now,
+                schema_version,
+            };
+            w.result = Some(result.clone());
+            w.current_claim = None;
+            store.transition(w, WorkloadState::Succeeded, worker, now, BTreeMap::new())?;
+            Ok(result)
+        })?;
+        self.result_ids.insert(result_id.into(), id.into());
+        self.persist()?;
+        Ok(result)
+    }
+    pub fn fail(
+        &mut self,
+        id: &str,
+        claim: &str,
+        worker: &str,
+        token: u64,
+        kind: &str,
+        error: &str,
+        now: i64,
+    ) -> Result<WorkloadState, WorkloadError> {
+        if sensitive_text(error) {
+            return Err(WorkloadError::new(
+                "WORKLOAD_SECRET_REJECTED",
+                "errors may not contain secret material",
+            ));
+        }
+        self.edit(id, |store, w| {
+            Self::fence(w, claim, worker, token, now)?;
+            let attempt = w.attempts.last_mut().unwrap();
+            attempt.status = AttemptStatus::Failed;
+            attempt.ended_at = Some(now);
+            attempt.error = Some(error.into());
+            w.last_failure = Some(format!("{kind}: {error}"));
+            w.current_claim = None;
+            store.transition(
+                w,
+                WorkloadState::Failed,
+                worker,
+                now,
+                [("failure_kind".into(), kind.into())].into(),
+            )?;
+            let retryable = w.retry_policy.retry_on.contains(kind)
+                && w.attempt_count < w.retry_policy.max_attempts;
+            if retryable {
+                w.retry_count += 1;
+                let multiplier = if w.retry_policy.backoff == BackoffPolicy::Exponential {
+                    2u64.saturating_pow(w.retry_count.saturating_sub(1))
+                } else {
+                    1
+                };
+                let delay = w
+                    .retry_policy
+                    .initial_delay_ms
+                    .saturating_mul(multiplier)
+                    .min(w.retry_policy.max_delay_ms);
+                w.next_attempt_at = Some(now + (delay / 1000) as i64);
+                store.transition(w, WorkloadState::Ready, "rust.retry", now, BTreeMap::new())?;
+            } else {
+                store.transition(
+                    w,
+                    WorkloadState::DeadLetter,
+                    "rust.retry",
+                    now,
+                    BTreeMap::new(),
+                )?;
+            }
+            Ok(w.state)
+        })
+    }
+    pub fn cancel(
+        &mut self,
+        id: &str,
+        actor: &str,
+        now: i64,
+    ) -> Result<WorkloadState, WorkloadError> {
+        self.edit(id, |store, w| {
+            match w.state {
+                WorkloadState::Ready
+                | WorkloadState::Created
+                | WorkloadState::Failed
+                | WorkloadState::DeadLetter => {
+                    w.current_claim = None;
+                    store.transition(w, WorkloadState::Cancelled, actor, now, BTreeMap::new())?
+                }
+                WorkloadState::Claimed | WorkloadState::Running | WorkloadState::Checkpointed => {
+                    store.transition(
+                        w,
+                        WorkloadState::CancellationRequested,
+                        actor,
+                        now,
+                        BTreeMap::new(),
+                    )?
+                }
+                WorkloadState::CancellationRequested => {}
+                _ => {
+                    return Err(WorkloadError::new(
+                        "WORKLOAD_TRANSITION_DENIED",
+                        "terminal workload cannot be cancelled",
+                    ))
+                }
+            }
+            Ok(w.state)
+        })
+    }
+    pub fn confirm_cancel(
+        &mut self,
+        id: &str,
+        claim: &str,
+        worker: &str,
+        token: u64,
+        now: i64,
+    ) -> Result<(), WorkloadError> {
+        self.edit(id, |store, w| {
+            Self::fence(w, claim, worker, token, now)?;
+            if w.state != WorkloadState::CancellationRequested {
+                return Err(WorkloadError::new(
+                    "WORKLOAD_TRANSITION_DENIED",
+                    "cancellation was not requested",
+                ));
+            }
+            if let Some(attempt) = w.attempts.last_mut() {
+                attempt.status = AttemptStatus::Cancelled;
+                attempt.ended_at = Some(now);
+            }
+            w.current_claim = None;
+            store.transition(w, WorkloadState::Cancelled, worker, now, BTreeMap::new())
+        })
+    }
+    pub fn timeout(
+        &mut self,
+        id: &str,
+        actor: &str,
+        now: i64,
+    ) -> Result<WorkloadState, WorkloadError> {
+        self.edit(id, |store, w| {
+            if !matches!(
+                w.state,
+                WorkloadState::Ready
+                    | WorkloadState::Claimed
+                    | WorkloadState::Running
+                    | WorkloadState::Checkpointed
+            ) {
+                return Err(WorkloadError::new(
+                    "WORKLOAD_TRANSITION_DENIED",
+                    "workload cannot time out in its current state",
+                ));
+            }
+            if let Some(attempt) = w.attempts.last_mut() {
+                attempt.status = AttemptStatus::TimedOut;
+                attempt.ended_at = Some(now);
+            }
+            w.current_claim = None;
+            store.transition(w, WorkloadState::TimedOut, actor, now, BTreeMap::new())?;
+            if w.attempt_count < w.retry_policy.max_attempts
+                && w.retry_policy.retry_on.contains("timeout")
+            {
+                w.retry_count += 1;
+                w.next_attempt_at = Some(now + (w.retry_policy.initial_delay_ms / 1000) as i64);
+                store.transition(w, WorkloadState::Ready, "rust.retry", now, BTreeMap::new())?;
+            } else {
+                store.transition(
+                    w,
+                    WorkloadState::DeadLetter,
+                    "rust.timeout",
+                    now,
+                    BTreeMap::new(),
+                )?;
+            }
+            Ok(w.state)
+        })
+    }
+    pub fn retry(&mut self, id: &str, actor: &str, now: i64) -> Result<(), WorkloadError> {
+        self.edit(id, |store, w| {
+            if !matches!(
+                w.state,
+                WorkloadState::Failed | WorkloadState::DeadLetter | WorkloadState::TimedOut
+            ) {
+                return Err(WorkloadError::new(
+                    "WORKLOAD_TRANSITION_DENIED",
+                    "workload is not retryable",
+                ));
+            }
+            w.next_attempt_at = Some(now);
+            w.current_claim = None;
+            store.transition(w, WorkloadState::Ready, actor, now, BTreeMap::new())
+        })
+    }
+    pub fn recover_expired(&mut self, now: i64) -> Result<Vec<String>, WorkloadError> {
+        let ids = self
+            .workloads
+            .values()
+            .filter(|w| {
+                w.current_claim
+                    .as_ref()
+                    .is_some_and(|c| c.expires_at <= now)
+                    && matches!(
+                        w.state,
+                        WorkloadState::Claimed
+                            | WorkloadState::Running
+                            | WorkloadState::Checkpointed
+                            | WorkloadState::CancellationRequested
+                    )
+            })
+            .map(|w| w.workload_id.clone())
+            .collect::<Vec<_>>();
+        for id in &ids {
+            self.edit(id, |store, w| {
+                if let Some(attempt) = w.attempts.last_mut() {
+                    attempt.status = AttemptStatus::WorkerLost;
+                    attempt.ended_at = Some(now);
+                }
+                w.current_claim = None;
+                if w.state == WorkloadState::CancellationRequested {
+                    store.transition(
+                        w,
+                        WorkloadState::Cancelled,
+                        "rust.recovery",
+                        now,
+                        BTreeMap::new(),
+                    )?;
+                } else {
+                    store.transition(
+                        w,
+                        WorkloadState::Recovering,
+                        "rust.recovery",
+                        now,
+                        BTreeMap::new(),
+                    )?;
+                    store.transition(
+                        w,
+                        WorkloadState::Ready,
+                        "rust.recovery",
+                        now,
+                        BTreeMap::new(),
+                    )?;
+                }
+                Ok(())
+            })?;
+        }
+        Ok(ids)
+    }
+}
+
+pub trait WorkScheduler {
+    fn enqueue(&mut self, workload_id: &str);
+    fn claim(
+        &mut self,
+        store: &mut WorkloadStore,
+        tenant: &str,
+        application: &str,
+        worker: &str,
+        capabilities: &BTreeSet<String>,
+        now: i64,
+        lease_ms: u64,
+    ) -> Result<Option<WorkloadClaim>, WorkloadError>;
+    fn release(&mut self, claim_id: &str);
+}
+#[derive(Default)]
+pub struct LocalDurableScheduler {
+    queue: BTreeSet<String>,
+}
+impl WorkScheduler for LocalDurableScheduler {
+    fn enqueue(&mut self, id: &str) {
+        self.queue.insert(id.into());
+    }
+    fn claim(
+        &mut self,
+        store: &mut WorkloadStore,
+        tenant: &str,
+        application: &str,
+        worker: &str,
+        capabilities: &BTreeSet<String>,
+        now: i64,
+        lease_ms: u64,
+    ) -> Result<Option<WorkloadClaim>, WorkloadError> {
+        let selected = store
+            .list(tenant, application)
+            .into_iter()
+            .filter(|w| {
+                w.state == WorkloadState::Ready && w.capability_snapshot.is_subset(capabilities)
+            })
+            .find(|w| {
+                w.dependencies.iter().all(|d| {
+                    store
+                        .workloads
+                        .get(&d.workload_id)
+                        .is_some_and(|dependency| dependency.state == d.required_state)
+                })
+            })
+            .map(|w| w.workload_id);
+        match selected {
+            Some(id) => store
+                .claim(&id, worker, capabilities, now, lease_ms)
+                .map(Some),
+            None => Ok(None),
+        }
+    }
+    fn release(&mut self, _: &str) {}
+}
+fn legal(from: WorkloadState, to: WorkloadState) -> bool {
+    use WorkloadState::*;
+    matches!(
+        (from, to),
+        (Created, Ready)
+            | (Ready, Claimed)
+            | (Claimed, Running)
+            | (Running, Checkpointed)
+            | (Checkpointed, Checkpointed)
+            | (Running, Succeeded)
+            | (Checkpointed, Succeeded)
+            | (Claimed, CancellationRequested)
+            | (Running, CancellationRequested)
+            | (Checkpointed, CancellationRequested)
+            | (CancellationRequested, Cancelled)
+            | (Ready, Cancelled)
+            | (Created, Cancelled)
+            | (Failed, Cancelled)
+            | (DeadLetter, Cancelled)
+            | (Running, Failed)
+            | (Checkpointed, Failed)
+            | (Claimed, Recovering)
+            | (Running, Recovering)
+            | (Checkpointed, Recovering)
+            | (Recovering, Ready)
+            | (Failed, Ready)
+            | (TimedOut, Ready)
+            | (DeadLetter, Ready)
+            | (Failed, DeadLetter)
+            | (Running, TimedOut)
+            | (Claimed, TimedOut)
+            | (Checkpointed, TimedOut)
+            | (Ready, TimedOut)
+            | (TimedOut, DeadLetter)
+    )
+}
+pub fn can_transition(from: WorkloadState, to: WorkloadState) -> bool {
+    legal(from, to)
+}
+fn event_name(state: WorkloadState) -> &'static str {
+    use WorkloadState::*;
+    match state {
+        Created => "workload.created",
+        Ready => "workload.ready",
+        Claimed => "workload.claimed",
+        Running => "workload.started",
+        Checkpointed => "workload.checkpointed",
+        Recovering => "workload.recovered",
+        CancellationRequested => "workload.cancellation_requested",
+        Succeeded => "workload.completed",
+        Failed => "workload.failed",
+        Cancelled => "workload.cancelled",
+        TimedOut => "workload.timed_out",
+        DeadLetter => "workload.dead_lettered",
+    }
+}
+fn ensure_safe(value: &Value) -> Result<(), WorkloadError> {
+    fn walk(value: &Value) -> bool {
+        match value {
+            Value::Object(map) => map.iter().any(|(k, v)| {
+                matches!(
+                    k.to_ascii_lowercase().as_str(),
+                    "password" | "secret" | "token" | "api_key" | "private_key"
+                ) || walk(v)
+            }),
+            Value::Array(values) => values.iter().any(walk),
+            Value::String(v) => sensitive_text(v),
+            _ => false,
+        }
+    }
+    if walk(value) {
+        Err(WorkloadError::new(
+            "WORKLOAD_SECRET_REJECTED",
+            "plaintext secrets are forbidden in workload state",
+        ))
+    } else {
+        Ok(())
+    }
+}
+fn sensitive_text(v: &str) -> bool {
+    v.contains("BEGIN PRIVATE KEY") || v.starts_with("sk-") || v.starts_with("fdb_live_")
+}
+fn io(e: impl std::fmt::Display) -> WorkloadError {
+    WorkloadError::new("WORKLOAD_STORAGE_FAILURE", e.to_string())
+}
+fn hash(value: &Value) -> String {
+    hex(&Sha256::digest(
+        serde_json::to_vec(value).unwrap_or_default(),
+    ))
+}
+fn hex(bytes: &[u8]) -> String {
+    bytes.iter().map(|v| format!("{v:02x}")).collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::authorization::{
+        Grant, GrantConstraints, GrantSigner, GrantStatus, ResourceScope, ResourceScopeKind,
+    };
+    fn create() -> CreateWorkload {
+        let signer = GrantSigner::new("k", vec![1; 32]).unwrap();
+        let subject = Subject::Agent("a1".into());
+        let grant = Grant {
+            contract_version: 1,
+            id: "g".into(),
+            issuer: Subject::Human("h".into()),
+            subject: subject.clone(),
+            tenant_id: "t".into(),
+            application_id: "app".into(),
+            capabilities: ["workload:execute".into()].into(),
+            resources: vec![ResourceScope {
+                kind: ResourceScopeKind::Workload,
+                uri: "flow://app/workloads/*".into(),
+            }],
+            constraints: GrantConstraints::default(),
+            issued_at: 1,
+            expires_at: 9999,
+            parent_grant: None,
+            revision: 1,
+            status: GrantStatus::Active,
+        };
+        CreateWorkload {
+            tenant_id: "t".into(),
+            application_id: "app".into(),
+            environment: "production".into(),
+            definition_id: "send".into(),
+            definition_revision: "r1".into(),
+            trigger: "manual".into(),
+            input: serde_json::json!({"id":1}),
+            requested_by: subject,
+            authorization_snapshot: signer.sign(grant).unwrap(),
+            capability_snapshot: ["email:send".into()].into(),
+            connection_scopes: BTreeSet::new(),
+            priority: 0,
+            idempotency_key: "one".into(),
+            retry_policy: Default::default(),
+            timeouts: Default::default(),
+            dependencies: vec![],
+            created_at: 10,
+        }
+    }
+    fn store(name: &str) -> (PathBuf, WorkloadStore) {
+        let path =
+            std::env::temp_dir().join(format!("workload-{name}-{}.json", std::process::id()));
+        let store = WorkloadStore::load(&path).unwrap();
+        (path, store)
+    }
+    #[test]
+    fn idempotent_create_and_legal_lifecycle() {
+        let (path, mut store) = store("lifecycle");
+        let one = store.create(create()).unwrap();
+        let two = store.create(create()).unwrap();
+        assert_eq!(one.workload_id, two.workload_id);
+        let caps = ["email:send".into()].into();
+        let claim = store
+            .claim(&one.workload_id, "worker", &caps, 11, 10_000)
+            .unwrap();
+        store
+            .start(
+                &one.workload_id,
+                &claim.claim_id,
+                "worker",
+                claim.fencing_token,
+                12,
+            )
+            .unwrap();
+        let result = store
+            .complete(
+                &one.workload_id,
+                &claim.claim_id,
+                "worker",
+                claim.fencing_token,
+                "result-1",
+                serde_json::json!({"ok":true}),
+                vec![],
+                1,
+                13,
+            )
+            .unwrap();
+        assert_eq!(result.result_id, "result-1");
+        assert!(store
+            .start(
+                &one.workload_id,
+                &claim.claim_id,
+                "worker",
+                claim.fencing_token,
+                14
+            )
+            .is_err());
+        let recovered = WorkloadStore::load(&path).unwrap();
+        assert_eq!(
+            recovered.get(&one.workload_id).unwrap().state,
+            WorkloadState::Succeeded
+        );
+        let _ = fs::remove_file(path);
+    }
+    #[test]
+    fn stale_fence_and_lease_recovery() {
+        let (path, mut store) = store("fence");
+        let w = store.create(create()).unwrap();
+        let caps = ["email:send".into()].into();
+        let old = store.claim(&w.workload_id, "a", &caps, 11, 1000).unwrap();
+        store
+            .start(&w.workload_id, &old.claim_id, "a", old.fencing_token, 11)
+            .unwrap();
+        store.recover_expired(13).unwrap();
+        let new = store.claim(&w.workload_id, "b", &caps, 13, 10_000).unwrap();
+        assert!(new.fencing_token > old.fencing_token);
+        assert_eq!(
+            store
+                .complete(
+                    &w.workload_id,
+                    &old.claim_id,
+                    "a",
+                    old.fencing_token,
+                    "stale",
+                    Value::Null,
+                    vec![],
+                    1,
+                    14
+                )
+                .unwrap_err()
+                .code,
+            "WORKLOAD_STALE_FENCE"
+        );
+        let _ = fs::remove_file(path);
+    }
+    #[test]
+    fn retry_dead_letter_checkpoint_and_secret_safety() {
+        let (path, mut store) = store("retry");
+        let mut input = create();
+        input.retry_policy.max_attempts = 1;
+        let w = store.create(input).unwrap();
+        let caps = ["email:send".into()].into();
+        let claim = store.claim(&w.workload_id, "a", &caps, 11, 10_000).unwrap();
+        store
+            .start(
+                &w.workload_id,
+                &claim.claim_id,
+                "a",
+                claim.fencing_token,
+                12,
+            )
+            .unwrap();
+        assert!(store
+            .checkpoint(
+                &w.workload_id,
+                &claim.claim_id,
+                "a",
+                claim.fencing_token,
+                serde_json::json!({"password":"bad"}),
+                13
+            )
+            .is_err());
+        assert_eq!(
+            store
+                .fail(
+                    &w.workload_id,
+                    &claim.claim_id,
+                    "a",
+                    claim.fencing_token,
+                    "worker_failure",
+                    "crashed",
+                    13
+                )
+                .unwrap(),
+            WorkloadState::DeadLetter
+        );
+        let _ = fs::remove_file(path);
+    }
+    #[test]
+    fn cancellation_race_and_duplicate_completion_are_deterministic() {
+        let (path, mut store) = store("cancel");
+        let w = store.create(create()).unwrap();
+        let caps = ["email:send".into()].into();
+        let claim = store.claim(&w.workload_id, "a", &caps, 11, 10_000).unwrap();
+        store
+            .start(
+                &w.workload_id,
+                &claim.claim_id,
+                "a",
+                claim.fencing_token,
+                12,
+            )
+            .unwrap();
+        assert_eq!(
+            store.cancel(&w.workload_id, "user", 13).unwrap(),
+            WorkloadState::CancellationRequested
+        );
+        assert_eq!(
+            store
+                .complete(
+                    &w.workload_id,
+                    &claim.claim_id,
+                    "a",
+                    claim.fencing_token,
+                    "r",
+                    Value::Null,
+                    vec![],
+                    1,
+                    14
+                )
+                .unwrap_err()
+                .code,
+            "WORKLOAD_TRANSITION_DENIED"
+        );
+        store
+            .confirm_cancel(
+                &w.workload_id,
+                &claim.claim_id,
+                "a",
+                claim.fencing_token,
+                14,
+            )
+            .unwrap();
+        assert_eq!(
+            store.get(&w.workload_id).unwrap().state,
+            WorkloadState::Cancelled
+        );
+        let _ = fs::remove_file(path);
+    }
+    #[test]
+    fn scheduler_respects_dependencies_and_tenant_application_isolation() {
+        let (path, mut store) = store("dependency");
+        let parent = store.create(create()).unwrap();
+        let mut child = create();
+        child.idempotency_key = "child".into();
+        child.dependencies = vec![WorkloadDependency {
+            workload_id: parent.workload_id.clone(),
+            required_state: WorkloadState::Succeeded,
+        }];
+        let child = store.create(child).unwrap();
+        assert_eq!(store.list("other", "app").len(), 0);
+        let caps = ["email:send".into()].into();
+        let mut scheduler = LocalDurableScheduler::default();
+        let first = scheduler
+            .claim(&mut store, "t", "app", "worker", &caps, 11, 10_000)
+            .unwrap()
+            .unwrap();
+        assert_eq!(first.workload_id, parent.workload_id);
+        assert_ne!(first.workload_id, child.workload_id);
+        let _ = fs::remove_file(path);
+    }
+}