@feltdb/core 0.13.0 → 0.15.0

package/dist/create/server-source/crates/feltdb/src/workload.rs

@@ -1,7 +1,7 @@
 //! Durable workload authority. Workers execute; this state machine decides.
 use crate::{
     authorization::{SignedGrant, Subject},
-    AtomicMutation, FeltDb, FlowError, RecordPrecondition,
+    AtomicMutation, FeltDb, FlowError, RecordPrecondition, StoredRow,
 };
 use rand::{distributions::Alphanumeric, Rng};
 use serde::{Deserialize, Serialize};
@@ -11,6 +11,11 @@ use std::collections::{BTreeMap, BTreeSet};
 use std::path::Path;
 pub const WORKLOAD_CONTRACT_VERSION: u32 = 1;
 pub const DURABLE_WORK_COLLECTION: &str = "system_durable_work";
+/// Workloads admitted at this contract version or later have their `timeouts`
+/// enforced. Earlier records were admitted when timeouts were stored and
+/// ignored; they keep that behaviour, so an upgrade never times out work whose
+/// requester was told nothing would.
+pub const DEADLINE_CONTRACT_VERSION: u32 = 2;
 
 #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
 #[serde(rename_all = "SCREAMING_SNAKE_CASE")]
@@ -153,6 +158,13 @@ pub struct WorkloadExecutionContext {
     pub secret_references: Vec<String>,
     pub expires_at: i64,
     pub fencing_token: u64,
+    /// The work's input, so the claiming worker can execute it.
+    #[serde(default)]
+    pub input: Value,
+    /// Every committed checkpoint, from this and earlier attempts, in order. A
+    /// resumed attempt reads these to learn how far lost attempts got.
+    #[serde(default)]
+    pub checkpoints: Vec<WorkloadCheckpoint>,
 }
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct WorkloadEvent {
@@ -269,12 +281,16 @@ pub struct CreateWorkload {
 pub struct WorkloadError {
     pub code: String,
     pub message: String,
+    /// Structured detail for the caller, e.g. which request fields conflict.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub details: Option<Value>,
 }
 impl WorkloadError {
     fn new(code: &str, message: impl Into<String>) -> Self {
         Self {
             code: code.into(),
             message: message.into(),
+            details: None,
         }
     }
 }
@@ -294,6 +310,15 @@ pub struct WorkloadStore {
     /// Rebuildable cache of FeltDB-owned document versions. This is never
     /// serialized and never advances independently of a canonical commit.
     document_versions: BTreeMap<String, u64>,
+    /// The store revision this projection last reconciled with. A store that
+    /// has moved past it (a commit by another writer, or a replicated
+    /// operation) is re-read before the projection answers.
+    synced_revision: u64,
+    /// Always false outside tests, which set it to act on the projection
+    /// without re-reading the store: that reproduces another writer
+    /// committing between a refresh and a commit — the window only the storage
+    /// guard closes.
+    refresh_disabled: bool,
 }
 
 #[derive(Serialize, Deserialize)]
@@ -314,6 +339,9 @@ impl WorkloadStore {
     }
 
     pub fn from_db(db: FeltDb) -> Result<Self, WorkloadError> {
+        // Read before listing: a commit that lands in between is re-read by
+        // the next refresh rather than missed.
+        let synced_revision = db.current_revision().map_err(io)?;
         let mut workloads = BTreeMap::new();
         let mut document_versions = BTreeMap::new();
         for row in db.list_collection(DURABLE_WORK_COLLECTION).map_err(io)? {
@@ -342,9 +370,121 @@ impl WorkloadStore {
             idempotency,
             result_ids,
             document_versions,
+            synced_revision,
+            refresh_disabled: false,
         })
     }
 
+    /// Reconcile the projection with the committed store.
+    ///
+    /// The store, not this projection, is authoritative. When its revision has
+    /// moved since the last reconciliation, every workload row whose document
+    /// version differs from the projection's is re-read; rows are borrowed and
+    /// only changed ones are cloned. Commits by another writer of the store and
+    /// operations replicated from a peer both become visible here, without a
+    /// restart.
+    pub fn refresh(&mut self) -> Result<(), WorkloadError> {
+        if self.refresh_disabled {
+            return Ok(());
+        }
+        let revision = self.db.current_revision().map_err(io)?;
+        if revision == self.synced_revision {
+            return Ok(());
+        }
+        let known = &self.document_versions;
+        let changed = self
+            .db
+            .query_collection(DURABLE_WORK_COLLECTION, None, |row| {
+                let version = row
+                    .value
+                    .get("__version")
+                    .and_then(Value::as_u64)
+                    .unwrap_or(1);
+                let id = row.key.rsplit('/').next().unwrap_or_default();
+                known.get(id) != Some(&version)
+            })
+            .map_err(io)?;
+        for row in changed {
+            self.adopt(row)?;
+        }
+        self.synced_revision = revision;
+        Ok(())
+    }
+
+    /// Re-read one workload from the store when its committed version differs
+    /// from the projection's: a keyed read, not a scan. An id the projection
+    /// has never seen (admitted by another writer) falls back to [`Self::refresh`].
+    fn refresh_id(&mut self, id: &str) -> Result<(), WorkloadError> {
+        match self.workloads.get(id).map(work_key) {
+            Some(key) => self.refresh_key(&key).map(|_| ()),
+            None => self.refresh(),
+        }
+    }
+
+    fn refresh_key(&mut self, key: &str) -> Result<Option<Workload>, WorkloadError> {
+        if self.refresh_disabled {
+            let id = key.rsplit('/').next().unwrap_or_default();
+            return Ok(self.workloads.get(id).cloned());
+        }
+        let Some(row) = self.db.live_row(DURABLE_WORK_COLLECTION, key) else {
+            return Ok(None);
+        };
+        let version = row
+            .value
+            .get("__version")
+            .and_then(Value::as_u64)
+            .unwrap_or(1);
+        let id = key.rsplit('/').next().unwrap_or_default();
+        if self.document_versions.get(id) == Some(&version) {
+            return Ok(self.workloads.get(id).cloned());
+        }
+        self.adopt(row).map(Some)
+    }
+
+    /// Install one committed row in the projection and its derived indexes.
+    fn adopt(&mut self, row: StoredRow) -> Result<Workload, WorkloadError> {
+        let version = row
+            .value
+            .get("__version")
+            .and_then(Value::as_u64)
+            .unwrap_or(1);
+        let workload: Workload = serde_json::from_value(row.value).map_err(io)?;
+        if let Some(result) = &workload.result {
+            self.result_ids
+                .insert(result.result_id.clone(), workload.workload_id.clone());
+        }
+        self.idempotency.insert(
+            idempotency_scope(
+                &workload.tenant_id,
+                &workload.application_id,
+                &workload.requested_by,
+                &workload.idempotency_key,
+            ),
+            workload.workload_id.clone(),
+        );
+        self.document_versions
+            .insert(workload.workload_id.clone(), version);
+        self.workloads
+            .insert(workload.workload_id.clone(), workload.clone());
+        Ok(workload)
+    }
+
+    /// A workload as committed in the store (see [`Self::refresh`]).
+    pub fn fresh(&mut self, id: &str) -> Result<&Workload, WorkloadError> {
+        self.refresh_id(id)?;
+        self.get(id)
+    }
+
+    /// An application's workloads as committed in the store.
+    pub fn fresh_list(
+        &mut self,
+        tenant: &str,
+        application: &str,
+    ) -> Result<Vec<Workload>, WorkloadError> {
+        self.refresh()?;
+        Ok(self.list(tenant, application))
+    }
+
     pub fn migrate_legacy_file(db: FeltDb, path: &Path) -> Result<Self, WorkloadError> {
         let bytes = std::fs::read(path).map_err(io)?;
         let legacy: LegacyWorkloadStore = serde_json::from_slice(&bytes).map_err(io)?;
@@ -469,6 +609,21 @@ impl WorkloadStore {
                 "idempotency key is required",
             ));
         }
+        let t = &input.timeouts;
+        if [
+            t.queue_ms,
+            t.claim_ms,
+            t.execution_ms,
+            t.heartbeat_ms,
+            t.overall_ms,
+        ]
+        .contains(&Some(0))
+        {
+            return Err(WorkloadError::new(
+                "WORKLOAD_INVALID",
+                "a timeout must be at least 1 ms; omit it for no deadline",
+            ));
+        }
         if input.authorization_snapshot.grant.subject != input.requested_by
             || input.authorization_snapshot.grant.tenant_id != input.tenant_id
             || input.authorization_snapshot.grant.application_id != input.application_id
@@ -484,19 +639,25 @@ impl WorkloadStore {
             &input.requested_by,
             &input.idempotency_key,
         );
-        if let Some(id) = self.idempotency.get(&idempotency) {
-            let existing = self.workloads[id].clone();
-            return replay(existing, &input);
-        }
         // The scoped idempotency identity is the record identity. Competing
         // admissions therefore contend on one canonical require-absent CAS
         // even when their payloads differ; no separate durable uniqueness
         // index is needed.
         let id = format!("workload_{}", &hex(&Sha256::digest(&idempotency))[..24]);
+        // Another writer, or a peer, may have admitted this identity: the
+        // store is asked by key, not the projection.
+        let committed_key = format!("{}/{}/{}", input.tenant_id, input.application_id, id);
+        if let Some(existing) = self.refresh_key(&committed_key)? {
+            return replay(existing, &input);
+        }
+        if let Some(existing_id) = self.idempotency.get(&idempotency) {
+            let existing = self.workloads[existing_id].clone();
+            return replay(existing, &input);
+        }
         let request = input.clone();
         let input_hash = hash(&input.input);
         let mut workload = Workload {
-            contract_version: 1,
+            contract_version: DEADLINE_CONTRACT_VERSION,
             workload_id: id.clone(),
             tenant_id: input.tenant_id,
             application_id: input.application_id,
@@ -585,6 +746,9 @@ impl WorkloadStore {
         id: &str,
         action: impl FnOnce(&mut Self, &mut Workload) -> Result<R, WorkloadError>,
     ) -> Result<R, WorkloadError> {
+        // Act only on the committed version: a projection that missed another
+        // writer's commit would otherwise lose its guarded commit at best.
+        self.refresh_id(id)?;
         let workload = self
             .workloads
             .get(id)
@@ -610,30 +774,18 @@ impl WorkloadStore {
     pub(crate) fn db_for_tests(&self) -> FeltDb {
         self.db.clone()
     }
+    /// See `refresh_disabled`.
+    #[cfg(test)]
+    pub(crate) fn without_refresh(mut self) -> Self {
+        self.refresh_disabled = true;
+        self
+    }
     /// Replace this projection's copy of a workload with the committed row.
     fn reload_committed(&mut self, key: &str) -> Result<Option<Workload>, WorkloadError> {
-        let committed = self
-            .db
-            .query_collection(DURABLE_WORK_COLLECTION, Some(1), |row| row.key == key)
-            .map_err(io)?;
-        let Some(row) = committed.into_iter().next() else {
-            return Ok(None);
-        };
-        let version = row
-            .value
-            .get("__version")
-            .and_then(Value::as_u64)
-            .unwrap_or(1);
-        let workload: Workload = serde_json::from_value(row.value).map_err(io)?;
-        if let Some(result) = &workload.result {
-            self.result_ids
-                .insert(result.result_id.clone(), workload.workload_id.clone());
+        match self.db.live_row(DURABLE_WORK_COLLECTION, key) {
+            Some(row) => self.adopt(row).map(Some),
+            None => Ok(None),
         }
-        self.document_versions
-            .insert(workload.workload_id.clone(), version);
-        self.workloads
-            .insert(workload.workload_id.clone(), workload.clone());
-        Ok(Some(workload))
     }
     fn transition(
         &mut self,
@@ -675,6 +827,7 @@ impl WorkloadStore {
         now: i64,
         lease_ms: u64,
     ) -> Result<WorkloadClaim, WorkloadError> {
+        self.enforce_deadline(id, now)?;
         // A Ready workload that has already used its budget (an operator
         // re-arm beyond it, or a record left by unbounded recovery before it
         // was bounded) is dead-lettered here rather than claimed again.
@@ -798,6 +951,7 @@ impl WorkloadStore {
         token: u64,
         now: i64,
     ) -> Result<(), WorkloadError> {
+        self.enforce_deadline(id, now)?;
         self.edit(id, |store, w| {
             Self::fence(w, claim, worker, token, now)?;
             let attempt = w.attempts.last_mut().unwrap();
@@ -815,6 +969,7 @@ impl WorkloadStore {
         now: i64,
         extend_ms: u64,
     ) -> Result<WorkloadClaim, WorkloadError> {
+        self.enforce_deadline(id, now)?;
         self.edit(id, |_, w| {
             Self::fence(w, claim, worker, token, now)?;
             let current = w.current_claim.as_mut().unwrap();
@@ -824,7 +979,7 @@ impl WorkloadStore {
         })
     }
     pub fn execution_context(
-        &self,
+        &mut self,
         id: &str,
         claim: &str,
         worker: &str,
@@ -832,7 +987,8 @@ impl WorkloadStore {
         now: i64,
         secret_references: Vec<String>,
     ) -> Result<WorkloadExecutionContext, WorkloadError> {
-        let workload = self.get(id)?;
+        self.enforce_deadline(id, now)?;
+        let workload = self.fresh(id)?;
         Self::fence(workload, claim, worker, token, now)?;
         if secret_references
             .iter()
@@ -853,6 +1009,8 @@ impl WorkloadStore {
             secret_references,
             expires_at: current.expires_at,
             fencing_token: token,
+            input: workload.input.clone(),
+            checkpoints: workload.checkpoints.clone(),
         })
     }
     pub fn checkpoint(
@@ -865,6 +1023,7 @@ impl WorkloadStore {
         now: i64,
     ) -> Result<WorkloadCheckpoint, WorkloadError> {
         ensure_safe(&state)?;
+        self.enforce_deadline(id, now)?;
         self.edit(id, |store, w| {
             Self::fence(w, claim, worker, token, now)?;
             if !matches!(
@@ -911,13 +1070,28 @@ impl WorkloadStore {
         ensure_safe(&output)?;
         if let Some(workload_id) = self.result_ids.get(result_id) {
             if workload_id == id {
-                return Ok(self.get(id)?.result.clone().unwrap());
+                // A replay is the reporting attempt retrying the same report
+                // (e.g. after a lost response). Anything else naming this
+                // result id did not produce it and must not be told it landed.
+                let result = self.get(id)?.result.clone().unwrap();
+                let same_report = result.produced_by == worker
+                    && result.provenance.get("claim_id").map(String::as_str) == Some(claim)
+                    && result.provenance.get("fencing_token") == Some(&token.to_string())
+                    && result.output_hash == hash(&output);
+                if same_report {
+                    return Ok(result);
+                }
+                return Err(WorkloadError::new(
+                    "WORKLOAD_RESULT_CONFLICT",
+                    "the result id was already recorded by a different report",
+                ));
             }
             return Err(WorkloadError::new(
                 "WORKLOAD_RESULT_DUPLICATE",
                 "result id belongs to another workload",
             ));
         }
+        self.enforce_deadline(id, now)?;
         let result = self.edit(id, |store, w| {
             Self::fence(w, claim, worker, token, now)?;
             if !matches!(
@@ -968,6 +1142,31 @@ impl WorkloadStore {
         kind: &str,
         error: &str,
         now: i64,
+    ) -> Result<WorkloadState, WorkloadError> {
+        self.fail_with_effect(id, claim, worker, token, kind, error, None, now)
+    }
+    /// A worker's failure report for its current attempt.
+    ///
+    /// Accepted from `CLAIMED` (the worker could not start) as well as from a
+    /// started attempt, fenced like every worker call. `effect` says what the
+    /// worker knows about external effects: `none` — it failed before causing
+    /// any (implied for an attempt that never started); `unknown` — an effect
+    /// may have happened. Without it, the policy decides by `kind`, as before.
+    /// An `unknown` effect is decided like a lost worker: retried only when the
+    /// policy retries `worker_failure`, whatever `kind` says. Repeating the same
+    /// report replays its outcome; any other report from an attempt that
+    /// already reported gets the fence's answer (`CLAIM_INVALID`/`STALE_FENCE`).
+    #[allow(clippy::too_many_arguments)]
+    pub fn fail_with_effect(
+        &mut self,
+        id: &str,
+        claim: &str,
+        worker: &str,
+        token: u64,
+        kind: &str,
+        error: &str,
+        effect: Option<&str>,
+        now: i64,
     ) -> Result<WorkloadState, WorkloadError> {
         if sensitive_text(error) {
             return Err(WorkloadError::new(
@@ -975,23 +1174,65 @@ impl WorkloadStore {
                 "errors may not contain secret material",
             ));
         }
+        if effect.is_some_and(|e| !matches!(e, "none" | "unknown")) {
+            return Err(WorkloadError::new(
+                "WORKLOAD_INVALID",
+                "effect must be \"none\" or \"unknown\"",
+            ));
+        }
+        // The same report repeated (a lost response) replays its outcome. Any
+        // other report from an attempt that already reported is answered by
+        // the fence, as before: its claim is no longer current.
+        let current = self.fresh(id)?.clone();
+        if let Some(attempt) = reported_attempt(&current, claim, worker, token) {
+            let same = attempt.status == AttemptStatus::Failed
+                && attempt.error.as_deref() == Some(error)
+                && failure_event(&current, &attempt.attempt_id).is_some_and(|event| {
+                    let recorded_effect = event.evidence.get("effect").map(String::as_str);
+                    event.evidence.get("failure_kind").map(String::as_str) == Some(kind)
+                        && (effect.is_none() || effect == recorded_effect)
+                });
+            if same {
+                return Ok(current.state);
+            }
+        }
+        self.enforce_deadline(id, now)?;
         self.edit(id, |store, w| {
             Self::fence(w, claim, worker, token, now)?;
+            let started = w.state != WorkloadState::Claimed;
+            let effect = effect.or((!started).then_some("none"));
             let attempt = w.attempts.last_mut().unwrap();
             attempt.status = AttemptStatus::Failed;
             attempt.ended_at = Some(now);
             attempt.error = Some(error.into());
+            let attempt_id = attempt.attempt_id.clone();
             w.last_failure = Some(format!("{kind}: {error}"));
             w.current_claim = None;
-            store.transition(
-                w,
-                WorkloadState::Failed,
-                worker,
-                now,
-                [("failure_kind".into(), kind.into())].into(),
-            )?;
-            let retryable =
-                w.retry_policy.retry_on.contains(kind) && w.attempt_count < attempt_budget(w);
+            let mut evidence: BTreeMap<String, String> = [
+                ("failure_kind".into(), kind.into()),
+                ("attempt_id".into(), attempt_id),
+            ]
+            .into();
+            if let Some(effect) = effect {
+                evidence.insert("effect".into(), effect.into());
+            }
+            store.transition(w, WorkloadState::Failed, worker, now, evidence)?;
+            let unknown = effect == Some("unknown");
+            let allowed =
+                w.retry_policy
+                    .retry_on
+                    .contains(if unknown { "worker_failure" } else { kind });
+            let decision = if !allowed && unknown {
+                "effect_unknown_not_retryable"
+            } else if !allowed {
+                "failure_kind_not_retryable"
+            } else if w.attempt_count >= attempt_budget(w) {
+                "attempt_budget_exhausted"
+            } else {
+                "retry"
+            };
+            let retryable = decision == "retry";
+            let evidence: BTreeMap<String, String> = [("decision".into(), decision.into())].into();
             if retryable {
                 w.retry_count += 1;
                 let multiplier = if w.retry_policy.backoff == BackoffPolicy::Exponential {
@@ -1005,15 +1246,9 @@ impl WorkloadStore {
                     .saturating_mul(multiplier)
                     .min(w.retry_policy.max_delay_ms);
                 w.next_attempt_at = Some(now + (delay / 1000) as i64);
-                store.transition(w, WorkloadState::Ready, "rust.retry", now, BTreeMap::new())?;
+                store.transition(w, WorkloadState::Ready, "rust.retry", now, evidence)?;
             } else {
-                store.transition(
-                    w,
-                    WorkloadState::DeadLetter,
-                    "rust.retry",
-                    now,
-                    BTreeMap::new(),
-                )?;
+                store.transition(w, WorkloadState::DeadLetter, "rust.retry", now, evidence)?;
             }
             Ok(w.state)
         })
@@ -1096,9 +1331,13 @@ impl WorkloadStore {
                     "workload cannot time out in its current state",
                 ));
             }
-            if let Some(attempt) = w.attempts.last_mut() {
-                attempt.status = AttemptStatus::TimedOut;
-                attempt.ended_at = Some(now);
+            // A Ready workload has no attempt in progress: the last attempt's
+            // recorded outcome is not this timeout's to overwrite.
+            if w.state != WorkloadState::Ready {
+                if let Some(attempt) = w.attempts.last_mut() {
+                    attempt.status = AttemptStatus::TimedOut;
+                    attempt.ended_at = Some(now);
+                }
             }
             w.current_claim = None;
             store.transition(w, WorkloadState::TimedOut, actor, now, BTreeMap::new())?;
@@ -1172,7 +1411,93 @@ impl WorkloadStore {
             store.transition(w, WorkloadState::Ready, actor, now, evidence)
         })
     }
+    /// Time out every workload past one of its deadlines (see
+    /// [`deadline_breach`]), as the recovery pass does each second. Deadlines
+    /// are computed from the workload's committed timestamps, so the decision
+    /// is the same after a restart and on any writer of the store.
+    pub fn enforce_deadlines(&mut self, now: i64) -> Result<Vec<String>, WorkloadError> {
+        self.refresh()?;
+        let due = self
+            .workloads
+            .values()
+            .filter_map(|w| deadline_breach(w, now).map(|breach| (w.workload_id.clone(), breach)))
+            .collect::<Vec<_>>();
+        let mut expired = Vec::with_capacity(due.len());
+        for (id, (deadline, at)) in due {
+            match self.expire(&id, deadline, at, now) {
+                Ok(()) => expired.push(id),
+                Err(error) if error.code == "WORKLOAD_CONFLICT" => {}
+                Err(error) => return Err(error),
+            }
+        }
+        Ok(expired)
+    }
+    /// Refuse a worker call on a workload past a deadline, timing it out
+    /// first, in the same way the recovery pass would have.
+    fn enforce_deadline(&mut self, id: &str, now: i64) -> Result<(), WorkloadError> {
+        let Some((deadline, at)) = deadline_breach(self.fresh(id)?, now) else {
+            return Ok(());
+        };
+        self.expire(id, deadline, at, now)?;
+        let mut error = WorkloadError::new(
+            "WORKLOAD_TIMED_OUT",
+            format!("the workload's {deadline} deadline passed"),
+        );
+        error.details = Some(serde_json::json!({ "deadline": deadline, "deadline_at": at }));
+        Err(error)
+    }
+    fn expire(&mut self, id: &str, deadline: &str, at: i64, now: i64) -> Result<(), WorkloadError> {
+        self.edit(id, |store, w| {
+            // Re-check on the committed version: a heartbeat or completion that
+            // committed first decides, and this changes nothing.
+            if deadline_breach(w, now).map(|(kind, _)| kind) != Some(deadline) {
+                return Err(WorkloadError::new(
+                    "WORKLOAD_CONFLICT",
+                    "the workload changed before its deadline was enforced",
+                ));
+            }
+            let lost = w.current_claim.take();
+            if w.state != WorkloadState::Ready {
+                if let Some(attempt) = w.attempts.last_mut() {
+                    attempt.status = AttemptStatus::TimedOut;
+                    attempt.ended_at = Some(now);
+                    attempt.error = Some(format!("timeout: {deadline} deadline passed"));
+                }
+            }
+            let mut evidence: BTreeMap<String, String> = [
+                ("deadline".into(), deadline.into()),
+                ("deadline_at".into(), at.to_string()),
+            ]
+            .into();
+            if let Some(claim) = &lost {
+                evidence.insert("claim_id".into(), claim.claim_id.clone());
+                evidence.insert("worker_id".into(), claim.worker_id.clone());
+            }
+            store.transition(w, WorkloadState::TimedOut, "rust.deadline", now, evidence)?;
+            // The overall and queue deadlines bound the work, not an attempt:
+            // another attempt cannot meet them.
+            let decision = if matches!(deadline, "overall" | "queue") {
+                "deadline_not_retryable"
+            } else if !w.retry_policy.retry_on.contains("timeout") {
+                "timeout_not_retryable"
+            } else if w.attempt_count >= attempt_budget(w) {
+                "attempt_budget_exhausted"
+            } else {
+                "retry"
+            };
+            let evidence: BTreeMap<String, String> = [("decision".into(), decision.into())].into();
+            if decision == "retry" {
+                w.retry_count += 1;
+                w.next_attempt_at = Some(now + (w.retry_policy.initial_delay_ms / 1000) as i64);
+                store.transition(w, WorkloadState::Ready, "rust.deadline", now, evidence)?;
+            } else {
+                store.transition(w, WorkloadState::DeadLetter, "rust.deadline", now, evidence)?;
+            }
+            Ok(())
+        })
+    }
     pub fn recover_expired(&mut self, now: i64) -> Result<Vec<String>, WorkloadError> {
+        self.refresh()?;
         let ids = self
             .workloads
             .values()
@@ -1193,6 +1518,25 @@ impl WorkloadStore {
         let mut recovered = Vec::with_capacity(ids.len());
         for id in ids {
             let outcome = self.edit(&id, |store, w| {
+                // Decide on the committed version: another writer may have
+                // recovered (or the worker completed) it since it was listed.
+                let still_expired = w
+                    .current_claim
+                    .as_ref()
+                    .is_some_and(|c| c.expires_at <= now)
+                    && matches!(
+                        w.state,
+                        WorkloadState::Claimed
+                            | WorkloadState::Running
+                            | WorkloadState::Checkpointed
+                            | WorkloadState::CancellationRequested
+                    );
+                if !still_expired {
+                    return Err(WorkloadError::new(
+                        "WORKLOAD_CONFLICT",
+                        "the workload changed before it was recovered",
+                    ));
+                }
                 let lost = w.current_claim.take();
                 if let Some(attempt) = w.attempts.last_mut() {
                     attempt.status = AttemptStatus::WorkerLost;
@@ -1340,6 +1684,7 @@ fn legal(from: WorkloadState, to: WorkloadState) -> bool {
             | (Created, Cancelled)
             | (Failed, Cancelled)
             | (DeadLetter, Cancelled)
+            | (Claimed, Failed)
             | (Running, Failed)
             | (Checkpointed, Failed)
             | (Claimed, Recovering)
@@ -1437,21 +1782,200 @@ fn idempotency_index(
 }
 /// Answer a repeated key: the original workload for the same request, a
 /// refusal for a different one. Never the original for a different request.
+/// The execution-affecting part of an admission, normalized, field by field.
+///
+/// A replay under the same key is answered with the original workload only when
+/// every one of these is equal. Optional fields are compared after their
+/// defaults apply (serde fills an omitted `retry_policy`, `timeouts`,
+/// `priority`, `connection_scopes` or `dependencies`); sets compare as sets and
+/// dependencies regardless of order; the input compares by its canonical hash,
+/// so member order is not part of it. Provenance that does not change what runs
+/// — `trigger`, `session_id`, `created_at`, and the authorization snapshot,
+/// which is verified before any replay — is not compared.
+fn execution_fields(
+    definition_id: &str,
+    definition_revision: &str,
+    environment: &str,
+    input_hash: &str,
+    capability_snapshot: &BTreeSet<String>,
+    connection_scopes: &BTreeSet<String>,
+    priority: i32,
+    retry_policy: &WorkloadRetryPolicy,
+    timeouts: &WorkloadTimeouts,
+    dependencies: &[WorkloadDependency],
+    not_before: Option<i64>,
+) -> Vec<(&'static str, Value)> {
+    let mut dependencies = dependencies
+        .iter()
+        .map(|d| serde_json::json!([d.workload_id, d.required_state]))
+        .collect::<Vec<_>>();
+    dependencies.sort_by_key(|d| d.to_string());
+    vec![
+        ("definition_id", definition_id.into()),
+        ("definition_revision", definition_revision.into()),
+        ("environment", environment.into()),
+        ("input", input_hash.into()),
+        (
+            "capability_snapshot",
+            serde_json::json!(capability_snapshot),
+        ),
+        ("connection_scopes", serde_json::json!(connection_scopes)),
+        ("priority", priority.into()),
+        ("retry_policy", serde_json::json!(retry_policy)),
+        ("timeouts", serde_json::json!(timeouts)),
+        ("dependencies", Value::Array(dependencies)),
+        ("not_before", serde_json::json!(not_before)),
+    ]
+}
+/// The `not_before` a workload was admitted with, from its creation event
+/// (`next_attempt_at` moves with retries; the admission evidence does not).
+fn admitted_not_before(w: &Workload) -> Option<i64> {
+    w.execution_history
+        .iter()
+        .find(|event| event.to == WorkloadState::Ready)
+        .and_then(|event| event.evidence.get("not_before"))
+        .and_then(|value| value.parse().ok())
+}
 fn replay(existing: Workload, input: &CreateWorkload) -> Result<(Workload, bool), WorkloadError> {
-    let same = existing.requested_by == input.requested_by
-        && existing.definition_id == input.definition_id
-        && existing.definition_revision == input.definition_revision
-        && existing.environment == input.environment
-        && existing.input_hash == hash(&input.input)
-        && existing.capability_snapshot == input.capability_snapshot;
-    if same {
-        Ok((existing, false))
-    } else {
-        Err(WorkloadError::new(
-            "IDEMPOTENCY_KEY_REUSED",
-            "idempotency key was already used for a different request",
-        ))
+    let recorded = execution_fields(
+        &existing.definition_id,
+        &existing.definition_revision,
+        &existing.environment,
+        &existing.input_hash,
+        &existing.capability_snapshot,
+        &existing.connection_scopes,
+        existing.priority,
+        &existing.retry_policy,
+        &existing.timeouts,
+        &existing.dependencies,
+        admitted_not_before(&existing),
+    );
+    let requested = execution_fields(
+        &input.definition_id,
+        &input.definition_revision,
+        &input.environment,
+        &hash(&input.input),
+        &input.capability_snapshot,
+        &input.connection_scopes,
+        input.priority,
+        &input.retry_policy,
+        &input.timeouts,
+        &input.dependencies,
+        input.not_before,
+    );
+    let mut conflicting = recorded
+        .iter()
+        .zip(&requested)
+        .filter(|((_, left), (_, right))| left != right)
+        .map(|((name, _), _)| *name)
+        .collect::<Vec<_>>();
+    if existing.requested_by != input.requested_by {
+        conflicting.insert(0, "requested_by");
+    }
+    if conflicting.is_empty() {
+        return Ok((existing, false));
+    }
+    let mut error = WorkloadError::new(
+        "IDEMPOTENCY_KEY_REUSED",
+        "idempotency key was already used for a different request",
+    );
+    error.details = Some(serde_json::json!({ "fields": conflicting }));
+    Err(error)
+}
+/// Seconds a millisecond timeout lasts, rounded up so a deadline is never early.
+fn deadline_seconds(ms: u64) -> i64 {
+    ms.div_ceil(1000) as i64
+}
+/// The first deadline a workload has passed at `now`, and when it fell.
+///
+/// | Deadline | Measured from | While |
+/// | --- | --- | --- |
+/// | `overall` | admission (`created_at`) | not terminal |
+/// | `queue` | becoming `READY` (or its `next_attempt_at`, if later) | `READY` |
+/// | `claim` | the claim | `CLAIMED` |
+/// | `execution` | the attempt's start | `RUNNING`, `CHECKPOINTED` |
+/// | `heartbeat` | the last heartbeat (or the claim) | claimed |
+///
+/// Only workloads admitted at [`DEADLINE_CONTRACT_VERSION`] or later.
+pub fn deadline_breach(w: &Workload, now: i64) -> Option<(&'static str, i64)> {
+    use WorkloadState::*;
+    if w.contract_version < DEADLINE_CONTRACT_VERSION {
+        return None;
     }
+    let active = matches!(w.state, Ready | Claimed | Running | Checkpointed);
+    if !active {
+        return None;
+    }
+    let t = &w.timeouts;
+    let claim = w.current_claim.as_ref();
+    let candidates = [
+        (
+            "overall",
+            t.overall_ms.map(|ms| w.created_at + deadline_seconds(ms)),
+        ),
+        (
+            "queue",
+            t.queue_ms.filter(|_| w.state == Ready).map(|ms| {
+                let ready_since = w
+                    .execution_history
+                    .iter()
+                    .rev()
+                    .find(|event| event.to == Ready)
+                    .map_or(w.created_at, |event| event.at);
+                ready_since.max(w.next_attempt_at.unwrap_or(i64::MIN)) + deadline_seconds(ms)
+            }),
+        ),
+        (
+            "claim",
+            t.claim_ms
+                .filter(|_| w.state == Claimed)
+                .and_then(|ms| claim.map(|c| c.claimed_at + deadline_seconds(ms))),
+        ),
+        (
+            "execution",
+            t.execution_ms
+                .filter(|_| matches!(w.state, Running | Checkpointed))
+                .and_then(|ms| {
+                    w.attempts
+                        .last()
+                        .and_then(|a| a.started_at)
+                        .map(|started| started + deadline_seconds(ms))
+                }),
+        ),
+        (
+            "heartbeat",
+            t.heartbeat_ms
+                .and_then(|ms| claim.map(|c| c.last_heartbeat + deadline_seconds(ms))),
+        ),
+    ];
+    candidates
+        .into_iter()
+        .find_map(|(kind, at)| at.filter(|at| now >= *at).map(|at| (kind, at)))
+}
+/// The attempt this fence named, when that attempt has already reported its
+/// outcome (failed or succeeded). Lost and timed-out attempts are not reports.
+fn reported_attempt<'w>(
+    w: &'w Workload,
+    claim: &str,
+    worker: &str,
+    token: u64,
+) -> Option<&'w WorkloadAttempt> {
+    w.attempts.iter().find(|attempt| {
+        attempt.claim_id == claim
+            && attempt.worker_id == worker
+            && attempt.fencing_token == token
+            && matches!(
+                attempt.status,
+                AttemptStatus::Failed | AttemptStatus::Succeeded
+            )
+    })
+}
+/// The `FAILED` event a failure report of `attempt_id` recorded.
+fn failure_event<'w>(w: &'w Workload, attempt_id: &str) -> Option<&'w WorkloadEvent> {
+    w.execution_history.iter().rev().find(|event| {
+        event.to == WorkloadState::Failed
+            && event.evidence.get("attempt_id").map(String::as_str) == Some(attempt_id)
+    })
 }
 /// Operator re-arms recorded in the workload's own history.
 fn operator_retries(w: &Workload) -> u32 {
@@ -1700,7 +2224,11 @@ mod tests {
             let id = store.create(create()).unwrap().workload_id;
             let claim = store.claim(&id, "worker-a", &capabilities(), 100, 10_000).unwrap();
             // A writer that loaded the workload before either recovery.
-            let mut stale = WorkloadStore::from_db(db.clone()).unwrap();
+            // Commits land in this projection's window between refresh and
+            // commit: only the storage guard can refuse them.
+            let mut stale = WorkloadStore::from_db(db.clone())
+                .unwrap()
+                .without_refresh();
 
             let (mut first, mut second, recovered) = race_recovery(&db, claim.expires_at);
             // Exactly one writer committed the recovery; the other reported none.
@@ -2001,7 +2529,11 @@ mod tests {
         let caps = ["email:send".into()].into();
 
         let mut winner = WorkloadStore::from_db(db.clone()).unwrap();
-        let mut loser = WorkloadStore::from_db(db.clone()).unwrap();
+        // The loser's claim races the winner's inside the window between its
+        // refresh and its commit, which only the storage guard closes.
+        let mut loser = WorkloadStore::from_db(db.clone())
+            .unwrap()
+            .without_refresh();
         winner.claim(&id, "worker-a", &caps, 11, 1_000).unwrap();
         let conflict = loser.claim(&id, "worker-b", &caps, 11, 1_000).unwrap_err();
         assert_eq!(conflict.code, "WORKLOAD_CONFLICT");
@@ -2023,7 +2555,9 @@ mod tests {
         let mut recovery = WorkloadStore::from_db(restarted_db.clone()).unwrap();
         recovery.recover_expired(13).unwrap();
         let mut restarted_winner = WorkloadStore::from_db(restarted_db.clone()).unwrap();
-        let mut restarted_loser = WorkloadStore::from_db(restarted_db.clone()).unwrap();
+        let mut restarted_loser = WorkloadStore::from_db(restarted_db.clone())
+            .unwrap()
+            .without_refresh();
         restarted_winner
             .claim(&id, "worker-c", &caps, 13, 10_000)
             .unwrap();
@@ -2370,52 +2904,182 @@ mod tests {
         let _ = fs::remove_file(path);
     }
     #[test]
-    fn cancellation_race_and_duplicate_completion_are_deterministic() {
-        let (path, mut store) = store("cancel");
+    fn a_repeated_completion_replays_only_for_the_claim_and_output_that_produced_it() {
+        let (path, mut store) = store("complete-replay");
         let w = store.create(create()).unwrap();
-        let caps = ["email:send".into()].into();
-        let claim = store.claim(&w.workload_id, "a", &caps, 11, 10_000).unwrap();
+        let id = w.workload_id.clone();
+        let caps = capabilities();
+        // Worker a loses its lease; worker b completes the work.
+        let lost = store.claim(&id, "a", &caps, 11, 1000).unwrap();
         store
-            .start(
-                &w.workload_id,
-                &claim.claim_id,
-                "a",
-                claim.fencing_token,
-                12,
+            .start(&id, &lost.claim_id, "a", lost.fencing_token, 11)
+            .unwrap();
+        store.recover_expired(13).unwrap();
+        let won = store.claim(&id, "b", &caps, 13, 10_000).unwrap();
+        store
+            .start(&id, &won.claim_id, "b", won.fencing_token, 13)
+            .unwrap();
+        let output = serde_json::json!({"sent": 1});
+        let first = store
+            .complete(
+                &id,
+                &won.claim_id,
+                "b",
+                won.fencing_token,
+                "r1",
+                output.clone(),
+                vec![],
+                1,
+                14,
+            )
+            .unwrap();
+        // The reporting attempt retrying after a lost response gets the same result.
+        let replay = store
+            .complete(
+                &id,
+                &won.claim_id,
+                "b",
+                won.fencing_token,
+                "r1",
+                output.clone(),
+                vec![],
+                1,
+                15,
             )
             .unwrap();
+        assert_eq!(replay.output_hash, first.output_hash);
+        assert_eq!(replay.attempt_id, first.attempt_id);
+        // The same result id with a different output is a conflict, not a replay.
         assert_eq!(
-            store.cancel(&w.workload_id, "user", 13).unwrap(),
-            WorkloadState::CancellationRequested
+            store
+                .complete(
+                    &id,
+                    &won.claim_id,
+                    "b",
+                    won.fencing_token,
+                    "r1",
+                    serde_json::json!({"sent": 2}),
+                    vec![],
+                    1,
+                    15
+                )
+                .unwrap_err()
+                .code,
+            "WORKLOAD_RESULT_CONFLICT"
         );
+        // A stale worker naming the winner's result id learns nothing landed for it.
         assert_eq!(
             store
                 .complete(
-                    &w.workload_id,
-                    &claim.claim_id,
+                    &id,
+                    &lost.claim_id,
                     "a",
-                    claim.fencing_token,
-                    "r",
-                    Value::Null,
+                    lost.fencing_token,
+                    "r1",
+                    output.clone(),
                     vec![],
                     1,
-                    14
+                    15
                 )
                 .unwrap_err()
                 .code,
-            "WORKLOAD_TRANSITION_DENIED"
+            "WORKLOAD_RESULT_CONFLICT"
         );
+        let committed = store.get(&id).unwrap();
+        assert_eq!(committed.state, WorkloadState::Succeeded);
+        assert_eq!(committed.result.as_ref().unwrap().output, output);
+        assert_eq!(committed.result.as_ref().unwrap().produced_by, "b");
+        let _ = fs::remove_file(path);
+    }
+    #[test]
+    fn a_resumed_attempt_reads_the_input_and_the_lost_attempts_checkpoints_through_its_fence() {
+        let (path, mut store) = store("resume-context");
+        let w = store.create(create()).unwrap();
+        let id = w.workload_id.clone();
+        let caps = capabilities();
+        let lost = store.claim(&id, "a", &caps, 11, 1000).unwrap();
         store
-            .confirm_cancel(
-                &w.workload_id,
-                &claim.claim_id,
+            .start(&id, &lost.claim_id, "a", lost.fencing_token, 11)
+            .unwrap();
+        store
+            .checkpoint(
+                &id,
+                &lost.claim_id,
                 "a",
-                claim.fencing_token,
-                14,
+                lost.fencing_token,
+                serde_json::json!({"step": "reserve", "status": "done"}),
+                11,
             )
             .unwrap();
+        store.recover_expired(13).unwrap();
+        let next = store.claim(&id, "b", &caps, 13, 10_000).unwrap();
+        let context = store
+            .execution_context(&id, &next.claim_id, "b", next.fencing_token, 13, vec![])
+            .unwrap();
+        assert_eq!(context.input, serde_json::json!({"id": 1}));
+        assert_eq!(context.checkpoints.len(), 1);
+        assert_eq!(context.checkpoints[0].state["step"], "reserve");
+        assert_ne!(
+            context.checkpoints[0].attempt_id, context.attempt_id,
+            "written by the lost attempt"
+        );
+        // The lost worker's fence reads nothing.
         assert_eq!(
-            store.get(&w.workload_id).unwrap().state,
+            store
+                .execution_context(&id, &lost.claim_id, "a", lost.fencing_token, 13, vec![])
+                .unwrap_err()
+                .code,
+            "WORKLOAD_STALE_FENCE"
+        );
+        let _ = fs::remove_file(path);
+    }
+    #[test]
+    fn cancellation_race_and_duplicate_completion_are_deterministic() {
+        let (path, mut store) = store("cancel");
+        let w = store.create(create()).unwrap();
+        let caps = ["email:send".into()].into();
+        let claim = store.claim(&w.workload_id, "a", &caps, 11, 10_000).unwrap();
+        store
+            .start(
+                &w.workload_id,
+                &claim.claim_id,
+                "a",
+                claim.fencing_token,
+                12,
+            )
+            .unwrap();
+        assert_eq!(
+            store.cancel(&w.workload_id, "user", 13).unwrap(),
+            WorkloadState::CancellationRequested
+        );
+        assert_eq!(
+            store
+                .complete(
+                    &w.workload_id,
+                    &claim.claim_id,
+                    "a",
+                    claim.fencing_token,
+                    "r",
+                    Value::Null,
+                    vec![],
+                    1,
+                    14
+                )
+                .unwrap_err()
+                .code,
+            "WORKLOAD_TRANSITION_DENIED"
+        );
+        store
+            .confirm_cancel(
+                &w.workload_id,
+                &claim.claim_id,
+                "a",
+                claim.fencing_token,
+                14,
+            )
+            .unwrap();
+        assert_eq!(
+            store.get(&w.workload_id).unwrap().state,
             WorkloadState::Cancelled
         );
         let _ = fs::remove_file(path);
@@ -2883,4 +3547,676 @@ mod tests {
         );
         let _ = fs::remove_file(path);
     }
+
+    // ---- PR 2: authoritative reads, canonical admission, failure reports, deadlines ----
+
+    fn version_of(store: &WorkloadStore, id: &str) -> u64 {
+        let db = store.db_for_tests();
+        let w = store.get(id).unwrap();
+        db.live_row(DURABLE_WORK_COLLECTION, &work_key(w))
+            .and_then(|row| row.value.get("__version").and_then(Value::as_u64))
+            .unwrap()
+    }
+
+    #[test]
+    fn another_writers_commits_are_visible_and_acted_on_without_restart() {
+        let (path, mut a) = store("two-writers");
+        let mut b = WorkloadStore::from_db(a.db_for_tests()).unwrap();
+        let id = a.create(create()).unwrap().workload_id;
+        // B never saw the admission, yet reads it from the store.
+        assert!(b.get(&id).is_err(), "the projection alone is stale");
+        assert_eq!(b.fresh(&id).unwrap().state, WorkloadState::Ready);
+        assert_eq!(b.fresh_list("t", "app").unwrap().len(), 1);
+        // A claims; B's claim decides on the committed claim, not its own view.
+        let claim = a
+            .claim(&id, "worker-a", &capabilities(), 11, 10_000)
+            .unwrap();
+        assert_eq!(
+            b.claim(&id, "worker-b", &capabilities(), 11, 10_000)
+                .unwrap_err()
+                .code,
+            "WORKLOAD_NOT_READY"
+        );
+        // A transition committed through A is what B fences against.
+        b.start(&id, &claim.claim_id, "worker-a", claim.fencing_token, 12)
+            .unwrap();
+        assert_eq!(a.fresh(&id).unwrap().state, WorkloadState::Running);
+        // B replays an admission A committed, by key, from the store.
+        let mut c = WorkloadStore::from_db(a.db_for_tests()).unwrap();
+        c.workloads.clear();
+        c.idempotency.clear();
+        c.document_versions.clear();
+        let (replayed, created) = c.admit(create()).unwrap();
+        assert!(!created);
+        assert_eq!(replayed.workload_id, id);
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn a_replay_matches_only_an_equivalent_request_and_names_every_conflicting_field() {
+        let (path, mut store) = store("canonical-admission");
+        let mut original = create();
+        original.input = serde_json::json!({"a": 1, "b": 2});
+        original.dependencies = vec![
+            WorkloadDependency {
+                workload_id: "w1".into(),
+                required_state: WorkloadState::Succeeded,
+            },
+            WorkloadDependency {
+                workload_id: "w2".into(),
+                required_state: WorkloadState::Succeeded,
+            },
+        ];
+        let first = store.create(original.clone()).unwrap();
+
+        // Equivalent after normalization: member order, dependency order, the
+        // defaults written out, and provenance that does not change what runs.
+        let mut equivalent = original.clone();
+        equivalent.input = serde_json::json!({"b": 2, "a": 1});
+        equivalent.dependencies.reverse();
+        equivalent.retry_policy = WorkloadRetryPolicy::default();
+        equivalent.timeouts = WorkloadTimeouts::default();
+        equivalent.priority = 0;
+        equivalent.trigger = "retried-by-client".into();
+        equivalent.session_id = Some("another-session".into());
+        equivalent.created_at = 99;
+        let (replayed, created) = store.admit(equivalent).unwrap();
+        assert!(!created);
+        assert_eq!(replayed.workload_id, first.workload_id);
+
+        let mutations: Vec<(&str, Box<dyn Fn(&mut CreateWorkload)>)> = vec![
+            (
+                "definition_id",
+                Box::new(|r| r.definition_id = "other".into()),
+            ),
+            (
+                "definition_revision",
+                Box::new(|r| r.definition_revision = "r2".into()),
+            ),
+            (
+                "environment",
+                Box::new(|r| r.environment = "staging".into()),
+            ),
+            (
+                "input",
+                Box::new(|r| r.input = serde_json::json!({"a": 1, "b": 3})),
+            ),
+            (
+                "capability_snapshot",
+                Box::new(|r| {
+                    r.capability_snapshot.insert("sms:send".into());
+                }),
+            ),
+            (
+                "connection_scopes",
+                Box::new(|r| {
+                    r.connection_scopes.insert("crm".into());
+                }),
+            ),
+            ("priority", Box::new(|r| r.priority = 5)),
+            (
+                "retry_policy",
+                Box::new(|r| r.retry_policy.max_attempts = 9),
+            ),
+            (
+                "retry_policy",
+                Box::new(|r| {
+                    r.retry_policy.retry_on.remove("worker_failure");
+                }),
+            ),
+            (
+                "timeouts",
+                Box::new(|r| r.timeouts.execution_ms = Some(60_000)),
+            ),
+            (
+                "dependencies",
+                Box::new(|r| r.dependencies.pop().map(|_| ()).unwrap()),
+            ),
+            ("not_before", Box::new(|r| r.not_before = Some(5_000))),
+        ];
+        for (field, mutate) in mutations {
+            let mut request = original.clone();
+            mutate(&mut request);
+            let error = store.admit(request).unwrap_err();
+            assert_eq!(error.code, "IDEMPOTENCY_KEY_REUSED", "{field}");
+            assert_eq!(
+                error.details.unwrap()["fields"],
+                serde_json::json!([field]),
+                "{field}"
+            );
+        }
+        // Several at once are all named.
+        let mut request = original.clone();
+        request.priority = 1;
+        request.environment = "staging".into();
+        let fields = store.admit(request).unwrap_err().details.unwrap()["fields"].clone();
+        assert_eq!(fields, serde_json::json!(["environment", "priority"]));
+
+        // After a restart: the same answers from the committed record.
+        drop(store);
+        let mut reopened = WorkloadStore::load(&path).unwrap();
+        assert!(!reopened.admit(original.clone()).unwrap().1);
+        let mut changed = original.clone();
+        changed.retry_policy.initial_delay_ms = 5;
+        assert_eq!(
+            reopened.admit(changed).unwrap_err().code,
+            "IDEMPOTENCY_KEY_REUSED"
+        );
+        assert_eq!(reopened.workloads.len(), 1, "nothing was created");
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn concurrent_conflicting_admissions_admit_one_and_refuse_the_other() {
+        for round in 0..10 {
+            let path = std::env::temp_dir().join(format!(
+                "workload-admission-race-{round}-{}.log",
+                std::process::id()
+            ));
+            let _ = fs::remove_file(&path);
+            let db = FeltDb::open(&path).unwrap();
+            let stores = [
+                WorkloadStore::from_db(db.clone()).unwrap(),
+                WorkloadStore::from_db(db.clone()).unwrap(),
+            ];
+            let barrier = std::sync::Barrier::new(2);
+            let outcomes = std::thread::scope(|scope| {
+                stores
+                    .into_iter()
+                    .enumerate()
+                    .map(|(i, mut store)| {
+                        let barrier = &barrier;
+                        scope.spawn(move || {
+                            let mut request = create();
+                            request.priority = i as i32;
+                            barrier.wait();
+                            store
+                                .admit(request)
+                                .map(|(w, created)| (w.priority, created))
+                        })
+                    })
+                    .collect::<Vec<_>>()
+                    .into_iter()
+                    .map(|handle| handle.join().unwrap())
+                    .collect::<Vec<_>>()
+            });
+            let admitted = outcomes
+                .iter()
+                .filter(|o| matches!(o, Ok((_, true))))
+                .count();
+            let refused = outcomes
+                .iter()
+                .filter(|o| {
+                    o.as_ref()
+                        .is_err_and(|e| e.code == "IDEMPOTENCY_KEY_REUSED")
+                })
+                .count();
+            assert_eq!((admitted, refused), (1, 1), "round {round}: {outcomes:?}");
+            let _ = fs::remove_file(path);
+        }
+    }
+
+    fn claimed(
+        name: &str,
+        policy: impl FnOnce(&mut CreateWorkload),
+    ) -> (PathBuf, WorkloadStore, String, WorkloadClaim) {
+        let (path, mut store) = store(name);
+        let mut request = create();
+        policy(&mut request);
+        let id = store.create(request).unwrap().workload_id;
+        let claim = store.claim(&id, "a", &capabilities(), 11, 10_000).unwrap();
+        (path, store, id, claim)
+    }
+
+    #[test]
+    fn a_worker_reports_failure_from_claimed_fenced_and_by_policy() {
+        // Could not start: a known failure before any effect, retried by kind.
+        let (path, mut store, id, claim) = claimed("fail-claimed", |_| {});
+        let state = store
+            .fail_with_effect(
+                &id,
+                &claim.claim_id,
+                "a",
+                claim.fencing_token,
+                "transient_connection",
+                "no route",
+                None,
+                12,
+            )
+            .unwrap();
+        assert_eq!(state, WorkloadState::Ready);
+        let w = store.get(&id).unwrap();
+        let failed = w
+            .execution_history
+            .iter()
+            .find(|e| e.to == WorkloadState::Failed)
+            .unwrap();
+        assert_eq!(
+            failed.evidence["effect"], "none",
+            "never started, so no effect"
+        );
+        assert_eq!(w.attempts[0].status, AttemptStatus::Failed);
+        // The identical report again (lost response) replays and commits nothing.
+        let version = version_of(&store, &id);
+        assert_eq!(
+            store
+                .fail_with_effect(
+                    &id,
+                    &claim.claim_id,
+                    "a",
+                    claim.fencing_token,
+                    "transient_connection",
+                    "no route",
+                    None,
+                    13
+                )
+                .unwrap(),
+            WorkloadState::Ready
+        );
+        assert_eq!(version_of(&store, &id), version);
+        // A different report from the same attempt is answered by its fence.
+        assert_eq!(
+            store
+                .fail_with_effect(
+                    &id,
+                    &claim.claim_id,
+                    "a",
+                    claim.fencing_token,
+                    "timeout",
+                    "other",
+                    None,
+                    13
+                )
+                .unwrap_err()
+                .code,
+            "WORKLOAD_CLAIM_INVALID"
+        );
+        // So is a completion: the failure stands.
+        assert_eq!(
+            store
+                .complete(
+                    &id,
+                    &claim.claim_id,
+                    "a",
+                    claim.fencing_token,
+                    "r",
+                    Value::Null,
+                    vec![],
+                    1,
+                    13
+                )
+                .unwrap_err()
+                .code,
+            "WORKLOAD_CLAIM_INVALID"
+        );
+        // Another worker's claim fences the old attempt out.
+        let next = store.claim(&id, "b", &capabilities(), 14, 10_000).unwrap();
+        assert_eq!(
+            store
+                .fail_with_effect(
+                    &id,
+                    &claim.claim_id,
+                    "a",
+                    claim.fencing_token,
+                    "timeout",
+                    "late",
+                    None,
+                    15
+                )
+                .unwrap_err()
+                .code,
+            "WORKLOAD_STALE_FENCE"
+        );
+        assert_eq!(
+            store
+                .get(&id)
+                .unwrap()
+                .current_claim
+                .as_ref()
+                .unwrap()
+                .claim_id,
+            next.claim_id
+        );
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn failure_policy_decides_retry_and_an_unknown_effect_is_decided_like_a_lost_worker() {
+        // A kind the policy does not retry dead-letters.
+        let (path, mut store, id, claim) = claimed("fail-kind", |_| {});
+        store
+            .start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
+            .unwrap();
+        assert_eq!(
+            store
+                .fail_with_effect(
+                    &id,
+                    &claim.claim_id,
+                    "a",
+                    claim.fencing_token,
+                    "validation",
+                    "bad input",
+                    Some("none"),
+                    13
+                )
+                .unwrap(),
+            WorkloadState::DeadLetter
+        );
+        assert_eq!(
+            store
+                .get(&id)
+                .unwrap()
+                .execution_history
+                .last()
+                .unwrap()
+                .evidence["decision"],
+            "failure_kind_not_retryable"
+        );
+        let _ = fs::remove_file(path);
+
+        // A retryable kind with an unknown effect, under a policy that does not
+        // re-run lost work: not retried automatically.
+        let (path, mut store, id, claim) = claimed("fail-unknown", |r| {
+            r.retry_policy.retry_on = ["timeout".into(), "transient_connection".into()].into();
+        });
+        store
+            .start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
+            .unwrap();
+        assert_eq!(
+            store
+                .fail_with_effect(
+                    &id,
+                    &claim.claim_id,
+                    "a",
+                    claim.fencing_token,
+                    "timeout",
+                    "charge sent, no answer",
+                    Some("unknown"),
+                    13
+                )
+                .unwrap(),
+            WorkloadState::DeadLetter
+        );
+        let w = store.get(&id).unwrap();
+        assert_eq!(
+            w.execution_history.last().unwrap().evidence["decision"],
+            "effect_unknown_not_retryable"
+        );
+        // An operator can still re-arm it, with a reason.
+        store
+            .retry_with_reason(&id, "operator", 14, Some("verified: not charged"))
+            .unwrap();
+        assert_eq!(store.get(&id).unwrap().state, WorkloadState::Ready);
+        let _ = fs::remove_file(path);
+
+        // The same report under a policy that re-runs lost work is retried.
+        let (path, mut store, id, claim) = claimed("fail-unknown-retry", |_| {});
+        store
+            .start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
+            .unwrap();
+        assert_eq!(
+            store
+                .fail_with_effect(
+                    &id,
+                    &claim.claim_id,
+                    "a",
+                    claim.fencing_token,
+                    "validation",
+                    "x",
+                    Some("unknown"),
+                    13
+                )
+                .unwrap(),
+            WorkloadState::Ready
+        );
+        // An invalid effect is refused before anything changes.
+        assert_eq!(
+            store
+                .fail_with_effect(&id, "c", "a", 9, "x", "y", Some("maybe"), 14)
+                .unwrap_err()
+                .code,
+            "WORKLOAD_INVALID"
+        );
+        let _ = fs::remove_file(path);
+
+        // The attempt budget bounds retries.
+        let (path, mut store, id, mut claim) =
+            claimed("fail-budget", |r| r.retry_policy.max_attempts = 2);
+        assert_eq!(
+            store
+                .fail(
+                    &id,
+                    &claim.claim_id,
+                    "a",
+                    claim.fencing_token,
+                    "timeout",
+                    "x",
+                    12
+                )
+                .unwrap(),
+            WorkloadState::Ready
+        );
+        let next = store.get(&id).unwrap().next_attempt_at.unwrap();
+        claim = store
+            .claim(&id, "a", &capabilities(), next, 10_000)
+            .unwrap();
+        assert_eq!(
+            store
+                .fail(
+                    &id,
+                    &claim.claim_id,
+                    "a",
+                    claim.fencing_token,
+                    "timeout",
+                    "x",
+                    next
+                )
+                .unwrap(),
+            WorkloadState::DeadLetter
+        );
+        assert_eq!(
+            store
+                .get(&id)
+                .unwrap()
+                .execution_history
+                .last()
+                .unwrap()
+                .evidence["decision"],
+            "attempt_budget_exhausted"
+        );
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn failure_racing_completion_commits_exactly_one_outcome() {
+        for completion_first in [true, false] {
+            let (path, mut store, id, claim) = claimed(&format!("race-{completion_first}"), |_| {});
+            store
+                .start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
+                .unwrap();
+            let complete = |s: &mut WorkloadStore| {
+                s.complete(
+                    &id,
+                    &claim.claim_id,
+                    "a",
+                    claim.fencing_token,
+                    "r",
+                    Value::Null,
+                    vec![],
+                    1,
+                    13,
+                )
+                .map(|_| ())
+            };
+            let fail = |s: &mut WorkloadStore| {
+                s.fail(
+                    &id,
+                    &claim.claim_id,
+                    "a",
+                    claim.fencing_token,
+                    "timeout",
+                    "x",
+                    13,
+                )
+                .map(|_| ())
+            };
+            let (first, second) = if completion_first {
+                (complete(&mut store), fail(&mut store))
+            } else {
+                (fail(&mut store), complete(&mut store))
+            };
+            assert!(first.is_ok());
+            assert_eq!(second.unwrap_err().code, "WORKLOAD_CLAIM_INVALID");
+            let w = store.get(&id).unwrap();
+            assert_eq!(
+                w.state,
+                if completion_first {
+                    WorkloadState::Succeeded
+                } else {
+                    WorkloadState::Ready
+                }
+            );
+            assert_eq!(w.attempts.len(), 1);
+            let _ = fs::remove_file(path);
+        }
+    }
+
+    #[test]
+    fn deadlines_are_enforced_from_committed_timestamps_and_race_safely() {
+        // execution: a heartbeat inside the deadline keeps the attempt; after it,
+        // every worker call is refused and the attempt is timed out, once.
+        let (path, mut store, id, claim) =
+            claimed("deadline-exec", |r| r.timeouts.execution_ms = Some(5_000));
+        store
+            .start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
+            .unwrap();
+        store
+            .heartbeat(&id, &claim.claim_id, "a", claim.fencing_token, 16, 10_000)
+            .unwrap();
+        assert!(store.enforce_deadlines(16).unwrap().is_empty());
+        let refused = store
+            .complete(
+                &id,
+                &claim.claim_id,
+                "a",
+                claim.fencing_token,
+                "late",
+                Value::Null,
+                vec![],
+                1,
+                17,
+            )
+            .unwrap_err();
+        assert_eq!(refused.code, "WORKLOAD_TIMED_OUT");
+        assert_eq!(refused.details.unwrap()["deadline"], "execution");
+        let w = store.get(&id).unwrap().clone();
+        assert_eq!(
+            w.state,
+            WorkloadState::Ready,
+            "timeout is retryable by default"
+        );
+        assert_eq!(w.attempts[0].status, AttemptStatus::TimedOut);
+        assert!(w.result.is_none(), "a late completion never lands");
+        assert!(
+            store.enforce_deadlines(30).unwrap().is_empty(),
+            "timed out once"
+        );
+        let _ = fs::remove_file(path);
+
+        // heartbeat: the gap since the last heartbeat; a heartbeat that commits
+        // first moves the deadline, the recovery pass after it changes nothing.
+        let (path, mut store, id, claim) = claimed("deadline-heartbeat", |r| {
+            r.timeouts.heartbeat_ms = Some(3_000)
+        });
+        store
+            .start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
+            .unwrap();
+        store
+            .heartbeat(&id, &claim.claim_id, "a", claim.fencing_token, 13, 10_000)
+            .unwrap();
+        assert!(store.enforce_deadlines(15).unwrap().is_empty());
+        assert_eq!(store.enforce_deadlines(16).unwrap(), vec![id.clone()]);
+        assert_eq!(
+            store
+                .heartbeat(&id, &claim.claim_id, "a", claim.fencing_token, 16, 10_000)
+                .unwrap_err()
+                .code,
+            "WORKLOAD_CLAIM_INVALID",
+            "the timed-out claim is gone"
+        );
+        let _ = fs::remove_file(path);
+
+        // claim: claimed but never started; not retried when the policy says so.
+        let (path, mut store, id, claim) = claimed("deadline-claim", |r| {
+            r.timeouts.claim_ms = Some(2_000);
+            r.retry_policy.retry_on = ["worker_failure".into()].into();
+        });
+        let refused = store
+            .start(&id, &claim.claim_id, "a", claim.fencing_token, 13)
+            .unwrap_err();
+        assert_eq!(refused.code, "WORKLOAD_TIMED_OUT");
+        let w = store.get(&id).unwrap();
+        assert_eq!(w.state, WorkloadState::DeadLetter);
+        assert_eq!(
+            w.execution_history.last().unwrap().evidence["decision"],
+            "timeout_not_retryable"
+        );
+        let _ = fs::remove_file(path);
+
+        // queue and overall bound the work: dead-lettered, never retried, and a
+        // queued workload's previous attempt is not rewritten.
+        let (path, mut store) = super::tests::store("deadline-queue");
+        let mut request = create();
+        request.timeouts.queue_ms = Some(4_000);
+        let id = store.create(request).unwrap().workload_id;
+        assert_eq!(
+            store
+                .claim(&id, "a", &capabilities(), 14, 10_000)
+                .unwrap_err()
+                .code,
+            "WORKLOAD_TIMED_OUT"
+        );
+        let w = store.get(&id).unwrap();
+        assert_eq!(w.state, WorkloadState::DeadLetter);
+        assert_eq!(
+            w.execution_history.last().unwrap().evidence["decision"],
+            "deadline_not_retryable"
+        );
+        assert!(w.attempts.is_empty());
+        let _ = fs::remove_file(path);
+
+        let (path, mut store, id, claim) =
+            claimed("deadline-overall", |r| r.timeouts.overall_ms = Some(10_000));
+        store
+            .start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
+            .unwrap();
+        // Restart: the deadline comes from the committed record.
+        drop(store);
+        let mut store = WorkloadStore::load(&path).unwrap();
+        assert!(store.enforce_deadlines(19).unwrap().is_empty());
+        assert_eq!(store.enforce_deadlines(20).unwrap(), vec![id.clone()]);
+        assert_eq!(store.get(&id).unwrap().state, WorkloadState::DeadLetter);
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn deadlines_are_versioned_and_zero_is_refused() {
+        let (path, mut store) = store("deadline-version");
+        let mut request = create();
+        request.timeouts.queue_ms = Some(0);
+        assert_eq!(store.create(request).unwrap_err().code, "WORKLOAD_INVALID");
+        // A record admitted before deadlines were enforced keeps its contract.
+        let mut request = create();
+        request.timeouts.queue_ms = Some(1_000);
+        let id = store.create(request).unwrap().workload_id;
+        assert_eq!(
+            store.get(&id).unwrap().contract_version,
+            DEADLINE_CONTRACT_VERSION
+        );
+        let mut legacy = store.get(&id).unwrap().clone();
+        legacy.contract_version = 1;
+        assert_eq!(deadline_breach(&legacy, 1_000_000), None);
+        assert!(deadline_breach(store.get(&id).unwrap(), 1_000_000).is_some());
+        let _ = fs::remove_file(path);
+    }
 }