@feltdb/core 0.12.2 → 0.15.0

package/dist/create/server-source/crates/feltdb/src/workload.rs

@@ -1,7 +1,7 @@
 //! Durable workload authority. Workers execute; this state machine decides.
 use crate::{
     authorization::{SignedGrant, Subject},
-    AtomicMutation, FeltDb, FlowError, RecordPrecondition,
+    AtomicMutation, FeltDb, FlowError, RecordPrecondition, StoredRow,
 };
 use rand::{distributions::Alphanumeric, Rng};
 use serde::{Deserialize, Serialize};
@@ -11,6 +11,11 @@ use std::collections::{BTreeMap, BTreeSet};
 use std::path::Path;
 pub const WORKLOAD_CONTRACT_VERSION: u32 = 1;
 pub const DURABLE_WORK_COLLECTION: &str = "system_durable_work";
+/// Workloads admitted at this contract version or later have their `timeouts`
+/// enforced. Earlier records were admitted when timeouts were stored and
+/// ignored; they keep that behaviour, so an upgrade never times out work whose
+/// requester was told nothing would.
+pub const DEADLINE_CONTRACT_VERSION: u32 = 2;
 
 #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
 #[serde(rename_all = "SCREAMING_SNAKE_CASE")]
@@ -153,6 +158,13 @@ pub struct WorkloadExecutionContext {
     pub secret_references: Vec<String>,
     pub expires_at: i64,
     pub fencing_token: u64,
+    /// The work's input, so the claiming worker can execute it.
+    #[serde(default)]
+    pub input: Value,
+    /// Every committed checkpoint, from this and earlier attempts, in order. A
+    /// resumed attempt reads these to learn how far lost attempts got.
+    #[serde(default)]
+    pub checkpoints: Vec<WorkloadCheckpoint>,
 }
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct WorkloadEvent {
@@ -269,12 +281,16 @@ pub struct CreateWorkload {
 pub struct WorkloadError {
     pub code: String,
     pub message: String,
+    /// Structured detail for the caller, e.g. which request fields conflict.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub details: Option<Value>,
 }
 impl WorkloadError {
     fn new(code: &str, message: impl Into<String>) -> Self {
         Self {
             code: code.into(),
             message: message.into(),
+            details: None,
         }
     }
 }
@@ -294,6 +310,15 @@ pub struct WorkloadStore {
     /// Rebuildable cache of FeltDB-owned document versions. This is never
     /// serialized and never advances independently of a canonical commit.
     document_versions: BTreeMap<String, u64>,
+    /// The store revision this projection last reconciled with. A store that
+    /// has moved past it (a commit by another writer, or a replicated
+    /// operation) is re-read before the projection answers.
+    synced_revision: u64,
+    /// Always false outside tests, which set it to act on the projection
+    /// without re-reading the store: that reproduces another writer
+    /// committing between a refresh and a commit — the window only the storage
+    /// guard closes.
+    refresh_disabled: bool,
 }
 
 #[derive(Serialize, Deserialize)]
@@ -314,6 +339,9 @@ impl WorkloadStore {
     }
 
     pub fn from_db(db: FeltDb) -> Result<Self, WorkloadError> {
+        // Read before listing: a commit that lands in between is re-read by
+        // the next refresh rather than missed.
+        let synced_revision = db.current_revision().map_err(io)?;
         let mut workloads = BTreeMap::new();
         let mut document_versions = BTreeMap::new();
         for row in db.list_collection(DURABLE_WORK_COLLECTION).map_err(io)? {
@@ -329,16 +357,9 @@ impl WorkloadStore {
             document_versions.insert(workload.workload_id.clone(), version);
             workloads.insert(workload.workload_id.clone(), workload);
         }
-        let mut idempotency = BTreeMap::new();
+        let idempotency = idempotency_index(&workloads, idempotency_scope);
         let mut result_ids = BTreeMap::new();
         for workload in workloads.values() {
-            idempotency.insert(
-                format!(
-                    "{}:{}:{}",
-                    workload.tenant_id, workload.application_id, workload.idempotency_key
-                ),
-                workload.workload_id.clone(),
-            );
             if let Some(result) = &workload.result {
                 result_ids.insert(result.result_id.clone(), workload.workload_id.clone());
             }
@@ -349,9 +370,121 @@ impl WorkloadStore {
             idempotency,
             result_ids,
             document_versions,
+            synced_revision,
+            refresh_disabled: false,
         })
     }
 
+    /// Reconcile the projection with the committed store.
+    ///
+    /// The store, not this projection, is authoritative. When its revision has
+    /// moved since the last reconciliation, every workload row whose document
+    /// version differs from the projection's is re-read; rows are borrowed and
+    /// only changed ones are cloned. Commits by another writer of the store and
+    /// operations replicated from a peer both become visible here, without a
+    /// restart.
+    pub fn refresh(&mut self) -> Result<(), WorkloadError> {
+        if self.refresh_disabled {
+            return Ok(());
+        }
+        let revision = self.db.current_revision().map_err(io)?;
+        if revision == self.synced_revision {
+            return Ok(());
+        }
+        let known = &self.document_versions;
+        let changed = self
+            .db
+            .query_collection(DURABLE_WORK_COLLECTION, None, |row| {
+                let version = row
+                    .value
+                    .get("__version")
+                    .and_then(Value::as_u64)
+                    .unwrap_or(1);
+                let id = row.key.rsplit('/').next().unwrap_or_default();
+                known.get(id) != Some(&version)
+            })
+            .map_err(io)?;
+        for row in changed {
+            self.adopt(row)?;
+        }
+        self.synced_revision = revision;
+        Ok(())
+    }
+
+    /// Re-read one workload from the store when its committed version differs
+    /// from the projection's: a keyed read, not a scan. An id the projection
+    /// has never seen (admitted by another writer) falls back to [`Self::refresh`].
+    fn refresh_id(&mut self, id: &str) -> Result<(), WorkloadError> {
+        match self.workloads.get(id).map(work_key) {
+            Some(key) => self.refresh_key(&key).map(|_| ()),
+            None => self.refresh(),
+        }
+    }
+
+    fn refresh_key(&mut self, key: &str) -> Result<Option<Workload>, WorkloadError> {
+        if self.refresh_disabled {
+            let id = key.rsplit('/').next().unwrap_or_default();
+            return Ok(self.workloads.get(id).cloned());
+        }
+        let Some(row) = self.db.live_row(DURABLE_WORK_COLLECTION, key) else {
+            return Ok(None);
+        };
+        let version = row
+            .value
+            .get("__version")
+            .and_then(Value::as_u64)
+            .unwrap_or(1);
+        let id = key.rsplit('/').next().unwrap_or_default();
+        if self.document_versions.get(id) == Some(&version) {
+            return Ok(self.workloads.get(id).cloned());
+        }
+        self.adopt(row).map(Some)
+    }
+
+    /// Install one committed row in the projection and its derived indexes.
+    fn adopt(&mut self, row: StoredRow) -> Result<Workload, WorkloadError> {
+        let version = row
+            .value
+            .get("__version")
+            .and_then(Value::as_u64)
+            .unwrap_or(1);
+        let workload: Workload = serde_json::from_value(row.value).map_err(io)?;
+        if let Some(result) = &workload.result {
+            self.result_ids
+                .insert(result.result_id.clone(), workload.workload_id.clone());
+        }
+        self.idempotency.insert(
+            idempotency_scope(
+                &workload.tenant_id,
+                &workload.application_id,
+                &workload.requested_by,
+                &workload.idempotency_key,
+            ),
+            workload.workload_id.clone(),
+        );
+        self.document_versions
+            .insert(workload.workload_id.clone(), version);
+        self.workloads
+            .insert(workload.workload_id.clone(), workload.clone());
+        Ok(workload)
+    }
+
+    /// A workload as committed in the store (see [`Self::refresh`]).
+    pub fn fresh(&mut self, id: &str) -> Result<&Workload, WorkloadError> {
+        self.refresh_id(id)?;
+        self.get(id)
+    }
+
+    /// An application's workloads as committed in the store.
+    pub fn fresh_list(
+        &mut self,
+        tenant: &str,
+        application: &str,
+    ) -> Result<Vec<Workload>, WorkloadError> {
+        self.refresh()?;
+        Ok(self.list(tenant, application))
+    }
+
     pub fn migrate_legacy_file(db: FeltDb, path: &Path) -> Result<Self, WorkloadError> {
         let bytes = std::fs::read(path).map_err(io)?;
         let legacy: LegacyWorkloadStore = serde_json::from_slice(&bytes).map_err(io)?;
@@ -382,7 +515,7 @@ impl WorkloadStore {
         }
         // The maps are verification evidence, not migrated authorities.
         let rebuilt = Self::from_db(store.db.clone())?;
-        if legacy.idempotency != rebuilt.idempotency
+        if legacy.idempotency != idempotency_index(&rebuilt.workloads, legacy_idempotency_scope)
             || legacy.result_ids != rebuilt.result_ids
             || legacy.event_sequence != derived_event_sequence(&rebuilt.workloads)
         {
@@ -454,6 +587,21 @@ impl WorkloadStore {
         Ok(())
     }
     pub fn create(&mut self, input: CreateWorkload) -> Result<Workload, WorkloadError> {
+        self.admit(input).map(|(workload, _)| workload)
+    }
+
+    /// Create a workload, or return the one this requester already created
+    /// under the same key.
+    ///
+    /// The key is scoped to the tenant, the application **and the requester**,
+    /// so one requester's key can never return, reveal or block another's
+    /// workload. A repeated key is answered with the original workload only when
+    /// the request is the same — definition, revision, environment, input and
+    /// capability snapshot; otherwise it is refused with
+    /// `IDEMPOTENCY_KEY_REUSED` and nothing is created. The requester's
+    /// authorization snapshot is checked before any replay. The boolean is
+    /// `true` when this call created the workload.
+    pub fn admit(&mut self, input: CreateWorkload) -> Result<(Workload, bool), WorkloadError> {
         ensure_safe(&input.input)?;
         if input.idempotency_key.trim().is_empty() {
             return Err(WorkloadError::new(
@@ -461,12 +609,20 @@ impl WorkloadStore {
                 "idempotency key is required",
             ));
         }
-        let idempotency = format!(
-            "{}:{}:{}",
-            input.tenant_id, input.application_id, input.idempotency_key
-        );
-        if let Some(id) = self.idempotency.get(&idempotency) {
-            return Ok(self.workloads[id].clone());
+        let t = &input.timeouts;
+        if [
+            t.queue_ms,
+            t.claim_ms,
+            t.execution_ms,
+            t.heartbeat_ms,
+            t.overall_ms,
+        ]
+        .contains(&Some(0))
+        {
+            return Err(WorkloadError::new(
+                "WORKLOAD_INVALID",
+                "a timeout must be at least 1 ms; omit it for no deadline",
+            ));
         }
         if input.authorization_snapshot.grant.subject != input.requested_by
             || input.authorization_snapshot.grant.tenant_id != input.tenant_id
@@ -477,14 +633,31 @@ impl WorkloadStore {
                 "authorization snapshot does not bind the requester",
             ));
         }
+        let idempotency = idempotency_scope(
+            &input.tenant_id,
+            &input.application_id,
+            &input.requested_by,
+            &input.idempotency_key,
+        );
         // The scoped idempotency identity is the record identity. Competing
         // admissions therefore contend on one canonical require-absent CAS
         // even when their payloads differ; no separate durable uniqueness
         // index is needed.
         let id = format!("workload_{}", &hex(&Sha256::digest(&idempotency))[..24]);
+        // Another writer, or a peer, may have admitted this identity: the
+        // store is asked by key, not the projection.
+        let committed_key = format!("{}/{}/{}", input.tenant_id, input.application_id, id);
+        if let Some(existing) = self.refresh_key(&committed_key)? {
+            return replay(existing, &input);
+        }
+        if let Some(existing_id) = self.idempotency.get(&idempotency) {
+            let existing = self.workloads[existing_id].clone();
+            return replay(existing, &input);
+        }
+        let request = input.clone();
         let input_hash = hash(&input.input);
         let mut workload = Workload {
-            contract_version: 1,
+            contract_version: DEADLINE_CONTRACT_VERSION,
             workload_id: id.clone(),
             tenant_id: input.tenant_id,
             application_id: input.application_id,
@@ -535,14 +708,23 @@ impl WorkloadStore {
             input.created_at,
             evidence,
         )?;
+        let key = work_key(&workload);
         self.idempotency.insert(idempotency.clone(), id.clone());
         self.workloads.insert(id.clone(), workload.clone());
         if let Err(error) = self.persist_workload(&id, "workload.created") {
             self.workloads.remove(&id);
             self.idempotency.remove(&idempotency);
+            // Another writer of the same store (a second process, or a replica)
+            // admitted this identity first. Converge on what it committed.
+            if error.code == "WORKLOAD_CONFLICT" {
+                if let Some(existing) = self.reload_committed(&key)? {
+                    self.idempotency.insert(idempotency, id);
+                    return replay(existing, &request);
+                }
+            }
             return Err(error);
         }
-        Ok(workload)
+        Ok((workload, true))
     }
     pub fn get(&self, id: &str) -> Result<&Workload, WorkloadError> {
         self.workloads
@@ -564,6 +746,9 @@ impl WorkloadStore {
         id: &str,
         action: impl FnOnce(&mut Self, &mut Workload) -> Result<R, WorkloadError>,
     ) -> Result<R, WorkloadError> {
+        // Act only on the committed version: a projection that missed another
+        // writer's commit would otherwise lose its guarded commit at best.
+        self.refresh_id(id)?;
         let workload = self
             .workloads
             .get(id)
@@ -573,11 +758,35 @@ impl WorkloadStore {
         let result = action(self, &mut edited)?;
         self.workloads.insert(id.into(), edited);
         if let Err(error) = self.persist_workload(id, "workload.mutated") {
-            self.workloads.insert(id.into(), workload);
+            self.workloads.insert(id.into(), workload.clone());
+            // Another writer of the same store committed this workload first.
+            // Nothing of this edit was committed; adopt what was, so this
+            // projection does not keep acting on a version that no longer
+            // exists (every later guarded commit from it would conflict).
+            if error.code == "WORKLOAD_CONFLICT" {
+                self.reload_committed(&work_key(&workload))?;
+            }
             return Err(error);
         }
         Ok(result)
     }
+    #[cfg(test)]
+    pub(crate) fn db_for_tests(&self) -> FeltDb {
+        self.db.clone()
+    }
+    /// See `refresh_disabled`.
+    #[cfg(test)]
+    pub(crate) fn without_refresh(mut self) -> Self {
+        self.refresh_disabled = true;
+        self
+    }
+    /// Replace this projection's copy of a workload with the committed row.
+    fn reload_committed(&mut self, key: &str) -> Result<Option<Workload>, WorkloadError> {
+        match self.db.live_row(DURABLE_WORK_COLLECTION, key) {
+            Some(row) => self.adopt(row).map(Some),
+            None => Ok(None),
+        }
+    }
     fn transition(
         &mut self,
         w: &mut Workload,
@@ -618,6 +827,28 @@ impl WorkloadStore {
         now: i64,
         lease_ms: u64,
     ) -> Result<WorkloadClaim, WorkloadError> {
+        self.enforce_deadline(id, now)?;
+        // A Ready workload that has already used its budget (an operator
+        // re-arm beyond it, or a record left by unbounded recovery before it
+        // was bounded) is dead-lettered here rather than claimed again.
+        let exhausted = self
+            .get(id)
+            .is_ok_and(|w| w.state == WorkloadState::Ready && w.attempt_count >= attempt_budget(w));
+        if exhausted {
+            self.edit(id, |store, w| {
+                let evidence = [
+                    ("reason".into(), "attempt_budget_exhausted".into()),
+                    ("attempts_used".into(), w.attempt_count.to_string()),
+                    ("attempt_budget".into(), attempt_budget(w).to_string()),
+                ]
+                .into();
+                store.transition(w, WorkloadState::DeadLetter, "rust.claim", now, evidence)
+            })?;
+            return Err(WorkloadError::new(
+                "WORKLOAD_ATTEMPTS_EXHAUSTED",
+                "workload has used its attempt budget",
+            ));
+        }
         self.edit(id, |store, w| {
             if w.state != WorkloadState::Ready {
                 return Err(WorkloadError::new(
@@ -720,6 +951,7 @@ impl WorkloadStore {
         token: u64,
         now: i64,
     ) -> Result<(), WorkloadError> {
+        self.enforce_deadline(id, now)?;
         self.edit(id, |store, w| {
             Self::fence(w, claim, worker, token, now)?;
             let attempt = w.attempts.last_mut().unwrap();
@@ -737,6 +969,7 @@ impl WorkloadStore {
         now: i64,
         extend_ms: u64,
     ) -> Result<WorkloadClaim, WorkloadError> {
+        self.enforce_deadline(id, now)?;
         self.edit(id, |_, w| {
             Self::fence(w, claim, worker, token, now)?;
             let current = w.current_claim.as_mut().unwrap();
@@ -746,7 +979,7 @@ impl WorkloadStore {
         })
     }
     pub fn execution_context(
-        &self,
+        &mut self,
         id: &str,
         claim: &str,
         worker: &str,
@@ -754,7 +987,8 @@ impl WorkloadStore {
         now: i64,
         secret_references: Vec<String>,
     ) -> Result<WorkloadExecutionContext, WorkloadError> {
-        let workload = self.get(id)?;
+        self.enforce_deadline(id, now)?;
+        let workload = self.fresh(id)?;
         Self::fence(workload, claim, worker, token, now)?;
         if secret_references
             .iter()
@@ -775,6 +1009,8 @@ impl WorkloadStore {
             secret_references,
             expires_at: current.expires_at,
             fencing_token: token,
+            input: workload.input.clone(),
+            checkpoints: workload.checkpoints.clone(),
         })
     }
     pub fn checkpoint(
@@ -787,6 +1023,7 @@ impl WorkloadStore {
         now: i64,
     ) -> Result<WorkloadCheckpoint, WorkloadError> {
         ensure_safe(&state)?;
+        self.enforce_deadline(id, now)?;
         self.edit(id, |store, w| {
             Self::fence(w, claim, worker, token, now)?;
             if !matches!(
@@ -833,13 +1070,28 @@ impl WorkloadStore {
         ensure_safe(&output)?;
         if let Some(workload_id) = self.result_ids.get(result_id) {
             if workload_id == id {
-                return Ok(self.get(id)?.result.clone().unwrap());
+                // A replay is the reporting attempt retrying the same report
+                // (e.g. after a lost response). Anything else naming this
+                // result id did not produce it and must not be told it landed.
+                let result = self.get(id)?.result.clone().unwrap();
+                let same_report = result.produced_by == worker
+                    && result.provenance.get("claim_id").map(String::as_str) == Some(claim)
+                    && result.provenance.get("fencing_token") == Some(&token.to_string())
+                    && result.output_hash == hash(&output);
+                if same_report {
+                    return Ok(result);
+                }
+                return Err(WorkloadError::new(
+                    "WORKLOAD_RESULT_CONFLICT",
+                    "the result id was already recorded by a different report",
+                ));
             }
             return Err(WorkloadError::new(
                 "WORKLOAD_RESULT_DUPLICATE",
                 "result id belongs to another workload",
             ));
         }
+        self.enforce_deadline(id, now)?;
         let result = self.edit(id, |store, w| {
             Self::fence(w, claim, worker, token, now)?;
             if !matches!(
@@ -890,6 +1142,31 @@ impl WorkloadStore {
         kind: &str,
         error: &str,
         now: i64,
+    ) -> Result<WorkloadState, WorkloadError> {
+        self.fail_with_effect(id, claim, worker, token, kind, error, None, now)
+    }
+    /// A worker's failure report for its current attempt.
+    ///
+    /// Accepted from `CLAIMED` (the worker could not start) as well as from a
+    /// started attempt, fenced like every worker call. `effect` says what the
+    /// worker knows about external effects: `none` — it failed before causing
+    /// any (implied for an attempt that never started); `unknown` — an effect
+    /// may have happened. Without it, the policy decides by `kind`, as before.
+    /// An `unknown` effect is decided like a lost worker: retried only when the
+    /// policy retries `worker_failure`, whatever `kind` says. Repeating the same
+    /// report replays its outcome; any other report from an attempt that
+    /// already reported gets the fence's answer (`CLAIM_INVALID`/`STALE_FENCE`).
+    #[allow(clippy::too_many_arguments)]
+    pub fn fail_with_effect(
+        &mut self,
+        id: &str,
+        claim: &str,
+        worker: &str,
+        token: u64,
+        kind: &str,
+        error: &str,
+        effect: Option<&str>,
+        now: i64,
     ) -> Result<WorkloadState, WorkloadError> {
         if sensitive_text(error) {
             return Err(WorkloadError::new(
@@ -897,23 +1174,65 @@ impl WorkloadStore {
                 "errors may not contain secret material",
             ));
         }
+        if effect.is_some_and(|e| !matches!(e, "none" | "unknown")) {
+            return Err(WorkloadError::new(
+                "WORKLOAD_INVALID",
+                "effect must be \"none\" or \"unknown\"",
+            ));
+        }
+        // The same report repeated (a lost response) replays its outcome. Any
+        // other report from an attempt that already reported is answered by
+        // the fence, as before: its claim is no longer current.
+        let current = self.fresh(id)?.clone();
+        if let Some(attempt) = reported_attempt(&current, claim, worker, token) {
+            let same = attempt.status == AttemptStatus::Failed
+                && attempt.error.as_deref() == Some(error)
+                && failure_event(&current, &attempt.attempt_id).is_some_and(|event| {
+                    let recorded_effect = event.evidence.get("effect").map(String::as_str);
+                    event.evidence.get("failure_kind").map(String::as_str) == Some(kind)
+                        && (effect.is_none() || effect == recorded_effect)
+                });
+            if same {
+                return Ok(current.state);
+            }
+        }
+        self.enforce_deadline(id, now)?;
         self.edit(id, |store, w| {
             Self::fence(w, claim, worker, token, now)?;
+            let started = w.state != WorkloadState::Claimed;
+            let effect = effect.or((!started).then_some("none"));
             let attempt = w.attempts.last_mut().unwrap();
             attempt.status = AttemptStatus::Failed;
             attempt.ended_at = Some(now);
             attempt.error = Some(error.into());
+            let attempt_id = attempt.attempt_id.clone();
             w.last_failure = Some(format!("{kind}: {error}"));
             w.current_claim = None;
-            store.transition(
-                w,
-                WorkloadState::Failed,
-                worker,
-                now,
-                [("failure_kind".into(), kind.into())].into(),
-            )?;
-            let retryable = w.retry_policy.retry_on.contains(kind)
-                && w.attempt_count < w.retry_policy.max_attempts;
+            let mut evidence: BTreeMap<String, String> = [
+                ("failure_kind".into(), kind.into()),
+                ("attempt_id".into(), attempt_id),
+            ]
+            .into();
+            if let Some(effect) = effect {
+                evidence.insert("effect".into(), effect.into());
+            }
+            store.transition(w, WorkloadState::Failed, worker, now, evidence)?;
+            let unknown = effect == Some("unknown");
+            let allowed =
+                w.retry_policy
+                    .retry_on
+                    .contains(if unknown { "worker_failure" } else { kind });
+            let decision = if !allowed && unknown {
+                "effect_unknown_not_retryable"
+            } else if !allowed {
+                "failure_kind_not_retryable"
+            } else if w.attempt_count >= attempt_budget(w) {
+                "attempt_budget_exhausted"
+            } else {
+                "retry"
+            };
+            let retryable = decision == "retry";
+            let evidence: BTreeMap<String, String> = [("decision".into(), decision.into())].into();
             if retryable {
                 w.retry_count += 1;
                 let multiplier = if w.retry_policy.backoff == BackoffPolicy::Exponential {
@@ -927,15 +1246,9 @@ impl WorkloadStore {
                     .saturating_mul(multiplier)
                     .min(w.retry_policy.max_delay_ms);
                 w.next_attempt_at = Some(now + (delay / 1000) as i64);
-                store.transition(w, WorkloadState::Ready, "rust.retry", now, BTreeMap::new())?;
+                store.transition(w, WorkloadState::Ready, "rust.retry", now, evidence)?;
             } else {
-                store.transition(
-                    w,
-                    WorkloadState::DeadLetter,
-                    "rust.retry",
-                    now,
-                    BTreeMap::new(),
-                )?;
+                store.transition(w, WorkloadState::DeadLetter, "rust.retry", now, evidence)?;
             }
             Ok(w.state)
         })
@@ -1018,14 +1331,17 @@ impl WorkloadStore {
                     "workload cannot time out in its current state",
                 ));
             }
-            if let Some(attempt) = w.attempts.last_mut() {
-                attempt.status = AttemptStatus::TimedOut;
-                attempt.ended_at = Some(now);
+            // A Ready workload has no attempt in progress: the last attempt's
+            // recorded outcome is not this timeout's to overwrite.
+            if w.state != WorkloadState::Ready {
+                if let Some(attempt) = w.attempts.last_mut() {
+                    attempt.status = AttemptStatus::TimedOut;
+                    attempt.ended_at = Some(now);
+                }
             }
             w.current_claim = None;
             store.transition(w, WorkloadState::TimedOut, actor, now, BTreeMap::new())?;
-            if w.attempt_count < w.retry_policy.max_attempts
-                && w.retry_policy.retry_on.contains("timeout")
+            if w.attempt_count < attempt_budget(w) && w.retry_policy.retry_on.contains("timeout")
             {
                 w.retry_count += 1;
                 w.next_attempt_at = Some(now + (w.retry_policy.initial_delay_ms / 1000) as i64);
@@ -1043,6 +1359,21 @@ impl WorkloadStore {
         })
     }
     pub fn retry(&mut self, id: &str, actor: &str, now: i64) -> Result<(), WorkloadError> {
+        self.retry_with_reason(id, actor, now, None)
+    }
+    /// An operator's re-arm of failed, dead-lettered or timed-out work.
+    ///
+    /// Each re-arm grants one more attempt, as it always has, and is recorded
+    /// in the history with its actor and reason. A workload may be re-armed at
+    /// most `retry_policy.max_attempts` times, so operator retries cannot
+    /// become an unbounded loop.
+    pub fn retry_with_reason(
+        &mut self,
+        id: &str,
+        actor: &str,
+        now: i64,
+        reason: Option<&str>,
+    ) -> Result<(), WorkloadError> {
         self.edit(id, |store, w| {
             if !matches!(
                 w.state,
@@ -1053,12 +1384,120 @@ impl WorkloadStore {
                     "workload is not retryable",
                 ));
             }
+            let previous = operator_retries(w);
+            if previous >= w.retry_policy.max_attempts {
+                return Err(WorkloadError::new(
+                    "WORKLOAD_RETRY_EXHAUSTED",
+                    "workload has used every operator retry its policy allows",
+                ));
+            }
             w.next_attempt_at = Some(now);
             w.current_claim = None;
-            store.transition(w, WorkloadState::Ready, actor, now, BTreeMap::new())
+            let mut evidence: BTreeMap<String, String> = [
+                ("retry".into(), "operator".into()),
+                ("operator_retry".into(), (previous + 1).to_string()),
+                (
+                    "attempt_budget".into(),
+                    w.retry_policy
+                        .max_attempts
+                        .saturating_add(previous + 1)
+                        .to_string(),
+                ),
+            ]
+            .into();
+            if let Some(reason) = reason.filter(|reason| !reason.trim().is_empty()) {
+                evidence.insert("reason".into(), reason.chars().take(256).collect());
+            }
+            store.transition(w, WorkloadState::Ready, actor, now, evidence)
+        })
+    }
+    /// Time out every workload past one of its deadlines (see
+    /// [`deadline_breach`]), as the recovery pass does each second. Deadlines
+    /// are computed from the workload's committed timestamps, so the decision
+    /// is the same after a restart and on any writer of the store.
+    pub fn enforce_deadlines(&mut self, now: i64) -> Result<Vec<String>, WorkloadError> {
+        self.refresh()?;
+        let due = self
+            .workloads
+            .values()
+            .filter_map(|w| deadline_breach(w, now).map(|breach| (w.workload_id.clone(), breach)))
+            .collect::<Vec<_>>();
+        let mut expired = Vec::with_capacity(due.len());
+        for (id, (deadline, at)) in due {
+            match self.expire(&id, deadline, at, now) {
+                Ok(()) => expired.push(id),
+                Err(error) if error.code == "WORKLOAD_CONFLICT" => {}
+                Err(error) => return Err(error),
+            }
+        }
+        Ok(expired)
+    }
+    /// Refuse a worker call on a workload past a deadline, timing it out
+    /// first, in the same way the recovery pass would have.
+    fn enforce_deadline(&mut self, id: &str, now: i64) -> Result<(), WorkloadError> {
+        let Some((deadline, at)) = deadline_breach(self.fresh(id)?, now) else {
+            return Ok(());
+        };
+        self.expire(id, deadline, at, now)?;
+        let mut error = WorkloadError::new(
+            "WORKLOAD_TIMED_OUT",
+            format!("the workload's {deadline} deadline passed"),
+        );
+        error.details = Some(serde_json::json!({ "deadline": deadline, "deadline_at": at }));
+        Err(error)
+    }
+    fn expire(&mut self, id: &str, deadline: &str, at: i64, now: i64) -> Result<(), WorkloadError> {
+        self.edit(id, |store, w| {
+            // Re-check on the committed version: a heartbeat or completion that
+            // committed first decides, and this changes nothing.
+            if deadline_breach(w, now).map(|(kind, _)| kind) != Some(deadline) {
+                return Err(WorkloadError::new(
+                    "WORKLOAD_CONFLICT",
+                    "the workload changed before its deadline was enforced",
+                ));
+            }
+            let lost = w.current_claim.take();
+            if w.state != WorkloadState::Ready {
+                if let Some(attempt) = w.attempts.last_mut() {
+                    attempt.status = AttemptStatus::TimedOut;
+                    attempt.ended_at = Some(now);
+                    attempt.error = Some(format!("timeout: {deadline} deadline passed"));
+                }
+            }
+            let mut evidence: BTreeMap<String, String> = [
+                ("deadline".into(), deadline.into()),
+                ("deadline_at".into(), at.to_string()),
+            ]
+            .into();
+            if let Some(claim) = &lost {
+                evidence.insert("claim_id".into(), claim.claim_id.clone());
+                evidence.insert("worker_id".into(), claim.worker_id.clone());
+            }
+            store.transition(w, WorkloadState::TimedOut, "rust.deadline", now, evidence)?;
+            // The overall and queue deadlines bound the work, not an attempt:
+            // another attempt cannot meet them.
+            let decision = if matches!(deadline, "overall" | "queue") {
+                "deadline_not_retryable"
+            } else if !w.retry_policy.retry_on.contains("timeout") {
+                "timeout_not_retryable"
+            } else if w.attempt_count >= attempt_budget(w) {
+                "attempt_budget_exhausted"
+            } else {
+                "retry"
+            };
+            let evidence: BTreeMap<String, String> = [("decision".into(), decision.into())].into();
+            if decision == "retry" {
+                w.retry_count += 1;
+                w.next_attempt_at = Some(now + (w.retry_policy.initial_delay_ms / 1000) as i64);
+                store.transition(w, WorkloadState::Ready, "rust.deadline", now, evidence)?;
+            } else {
+                store.transition(w, WorkloadState::DeadLetter, "rust.deadline", now, evidence)?;
+            }
+            Ok(())
         })
     }
     pub fn recover_expired(&mut self, now: i64) -> Result<Vec<String>, WorkloadError> {
+        self.refresh()?;
         let ids = self
             .workloads
             .values()
@@ -1076,41 +1515,90 @@ impl WorkloadStore {
             })
             .map(|w| w.workload_id.clone())
             .collect::<Vec<_>>();
-        for id in &ids {
-            self.edit(id, |store, w| {
+        let mut recovered = Vec::with_capacity(ids.len());
+        for id in ids {
+            let outcome = self.edit(&id, |store, w| {
+                // Decide on the committed version: another writer may have
+                // recovered (or the worker completed) it since it was listed.
+                let still_expired = w
+                    .current_claim
+                    .as_ref()
+                    .is_some_and(|c| c.expires_at <= now)
+                    && matches!(
+                        w.state,
+                        WorkloadState::Claimed
+                            | WorkloadState::Running
+                            | WorkloadState::Checkpointed
+                            | WorkloadState::CancellationRequested
+                    );
+                if !still_expired {
+                    return Err(WorkloadError::new(
+                        "WORKLOAD_CONFLICT",
+                        "the workload changed before it was recovered",
+                    ));
+                }
+                let lost = w.current_claim.take();
                 if let Some(attempt) = w.attempts.last_mut() {
                     attempt.status = AttemptStatus::WorkerLost;
                     attempt.ended_at = Some(now);
+                    attempt.error = Some("worker_lost: claim lease expired".into());
+                }
+                let mut evidence: BTreeMap<String, String> = [
+                    ("recovery_reason".into(), "lease_expired".into()),
+                    ("attempts_used".into(), w.attempt_count.to_string()),
+                    ("attempt_budget".into(), attempt_budget(w).to_string()),
+                ]
+                .into();
+                if let Some(claim) = &lost {
+                    evidence.insert("claim_id".into(), claim.claim_id.clone());
+                    evidence.insert("worker_id".into(), claim.worker_id.clone());
+                    evidence.insert("lease_expired_at".into(), claim.expires_at.to_string());
+                }
+                if let Some(attempt) = w.attempts.last() {
+                    evidence.insert("attempt_id".into(), attempt.attempt_id.clone());
                 }
-                w.current_claim = None;
                 if w.state == WorkloadState::CancellationRequested {
-                    store.transition(
-                        w,
-                        WorkloadState::Cancelled,
-                        "rust.recovery",
-                        now,
-                        BTreeMap::new(),
-                    )?;
+                    store.transition(w, WorkloadState::Cancelled, "rust.recovery", now, evidence)?;
                 } else {
                     store.transition(
                         w,
                         WorkloadState::Recovering,
                         "rust.recovery",
                         now,
-                        BTreeMap::new(),
+                        evidence,
                     )?;
+                    // A lost worker is the policy's `worker_failure`. Whether
+                    // the lost attempt had effects is unknown to FeltDB, so a
+                    // retry re-executes it: at-least-once, never exactly-once.
+                    let (to, decision) = if !w.retry_policy.retry_on.contains("worker_failure") {
+                        (WorkloadState::DeadLetter, "worker_failure_not_retryable")
+                    } else if w.attempt_count >= attempt_budget(w) {
+                        (WorkloadState::DeadLetter, "attempt_budget_exhausted")
+                    } else {
+                        (WorkloadState::Ready, "retry")
+                    };
                     store.transition(
                         w,
-                        WorkloadState::Ready,
+                        to,
                         "rust.recovery",
                         now,
-                        BTreeMap::new(),
+                        [("decision".into(), decision.into())].into(),
                     )?;
                 }
                 Ok(())
-            })?;
+            });
+            match outcome {
+                Ok(()) => recovered.push(id),
+                // Another writer of this store committed the workload first —
+                // in a race, its own recovery. Nothing of this attempt was
+                // committed and `edit` has adopted the committed workload, so
+                // the attempt is recovered once, by that writer. If it still
+                // needs recovery, the next pass sees it.
+                Err(error) if error.code == "WORKLOAD_CONFLICT" => {}
+                Err(error) => return Err(error),
+            }
         }
-        Ok(ids)
+        Ok(recovered)
     }
 }
 
@@ -1196,6 +1684,7 @@ fn legal(from: WorkloadState, to: WorkloadState) -> bool {
             | (Created, Cancelled)
             | (Failed, Cancelled)
             | (DeadLetter, Cancelled)
+            | (Claimed, Failed)
             | (Running, Failed)
             | (Checkpointed, Failed)
             | (Claimed, Recovering)
@@ -1211,6 +1700,8 @@ fn legal(from: WorkloadState, to: WorkloadState) -> bool {
             | (Checkpointed, TimedOut)
             | (Ready, TimedOut)
             | (TimedOut, DeadLetter)
+            | (Recovering, DeadLetter)
+            | (Ready, DeadLetter)
     )
 }
 pub fn can_transition(from: WorkloadState, to: WorkloadState) -> bool {
@@ -1259,6 +1750,248 @@ fn ensure_safe(value: &Value) -> Result<(), WorkloadError> {
 fn sensitive_text(v: &str) -> bool {
     v.contains("BEGIN PRIVATE KEY") || v.starts_with("sk-") || v.starts_with("fdb_live_")
 }
+/// The identity a caller's idempotency key is unique within: tenant,
+/// application and requester. Encoded as a JSON array, so no choice of key or
+/// identifier can make two different scopes coincide.
+fn idempotency_scope(tenant: &str, application: &str, requester: &Subject, key: &str) -> String {
+    serde_json::json!([tenant, application, requester.kind(), requester.id(), key]).to_string()
+}
+/// The scope before keys were bound to their requester, kept only to verify
+/// legacy store files during migration.
+fn legacy_idempotency_scope(
+    tenant: &str,
+    application: &str,
+    _requester: &Subject,
+    key: &str,
+) -> String {
+    format!("{tenant}:{application}:{key}")
+}
+fn idempotency_index(
+    workloads: &BTreeMap<String, Workload>,
+    scope: fn(&str, &str, &Subject, &str) -> String,
+) -> BTreeMap<String, String> {
+    workloads
+        .values()
+        .map(|w| {
+            (
+                scope(&w.tenant_id, &w.application_id, &w.requested_by, &w.idempotency_key),
+                w.workload_id.clone(),
+            )
+        })
+        .collect()
+}
+/// Answer a repeated key: the original workload for the same request, a
+/// refusal for a different one. Never the original for a different request.
+/// The execution-affecting part of an admission, normalized, field by field.
+///
+/// A replay under the same key is answered with the original workload only when
+/// every one of these is equal. Optional fields are compared after their
+/// defaults apply (serde fills an omitted `retry_policy`, `timeouts`,
+/// `priority`, `connection_scopes` or `dependencies`); sets compare as sets and
+/// dependencies regardless of order; the input compares by its canonical hash,
+/// so member order is not part of it. Provenance that does not change what runs
+/// — `trigger`, `session_id`, `created_at`, and the authorization snapshot,
+/// which is verified before any replay — is not compared.
+fn execution_fields(
+    definition_id: &str,
+    definition_revision: &str,
+    environment: &str,
+    input_hash: &str,
+    capability_snapshot: &BTreeSet<String>,
+    connection_scopes: &BTreeSet<String>,
+    priority: i32,
+    retry_policy: &WorkloadRetryPolicy,
+    timeouts: &WorkloadTimeouts,
+    dependencies: &[WorkloadDependency],
+    not_before: Option<i64>,
+) -> Vec<(&'static str, Value)> {
+    let mut dependencies = dependencies
+        .iter()
+        .map(|d| serde_json::json!([d.workload_id, d.required_state]))
+        .collect::<Vec<_>>();
+    dependencies.sort_by_key(|d| d.to_string());
+    vec![
+        ("definition_id", definition_id.into()),
+        ("definition_revision", definition_revision.into()),
+        ("environment", environment.into()),
+        ("input", input_hash.into()),
+        (
+            "capability_snapshot",
+            serde_json::json!(capability_snapshot),
+        ),
+        ("connection_scopes", serde_json::json!(connection_scopes)),
+        ("priority", priority.into()),
+        ("retry_policy", serde_json::json!(retry_policy)),
+        ("timeouts", serde_json::json!(timeouts)),
+        ("dependencies", Value::Array(dependencies)),
+        ("not_before", serde_json::json!(not_before)),
+    ]
+}
+/// The `not_before` a workload was admitted with, from its creation event
+/// (`next_attempt_at` moves with retries; the admission evidence does not).
+fn admitted_not_before(w: &Workload) -> Option<i64> {
+    w.execution_history
+        .iter()
+        .find(|event| event.to == WorkloadState::Ready)
+        .and_then(|event| event.evidence.get("not_before"))
+        .and_then(|value| value.parse().ok())
+}
+fn replay(existing: Workload, input: &CreateWorkload) -> Result<(Workload, bool), WorkloadError> {
+    let recorded = execution_fields(
+        &existing.definition_id,
+        &existing.definition_revision,
+        &existing.environment,
+        &existing.input_hash,
+        &existing.capability_snapshot,
+        &existing.connection_scopes,
+        existing.priority,
+        &existing.retry_policy,
+        &existing.timeouts,
+        &existing.dependencies,
+        admitted_not_before(&existing),
+    );
+    let requested = execution_fields(
+        &input.definition_id,
+        &input.definition_revision,
+        &input.environment,
+        &hash(&input.input),
+        &input.capability_snapshot,
+        &input.connection_scopes,
+        input.priority,
+        &input.retry_policy,
+        &input.timeouts,
+        &input.dependencies,
+        input.not_before,
+    );
+    let mut conflicting = recorded
+        .iter()
+        .zip(&requested)
+        .filter(|((_, left), (_, right))| left != right)
+        .map(|((name, _), _)| *name)
+        .collect::<Vec<_>>();
+    if existing.requested_by != input.requested_by {
+        conflicting.insert(0, "requested_by");
+    }
+    if conflicting.is_empty() {
+        return Ok((existing, false));
+    }
+    let mut error = WorkloadError::new(
+        "IDEMPOTENCY_KEY_REUSED",
+        "idempotency key was already used for a different request",
+    );
+    error.details = Some(serde_json::json!({ "fields": conflicting }));
+    Err(error)
+}
+/// Seconds a millisecond timeout lasts, rounded up so a deadline is never early.
+fn deadline_seconds(ms: u64) -> i64 {
+    ms.div_ceil(1000) as i64
+}
+/// The first deadline a workload has passed at `now`, and when it fell.
+///
+/// | Deadline | Measured from | While |
+/// | --- | --- | --- |
+/// | `overall` | admission (`created_at`) | not terminal |
+/// | `queue` | becoming `READY` (or its `next_attempt_at`, if later) | `READY` |
+/// | `claim` | the claim | `CLAIMED` |
+/// | `execution` | the attempt's start | `RUNNING`, `CHECKPOINTED` |
+/// | `heartbeat` | the last heartbeat (or the claim) | claimed |
+///
+/// Only workloads admitted at [`DEADLINE_CONTRACT_VERSION`] or later.
+pub fn deadline_breach(w: &Workload, now: i64) -> Option<(&'static str, i64)> {
+    use WorkloadState::*;
+    if w.contract_version < DEADLINE_CONTRACT_VERSION {
+        return None;
+    }
+    let active = matches!(w.state, Ready | Claimed | Running | Checkpointed);
+    if !active {
+        return None;
+    }
+    let t = &w.timeouts;
+    let claim = w.current_claim.as_ref();
+    let candidates = [
+        (
+            "overall",
+            t.overall_ms.map(|ms| w.created_at + deadline_seconds(ms)),
+        ),
+        (
+            "queue",
+            t.queue_ms.filter(|_| w.state == Ready).map(|ms| {
+                let ready_since = w
+                    .execution_history
+                    .iter()
+                    .rev()
+                    .find(|event| event.to == Ready)
+                    .map_or(w.created_at, |event| event.at);
+                ready_since.max(w.next_attempt_at.unwrap_or(i64::MIN)) + deadline_seconds(ms)
+            }),
+        ),
+        (
+            "claim",
+            t.claim_ms
+                .filter(|_| w.state == Claimed)
+                .and_then(|ms| claim.map(|c| c.claimed_at + deadline_seconds(ms))),
+        ),
+        (
+            "execution",
+            t.execution_ms
+                .filter(|_| matches!(w.state, Running | Checkpointed))
+                .and_then(|ms| {
+                    w.attempts
+                        .last()
+                        .and_then(|a| a.started_at)
+                        .map(|started| started + deadline_seconds(ms))
+                }),
+        ),
+        (
+            "heartbeat",
+            t.heartbeat_ms
+                .and_then(|ms| claim.map(|c| c.last_heartbeat + deadline_seconds(ms))),
+        ),
+    ];
+    candidates
+        .into_iter()
+        .find_map(|(kind, at)| at.filter(|at| now >= *at).map(|at| (kind, at)))
+}
+/// The attempt this fence named, when that attempt has already reported its
+/// outcome (failed or succeeded). Lost and timed-out attempts are not reports.
+fn reported_attempt<'w>(
+    w: &'w Workload,
+    claim: &str,
+    worker: &str,
+    token: u64,
+) -> Option<&'w WorkloadAttempt> {
+    w.attempts.iter().find(|attempt| {
+        attempt.claim_id == claim
+            && attempt.worker_id == worker
+            && attempt.fencing_token == token
+            && matches!(
+                attempt.status,
+                AttemptStatus::Failed | AttemptStatus::Succeeded
+            )
+    })
+}
+/// The `FAILED` event a failure report of `attempt_id` recorded.
+fn failure_event<'w>(w: &'w Workload, attempt_id: &str) -> Option<&'w WorkloadEvent> {
+    w.execution_history.iter().rev().find(|event| {
+        event.to == WorkloadState::Failed
+            && event.evidence.get("attempt_id").map(String::as_str) == Some(attempt_id)
+    })
+}
+/// Operator re-arms recorded in the workload's own history.
+fn operator_retries(w: &Workload) -> u32 {
+    w.execution_history
+        .iter()
+        .filter(|event| event.evidence.get("retry").map(String::as_str) == Some("operator"))
+        .count() as u32
+}
+/// How many claims the workload may have in total: its policy's
+/// `max_attempts`, plus one for each operator re-arm (which is what an operator
+/// retry has always granted).
+fn attempt_budget(w: &Workload) -> u32 {
+    w.retry_policy
+        .max_attempts
+        .saturating_add(operator_retries(w))
+}
 fn work_key(workload: &Workload) -> String {
     format!(
         "{}/{}/{}",
@@ -1312,8 +2045,12 @@ mod tests {
     };
     use std::{fs, path::PathBuf};
     fn create() -> CreateWorkload {
+        create_as("a1")
+    }
+    /// The fixture request, made by agent `agent`.
+    fn create_as(agent: &str) -> CreateWorkload {
         let signer = GrantSigner::new("k", vec![1; 32]).unwrap();
-        let subject = Subject::Agent("a1".into());
+        let subject = Subject::Agent(agent.into());
         let grant = Grant {
             contract_version: 1,
             id: "g".into(),
@@ -1355,66 +2092,404 @@ mod tests {
             created_at: 10,
         }
     }
-    fn store(name: &str) -> (PathBuf, WorkloadStore) {
-        let path =
-            std::env::temp_dir().join(format!("workload-{name}-{}.json", std::process::id()));
-        let store = WorkloadStore::load(&path).unwrap();
-        (path, store)
+    fn capabilities() -> BTreeSet<String> {
+        ["email:send".into()].into()
     }
+
     #[test]
-    fn legacy_migration_is_idempotent_conflict_detecting_and_source_independent() {
-        let (source_db, mut source) = store("legacy-source-builder");
-        let workload = source.create(create()).unwrap();
-        let legacy = LegacyWorkloadStore {
-            workloads: source.workloads.clone(),
-            idempotency: source.idempotency.clone(),
-            result_ids: source.result_ids.clone(),
-            event_sequence: derived_event_sequence(&source.workloads),
-        };
-        let legacy_path = source_db.with_extension("legacy-workloads.json");
-        fs::write(&legacy_path, serde_json::to_vec(&legacy).unwrap()).unwrap();
-        let canonical_path = source_db.with_extension("canonical.log");
-        let db = FeltDb::open(&canonical_path).unwrap();
-        let migrated = WorkloadStore::migrate_legacy_file(db.clone(), &legacy_path).unwrap();
-        assert_eq!(
-            migrated.get(&workload.workload_id).unwrap().input_hash,
-            workload.input_hash
-        );
-        WorkloadStore::migrate_legacy_file(db.clone(), &legacy_path).unwrap();
-        fs::remove_file(&legacy_path).unwrap();
-        let reopened = WorkloadStore::from_db(FeltDb::open(&canonical_path).unwrap()).unwrap();
-        assert_eq!(
-            reopened.get(&workload.workload_id).unwrap().workload_id,
-            workload.workload_id
-        );
+    fn idempotency_keys_are_bound_to_their_requester_and_request() {
+        let (path, mut store) = store("requester-scoped-keys");
+        let (original, created) = store.admit(create_as("a1")).unwrap();
+        assert!(created);
 
-        let mut conflicting = legacy;
-        conflicting
-            .workloads
-            .get_mut(&workload.workload_id)
-            .unwrap()
-            .priority += 1;
-        fs::write(&legacy_path, serde_json::to_vec(&conflicting).unwrap()).unwrap();
+        // The same requester and request: the original, not a second workload.
+        let (replayed, created) = store.admit(create_as("a1")).unwrap();
+        assert!(!created);
+        assert_eq!(replayed.workload_id, original.workload_id);
+
+        // The same key for a different request is refused and creates nothing.
+        let mut different = create_as("a1");
+        different.input = serde_json::json!({"id": 2});
+        assert_eq!(store.admit(different).unwrap_err().code, "IDEMPOTENCY_KEY_REUSED");
+        let mut other_definition = create_as("a1");
+        other_definition.definition_id = "other".into();
         assert_eq!(
-            WorkloadStore::migrate_legacy_file(db, &legacy_path)
-                .err()
-                .unwrap()
-                .code,
-            "WORKLOAD_MIGRATION_CONFLICT"
+            store.admit(other_definition).unwrap_err().code,
+            "IDEMPOTENCY_KEY_REUSED"
         );
-        let _ = fs::remove_file(source_db);
-        let _ = fs::remove_file(canonical_path);
-        let _ = fs::remove_file(legacy_path);
+
+        // Another requester's identical key is its own workload: it neither
+        // returns, reveals nor blocks the first requester's.
+        let (theirs, created) = store.admit(create_as("a2")).unwrap();
+        assert!(created);
+        assert_ne!(theirs.workload_id, original.workload_id);
+        assert_eq!(store.workloads.len(), 2);
+
+        // A snapshot that does not bind the requester is refused before any
+        // replay, so a key cannot be used to fetch a workload.
+        let mut unbound = create_as("a1");
+        unbound.requested_by = Subject::Agent("a2".into());
+        assert_eq!(store.admit(unbound).unwrap_err().code, "WORKLOAD_UNAUTHORIZED");
+        let _ = fs::remove_file(path);
     }
 
     #[test]
-    fn canonical_history_reconstructs_work_identity_provenance() {
-        let (path, mut store) = store("provenance");
+    fn lease_recovery_records_the_lost_worker_and_is_bounded_by_the_attempt_budget() {
+        let (path, mut store) = store("bounded-recovery");
         let workload = store.create(create()).unwrap();
-        let transaction = store
-            .db
-            .committed_transactions_after(0, usize::MAX)
-            .unwrap()
+        let id = workload.workload_id.clone();
+        for attempt in 1..=3u32 {
+            let now = 100 * attempt as i64;
+            let claim = store.claim(&id, "worker-a", &capabilities(), now, 10_000).unwrap();
+            assert_eq!(store.recover_expired(claim.expires_at).unwrap(), vec![id.clone()]);
+            let recovered = store.get(&id).unwrap();
+            let lost = recovered.attempts.last().unwrap();
+            assert_eq!(lost.status, AttemptStatus::WorkerLost);
+            assert_eq!(lost.claim_id, claim.claim_id);
+            let recovering = recovered
+                .execution_history
+                .iter()
+                .rev()
+                .find(|event| event.to == WorkloadState::Recovering)
+                .unwrap();
+            assert_eq!(recovering.evidence["recovery_reason"], "lease_expired");
+            assert_eq!(recovering.evidence["worker_id"], "worker-a");
+            assert_eq!(recovering.evidence["claim_id"], claim.claim_id);
+            assert_eq!(recovering.evidence["attempts_used"], attempt.to_string());
+            let decision = &recovered.execution_history.last().unwrap();
+            if attempt < 3 {
+                assert_eq!(recovered.state, WorkloadState::Ready);
+                assert_eq!(decision.evidence["decision"], "retry");
+            } else {
+                assert_eq!(recovered.state, WorkloadState::DeadLetter);
+                assert_eq!(decision.evidence["decision"], "attempt_budget_exhausted");
+            }
+        }
+        assert_eq!(store.get(&id).unwrap().attempt_count, 3);
+        assert_eq!(
+            store.claim(&id, "worker-b", &capabilities(), 1_000, 10_000).unwrap_err().code,
+            "WORKLOAD_NOT_READY"
+        );
+        let _ = fs::remove_file(path);
+    }
+
+    /// The recovery history of `id` as committed, read by a fresh projection.
+    fn committed_workload(db: &FeltDb, id: &str) -> Workload {
+        WorkloadStore::from_db(db.clone()).unwrap().get(id).unwrap().clone()
+    }
+
+    fn recoveries(workload: &Workload) -> usize {
+        workload
+            .execution_history
+            .iter()
+            .filter(|event| event.to == WorkloadState::Recovering)
+            .count()
+    }
+
+    /// Two writers of one store recover the same expired workload at once.
+    fn race_recovery(db: &FeltDb, now: i64) -> (WorkloadStore, WorkloadStore, Vec<Vec<String>>) {
+        let stores = [
+            WorkloadStore::from_db(db.clone()).unwrap(),
+            WorkloadStore::from_db(db.clone()).unwrap(),
+        ];
+        let barrier = std::sync::Barrier::new(2);
+        let results = std::thread::scope(|scope| {
+            let handles = stores
+                .into_iter()
+                .map(|mut store| {
+                    let barrier = &barrier;
+                    scope.spawn(move || {
+                        barrier.wait();
+                        let recovered = store.recover_expired(now).unwrap();
+                        (store, recovered)
+                    })
+                })
+                .collect::<Vec<_>>();
+            handles
+                .into_iter()
+                .map(|handle| handle.join().unwrap())
+                .collect::<Vec<_>>()
+        });
+        let mut results = results.into_iter();
+        let (first, first_recovered) = results.next().unwrap();
+        let (second, second_recovered) = results.next().unwrap();
+        (first, second, vec![first_recovered, second_recovered])
+    }
+
+    #[test]
+    fn concurrent_recovery_commits_one_transition_and_the_loser_converges() {
+        for round in 0..20 {
+            let (path, mut store) = store(&format!("concurrent-recovery-{round}"));
+            let db = store.db.clone();
+            let id = store.create(create()).unwrap().workload_id;
+            let claim = store.claim(&id, "worker-a", &capabilities(), 100, 10_000).unwrap();
+            // A writer that loaded the workload before either recovery.
+            // Commits land in this projection's window between refresh and
+            // commit: only the storage guard can refuse them.
+            let mut stale = WorkloadStore::from_db(db.clone())
+                .unwrap()
+                .without_refresh();
+
+            let (mut first, mut second, recovered) = race_recovery(&db, claim.expires_at);
+            // Exactly one writer committed the recovery; the other reported none.
+            let mut outcomes = recovered.clone();
+            outcomes.sort();
+            assert_eq!(outcomes, vec![vec![], vec![id.clone()]], "round {round}");
+
+            let committed = committed_workload(&db, &id);
+            assert_eq!(recoveries(&committed), 1, "one recovery transition");
+            assert_eq!(committed.state, WorkloadState::Ready);
+            assert_eq!(committed.attempt_count, 1);
+            assert_eq!(committed.attempts.len(), 1);
+            assert_eq!(committed.attempts[0].status, AttemptStatus::WorkerLost);
+            assert!(committed.current_claim.is_none());
+            let recovering = committed
+                .execution_history
+                .iter()
+                .find(|event| event.to == WorkloadState::Recovering)
+                .unwrap();
+            assert_eq!(recovering.evidence["claim_id"], claim.claim_id);
+            assert_eq!(recovering.evidence["attempts_used"], "1");
+
+            // The losing writer adopted the committed workload instead of
+            // keeping its own, uncommitted view.
+            let expected = serde_json::to_value(&committed).unwrap();
+            assert_eq!(serde_json::to_value(first.get(&id).unwrap()).unwrap(), expected);
+            assert_eq!(serde_json::to_value(second.get(&id).unwrap()).unwrap(), expected);
+            // And neither recovers it again, or commits anything more.
+            assert!(first.recover_expired(claim.expires_at + 1).unwrap().is_empty());
+            assert!(second.recover_expired(claim.expires_at + 1).unwrap().is_empty());
+            assert_eq!(
+                serde_json::to_value(committed_workload(&db, &id)).unwrap(),
+                expected,
+                "no further commit"
+            );
+
+            // The lost claimant cannot complete, even through a projection
+            // that still shows its claim as current and unexpired: the guarded
+            // commit refuses the stale version.
+            let refused = stale
+                .start(&id, &claim.claim_id, "worker-a", claim.fencing_token, claim.expires_at - 1)
+                .unwrap_err();
+            assert_eq!(refused.code, "WORKLOAD_CONFLICT");
+            assert_eq!(stale.get(&id).unwrap().state, WorkloadState::Ready);
+            let refused = stale
+                .complete(
+                    &id,
+                    &claim.claim_id,
+                    "worker-a",
+                    claim.fencing_token,
+                    "late-result",
+                    serde_json::json!({"late": true}),
+                    vec![],
+                    1,
+                    claim.expires_at - 1,
+                )
+                .unwrap_err();
+            assert_eq!(refused.code, "WORKLOAD_CLAIM_INVALID");
+            // Through a current projection its fence no longer matches.
+            let refused = first
+                .heartbeat(&id, &claim.claim_id, "worker-a", claim.fencing_token, claim.expires_at - 1, 10_000)
+                .unwrap_err();
+            assert_eq!(refused.code, "WORKLOAD_CLAIM_INVALID");
+            assert_eq!(serde_json::to_value(committed_workload(&db, &id)).unwrap(), expected);
+            drop((store, first, second, stale, db));
+            let _ = fs::remove_file(path);
+        }
+    }
+
+    #[test]
+    fn concurrent_recovery_never_exceeds_the_attempt_budget() {
+        let (path, mut store) = store("concurrent-recovery-budget");
+        let db = store.db.clone();
+        let id = store.create(create()).unwrap().workload_id;
+        for attempt in 1..=3i64 {
+            // Each attempt is claimed by an up-to-date writer and lost.
+            let mut claimer = WorkloadStore::from_db(db.clone()).unwrap();
+            let claim = claimer
+                .claim(&id, "worker-a", &capabilities(), 100 * attempt, 10_000)
+                .unwrap();
+            let (_, _, recovered) = race_recovery(&db, claim.expires_at);
+            assert_eq!(recovered.concat(), vec![id.clone()], "attempt {attempt}");
+            let committed = committed_workload(&db, &id);
+            assert_eq!(recoveries(&committed), attempt as usize);
+            assert_eq!(committed.attempt_count, attempt as u32);
+        }
+        let committed = committed_workload(&db, &id);
+        assert_eq!(committed.state, WorkloadState::DeadLetter);
+        assert_eq!(
+            committed.execution_history.last().unwrap().evidence["decision"],
+            "attempt_budget_exhausted"
+        );
+        assert!(committed
+            .attempts
+            .iter()
+            .all(|attempt| attempt.status == AttemptStatus::WorkerLost));
+        let refused = WorkloadStore::from_db(db.clone())
+            .unwrap()
+            .claim(&id, "worker-b", &capabilities(), 10_000, 10_000)
+            .unwrap_err();
+        assert_eq!(refused.code, "WORKLOAD_NOT_READY");
+        assert_eq!(committed_workload(&db, &id).attempt_count, 3);
+        drop((store, db));
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn a_policy_that_does_not_retry_worker_failure_dead_letters_a_lost_attempt() {
+        let (path, mut store) = store("no-worker-failure-retry");
+        let mut request = create();
+        request.retry_policy.retry_on = ["timeout".into()].into();
+        let id = store.create(request).unwrap().workload_id;
+        let claim = store.claim(&id, "worker-a", &capabilities(), 100, 10_000).unwrap();
+        store.recover_expired(claim.expires_at).unwrap();
+        let workload = store.get(&id).unwrap();
+        assert_eq!(workload.state, WorkloadState::DeadLetter);
+        assert_eq!(
+            workload.execution_history.last().unwrap().evidence["decision"],
+            "worker_failure_not_retryable"
+        );
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn claiming_exhausted_ready_work_dead_letters_it_instead() {
+        let (path, mut store) = store("exhausted-claim");
+        let id = store.create(create()).unwrap().workload_id;
+        // What unbounded recovery used to leave behind: Ready, budget spent.
+        let mut legacy = store.get(&id).unwrap().clone();
+        legacy.attempt_count = 7;
+        store.workloads.insert(id.clone(), legacy);
+        store.persist_workload(&id, "fixture.legacy").unwrap();
+        let refused = store.claim(&id, "worker-a", &capabilities(), 100, 10_000).unwrap_err();
+        assert_eq!(refused.code, "WORKLOAD_ATTEMPTS_EXHAUSTED");
+        let workload = store.get(&id).unwrap();
+        assert_eq!(workload.state, WorkloadState::DeadLetter);
+        assert_eq!(
+            workload.execution_history.last().unwrap().evidence["reason"],
+            "attempt_budget_exhausted"
+        );
+        // Durable: a restart sees the same outcome.
+        let reopened = WorkloadStore::from_db(store.db.clone()).unwrap();
+        assert_eq!(reopened.get(&id).unwrap().state, WorkloadState::DeadLetter);
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn operator_retry_grants_one_attempt_is_recorded_and_is_bounded() {
+        let (path, mut store) = store("operator-retry");
+        let id = store.create(create()).unwrap().workload_id;
+        // Claim and lose leases until the workload stops being claimable.
+        fn exhaust(store: &mut WorkloadStore, id: &str, now: &mut i64) {
+            loop {
+                *now += 100;
+                match store.claim(id, "worker-a", &capabilities(), *now, 10_000) {
+                    Ok(claim) => {
+                        store.recover_expired(claim.expires_at).unwrap();
+                    }
+                    Err(error) => {
+                        assert!(
+                            matches!(
+                                error.code.as_str(),
+                                "WORKLOAD_NOT_READY" | "WORKLOAD_ATTEMPTS_EXHAUSTED"
+                            ),
+                            "{error:?}"
+                        );
+                        return;
+                    }
+                }
+            }
+        }
+        let mut now = 100;
+        exhaust(&mut store, &id, &mut now);
+        assert_eq!(store.get(&id).unwrap().attempt_count, 3);
+        for retry in 1..=3u32 {
+            now += 100;
+            store
+                .retry_with_reason(&id, "operator-1", now, Some("dependency restored"))
+                .unwrap();
+            let event = store.get(&id).unwrap().execution_history.last().unwrap().clone();
+            assert_eq!(event.actor, "operator-1");
+            assert_eq!(event.evidence["retry"], "operator");
+            assert_eq!(event.evidence["reason"], "dependency restored");
+            assert_eq!(event.evidence["operator_retry"], retry.to_string());
+            exhaust(&mut store, &id, &mut now);
+            assert_eq!(
+                store.get(&id).unwrap().attempt_count,
+                3 + retry,
+                "each operator retry grants exactly one attempt"
+            );
+            assert_eq!(store.get(&id).unwrap().state, WorkloadState::DeadLetter);
+        }
+        assert_eq!(
+            store.retry(&id, "operator-1", now + 100).unwrap_err().code,
+            "WORKLOAD_RETRY_EXHAUSTED"
+        );
+        assert_eq!(store.get(&id).unwrap().state, WorkloadState::DeadLetter);
+        let _ = fs::remove_file(path);
+    }
+
+    fn store(name: &str) -> (PathBuf, WorkloadStore) {
+        let path =
+            std::env::temp_dir().join(format!("workload-{name}-{}.json", std::process::id()));
+        let store = WorkloadStore::load(&path).unwrap();
+        (path, store)
+    }
+    #[test]
+    fn legacy_migration_is_idempotent_conflict_detecting_and_source_independent() {
+        let (source_db, mut source) = store("legacy-source-builder");
+        let workload = source.create(create()).unwrap();
+        let legacy = LegacyWorkloadStore {
+            workloads: source.workloads.clone(),
+            // What a pre-requester-scope release wrote.
+            idempotency: idempotency_index(&source.workloads, legacy_idempotency_scope),
+            result_ids: source.result_ids.clone(),
+            event_sequence: derived_event_sequence(&source.workloads),
+        };
+        let legacy_path = source_db.with_extension("legacy-workloads.json");
+        fs::write(&legacy_path, serde_json::to_vec(&legacy).unwrap()).unwrap();
+        let canonical_path = source_db.with_extension("canonical.log");
+        let db = FeltDb::open(&canonical_path).unwrap();
+        let migrated = WorkloadStore::migrate_legacy_file(db.clone(), &legacy_path).unwrap();
+        assert_eq!(
+            migrated.get(&workload.workload_id).unwrap().input_hash,
+            workload.input_hash
+        );
+        WorkloadStore::migrate_legacy_file(db.clone(), &legacy_path).unwrap();
+        fs::remove_file(&legacy_path).unwrap();
+        let reopened = WorkloadStore::from_db(FeltDb::open(&canonical_path).unwrap()).unwrap();
+        assert_eq!(
+            reopened.get(&workload.workload_id).unwrap().workload_id,
+            workload.workload_id
+        );
+
+        let mut conflicting = legacy;
+        conflicting
+            .workloads
+            .get_mut(&workload.workload_id)
+            .unwrap()
+            .priority += 1;
+        fs::write(&legacy_path, serde_json::to_vec(&conflicting).unwrap()).unwrap();
+        assert_eq!(
+            WorkloadStore::migrate_legacy_file(db, &legacy_path)
+                .err()
+                .unwrap()
+                .code,
+            "WORKLOAD_MIGRATION_CONFLICT"
+        );
+        let _ = fs::remove_file(source_db);
+        let _ = fs::remove_file(canonical_path);
+        let _ = fs::remove_file(legacy_path);
+    }
+
+    #[test]
+    fn canonical_history_reconstructs_work_identity_provenance() {
+        let (path, mut store) = store("provenance");
+        let workload = store.create(create()).unwrap();
+        let transaction = store
+            .db
+            .committed_transactions_after(0, usize::MAX)
+            .unwrap()
             .into_iter()
             .last()
             .unwrap();
@@ -1454,11 +2529,18 @@ mod tests {
         let caps = ["email:send".into()].into();
 
         let mut winner = WorkloadStore::from_db(db.clone()).unwrap();
-        let mut loser = WorkloadStore::from_db(db.clone()).unwrap();
+        // The loser's claim races the winner's inside the window between its
+        // refresh and its commit, which only the storage guard closes.
+        let mut loser = WorkloadStore::from_db(db.clone())
+            .unwrap()
+            .without_refresh();
         winner.claim(&id, "worker-a", &caps, 11, 1_000).unwrap();
         let conflict = loser.claim(&id, "worker-b", &caps, 11, 1_000).unwrap_err();
         assert_eq!(conflict.code, "WORKLOAD_CONFLICT");
-        assert_eq!(loser.get(&id).unwrap().state, WorkloadState::Ready);
+        // The loser committed nothing and now projects the winner's claim.
+        let projected = loser.get(&id).unwrap();
+        assert_eq!(projected.state, WorkloadState::Claimed);
+        assert_eq!(projected.current_claim.as_ref().unwrap().worker_id, "worker-a");
 
         let row = db
             .list_collection(DURABLE_WORK_COLLECTION)
@@ -1473,7 +2555,9 @@ mod tests {
         let mut recovery = WorkloadStore::from_db(restarted_db.clone()).unwrap();
         recovery.recover_expired(13).unwrap();
         let mut restarted_winner = WorkloadStore::from_db(restarted_db.clone()).unwrap();
-        let mut restarted_loser = WorkloadStore::from_db(restarted_db.clone()).unwrap();
+        let mut restarted_loser = WorkloadStore::from_db(restarted_db.clone())
+            .unwrap()
+            .without_refresh();
         restarted_winner
             .claim(&id, "worker-c", &caps, 13, 10_000)
             .unwrap();
@@ -1481,10 +2565,9 @@ mod tests {
             .claim(&id, "worker-d", &caps, 13, 10_000)
             .unwrap_err();
         assert_eq!(conflict.code, "WORKLOAD_CONFLICT");
-        assert_eq!(
-            restarted_loser.get(&id).unwrap().state,
-            WorkloadState::Ready
-        );
+        let projected = restarted_loser.get(&id).unwrap();
+        assert_eq!(projected.state, WorkloadState::Claimed);
+        assert_eq!(projected.current_claim.as_ref().unwrap().worker_id, "worker-c");
         drop((recovery, restarted_winner, restarted_loser, restarted_db));
         let _ = fs::remove_file(path);
     }
@@ -1499,22 +2582,103 @@ mod tests {
         let mut first = WorkloadStore::from_db(db.clone()).unwrap();
         let mut second = WorkloadStore::from_db(db.clone()).unwrap();
         let accepted = first.create(create()).unwrap();
+        // A second writer of the same store loses the require-absent CAS and
+        // converges on what was committed: a different request is refused as a
+        // reused key, never answered with the original.
         let mut competing = create();
         competing.input = serde_json::json!({"id": 2});
         let conflict = second.create(competing).unwrap_err();
-        assert_eq!(conflict.code, "WORKLOAD_CONFLICT");
-        assert!(
-            second.workloads.is_empty(),
-            "failed projection was rolled back"
+        assert_eq!(conflict.code, "IDEMPOTENCY_KEY_REUSED");
+        assert_eq!(
+            second.workloads.keys().collect::<Vec<_>>(),
+            vec![&accepted.workload_id],
+            "only the committed workload is projected"
         );
+        // The same request from a third, equally stale writer converges on it.
+        let mut third = WorkloadStore::from_db(db.clone()).unwrap();
+        third.workloads.clear();
+        third.idempotency.clear();
+        third.document_versions.clear();
+        let (converged, created) = third.admit(create()).unwrap();
+        assert!(!created);
+        assert_eq!(converged.workload_id, accepted.workload_id);
 
-        drop((first, second, db));
+        drop((first, second, third, db));
         let restarted = WorkloadStore::from_db(FeltDb::open(&path).unwrap()).unwrap();
         assert_eq!(restarted.workloads.len(), 1);
-        assert_eq!(restarted.idempotency["t:app:one"], accepted.workload_id);
+        assert_eq!(
+            restarted.idempotency[&idempotency_scope("t", "app", &accepted.requested_by, "one")],
+            accepted.workload_id
+        );
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn idempotency_scopes_cannot_be_made_to_coincide() {
+        let (path, mut store) = store("scope-separators");
+        let mut first = create_as("x:k1");
+        first.idempotency_key = "k2".into();
+        let mut second = create_as("x");
+        second.idempotency_key = "k1:k2".into();
+        let first = store.create(first).unwrap();
+        let second = store.create(second).unwrap();
+        assert_ne!(first.workload_id, second.workload_id);
         let _ = fs::remove_file(path);
     }
 
+    #[test]
+    fn concurrent_admissions_by_competing_writers_converge_or_conflict() {
+        for round in 0..20 {
+            let (path, seed) = store(&format!("concurrent-admission-{round}"));
+            let db = seed.db.clone();
+            let admit_concurrently = |requests: [CreateWorkload; 2]| {
+                let barrier = std::sync::Barrier::new(2);
+                std::thread::scope(|scope| {
+                    requests
+                        .into_iter()
+                        .map(|request| {
+                            let mut writer = WorkloadStore::from_db(db.clone()).unwrap();
+                            let barrier = &barrier;
+                            scope.spawn(move || {
+                                barrier.wait();
+                                writer.admit(request)
+                            })
+                        })
+                        .collect::<Vec<_>>()
+                        .into_iter()
+                        .map(|handle| handle.join().unwrap())
+                        .collect::<Vec<_>>()
+                })
+            };
+            // The same request: both writers answer with one workload, and
+            // exactly one of them created it.
+            let same = admit_concurrently([create(), create()]);
+            let (first, first_created) = same[0].as_ref().unwrap().clone();
+            let (second, second_created) = same[1].as_ref().unwrap().clone();
+            assert_eq!(first.workload_id, second.workload_id);
+            assert!(first_created ^ second_created, "round {round}");
+
+            // Different requests under one key: one is admitted, the other is
+            // refused as a reused key and never answered with the winner.
+            let mut left = create();
+            left.idempotency_key = "contested".into();
+            let mut right = left.clone();
+            right.input = serde_json::json!({"id": 2});
+            let contested = admit_concurrently([left, right]);
+            let admitted = contested.iter().filter(|outcome| outcome.is_ok()).count();
+            assert_eq!(admitted, 1, "round {round}");
+            assert!(contested.iter().any(|outcome| outcome
+                .as_ref()
+                .is_err_and(|error| error.code == "IDEMPOTENCY_KEY_REUSED")));
+            let (winner, _) = contested.iter().find_map(|o| o.as_ref().ok()).unwrap();
+            let stored = WorkloadStore::from_db(db.clone()).unwrap();
+            assert_eq!(stored.workloads.len(), 2);
+            assert_eq!(stored.get(&winner.workload_id).unwrap().input, winner.input);
+            drop((seed, stored, db));
+            let _ = fs::remove_file(path);
+        }
+    }
+
     #[test]
     fn pre_versioned_canonical_rows_upgrade_through_the_standard_cas_contract() {
         let (path, mut store) = store("pre-versioned-upgrade");
@@ -1589,7 +2753,9 @@ mod tests {
         store.document_versions.clear();
         let rebuilt = WorkloadStore::from_db(store.db.clone()).unwrap();
         assert_eq!(
-            rebuilt.idempotency.get("t:app:one"),
+            rebuilt
+                .idempotency
+                .get(&idempotency_scope("t", "app", &workload.requested_by, "one")),
             Some(&workload.workload_id)
         );
         assert_eq!(
@@ -1738,6 +2904,136 @@ mod tests {
         let _ = fs::remove_file(path);
     }
     #[test]
+    fn a_repeated_completion_replays_only_for_the_claim_and_output_that_produced_it() {
+        let (path, mut store) = store("complete-replay");
+        let w = store.create(create()).unwrap();
+        let id = w.workload_id.clone();
+        let caps = capabilities();
+        // Worker a loses its lease; worker b completes the work.
+        let lost = store.claim(&id, "a", &caps, 11, 1000).unwrap();
+        store
+            .start(&id, &lost.claim_id, "a", lost.fencing_token, 11)
+            .unwrap();
+        store.recover_expired(13).unwrap();
+        let won = store.claim(&id, "b", &caps, 13, 10_000).unwrap();
+        store
+            .start(&id, &won.claim_id, "b", won.fencing_token, 13)
+            .unwrap();
+        let output = serde_json::json!({"sent": 1});
+        let first = store
+            .complete(
+                &id,
+                &won.claim_id,
+                "b",
+                won.fencing_token,
+                "r1",
+                output.clone(),
+                vec![],
+                1,
+                14,
+            )
+            .unwrap();
+        // The reporting attempt retrying after a lost response gets the same result.
+        let replay = store
+            .complete(
+                &id,
+                &won.claim_id,
+                "b",
+                won.fencing_token,
+                "r1",
+                output.clone(),
+                vec![],
+                1,
+                15,
+            )
+            .unwrap();
+        assert_eq!(replay.output_hash, first.output_hash);
+        assert_eq!(replay.attempt_id, first.attempt_id);
+        // The same result id with a different output is a conflict, not a replay.
+        assert_eq!(
+            store
+                .complete(
+                    &id,
+                    &won.claim_id,
+                    "b",
+                    won.fencing_token,
+                    "r1",
+                    serde_json::json!({"sent": 2}),
+                    vec![],
+                    1,
+                    15
+                )
+                .unwrap_err()
+                .code,
+            "WORKLOAD_RESULT_CONFLICT"
+        );
+        // A stale worker naming the winner's result id learns nothing landed for it.
+        assert_eq!(
+            store
+                .complete(
+                    &id,
+                    &lost.claim_id,
+                    "a",
+                    lost.fencing_token,
+                    "r1",
+                    output.clone(),
+                    vec![],
+                    1,
+                    15
+                )
+                .unwrap_err()
+                .code,
+            "WORKLOAD_RESULT_CONFLICT"
+        );
+        let committed = store.get(&id).unwrap();
+        assert_eq!(committed.state, WorkloadState::Succeeded);
+        assert_eq!(committed.result.as_ref().unwrap().output, output);
+        assert_eq!(committed.result.as_ref().unwrap().produced_by, "b");
+        let _ = fs::remove_file(path);
+    }
+    #[test]
+    fn a_resumed_attempt_reads_the_input_and_the_lost_attempts_checkpoints_through_its_fence() {
+        let (path, mut store) = store("resume-context");
+        let w = store.create(create()).unwrap();
+        let id = w.workload_id.clone();
+        let caps = capabilities();
+        let lost = store.claim(&id, "a", &caps, 11, 1000).unwrap();
+        store
+            .start(&id, &lost.claim_id, "a", lost.fencing_token, 11)
+            .unwrap();
+        store
+            .checkpoint(
+                &id,
+                &lost.claim_id,
+                "a",
+                lost.fencing_token,
+                serde_json::json!({"step": "reserve", "status": "done"}),
+                11,
+            )
+            .unwrap();
+        store.recover_expired(13).unwrap();
+        let next = store.claim(&id, "b", &caps, 13, 10_000).unwrap();
+        let context = store
+            .execution_context(&id, &next.claim_id, "b", next.fencing_token, 13, vec![])
+            .unwrap();
+        assert_eq!(context.input, serde_json::json!({"id": 1}));
+        assert_eq!(context.checkpoints.len(), 1);
+        assert_eq!(context.checkpoints[0].state["step"], "reserve");
+        assert_ne!(
+            context.checkpoints[0].attempt_id, context.attempt_id,
+            "written by the lost attempt"
+        );
+        // The lost worker's fence reads nothing.
+        assert_eq!(
+            store
+                .execution_context(&id, &lost.claim_id, "a", lost.fencing_token, 13, vec![])
+                .unwrap_err()
+                .code,
+            "WORKLOAD_STALE_FENCE"
+        );
+        let _ = fs::remove_file(path);
+    }
+    #[test]
     fn cancellation_race_and_duplicate_completion_are_deterministic() {
         let (path, mut store) = store("cancel");
         let w = store.create(create()).unwrap();
@@ -2251,4 +3547,676 @@ mod tests {
         );
         let _ = fs::remove_file(path);
     }
+
+    // ---- PR 2: authoritative reads, canonical admission, failure reports, deadlines ----
+
+    fn version_of(store: &WorkloadStore, id: &str) -> u64 {
+        let db = store.db_for_tests();
+        let w = store.get(id).unwrap();
+        db.live_row(DURABLE_WORK_COLLECTION, &work_key(w))
+            .and_then(|row| row.value.get("__version").and_then(Value::as_u64))
+            .unwrap()
+    }
+
+    #[test]
+    fn another_writers_commits_are_visible_and_acted_on_without_restart() {
+        let (path, mut a) = store("two-writers");
+        let mut b = WorkloadStore::from_db(a.db_for_tests()).unwrap();
+        let id = a.create(create()).unwrap().workload_id;
+        // B never saw the admission, yet reads it from the store.
+        assert!(b.get(&id).is_err(), "the projection alone is stale");
+        assert_eq!(b.fresh(&id).unwrap().state, WorkloadState::Ready);
+        assert_eq!(b.fresh_list("t", "app").unwrap().len(), 1);
+        // A claims; B's claim decides on the committed claim, not its own view.
+        let claim = a
+            .claim(&id, "worker-a", &capabilities(), 11, 10_000)
+            .unwrap();
+        assert_eq!(
+            b.claim(&id, "worker-b", &capabilities(), 11, 10_000)
+                .unwrap_err()
+                .code,
+            "WORKLOAD_NOT_READY"
+        );
+        // A transition committed through A is what B fences against.
+        b.start(&id, &claim.claim_id, "worker-a", claim.fencing_token, 12)
+            .unwrap();
+        assert_eq!(a.fresh(&id).unwrap().state, WorkloadState::Running);
+        // B replays an admission A committed, by key, from the store.
+        let mut c = WorkloadStore::from_db(a.db_for_tests()).unwrap();
+        c.workloads.clear();
+        c.idempotency.clear();
+        c.document_versions.clear();
+        let (replayed, created) = c.admit(create()).unwrap();
+        assert!(!created);
+        assert_eq!(replayed.workload_id, id);
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn a_replay_matches_only_an_equivalent_request_and_names_every_conflicting_field() {
+        let (path, mut store) = store("canonical-admission");
+        let mut original = create();
+        original.input = serde_json::json!({"a": 1, "b": 2});
+        original.dependencies = vec![
+            WorkloadDependency {
+                workload_id: "w1".into(),
+                required_state: WorkloadState::Succeeded,
+            },
+            WorkloadDependency {
+                workload_id: "w2".into(),
+                required_state: WorkloadState::Succeeded,
+            },
+        ];
+        let first = store.create(original.clone()).unwrap();
+
+        // Equivalent after normalization: member order, dependency order, the
+        // defaults written out, and provenance that does not change what runs.
+        let mut equivalent = original.clone();
+        equivalent.input = serde_json::json!({"b": 2, "a": 1});
+        equivalent.dependencies.reverse();
+        equivalent.retry_policy = WorkloadRetryPolicy::default();
+        equivalent.timeouts = WorkloadTimeouts::default();
+        equivalent.priority = 0;
+        equivalent.trigger = "retried-by-client".into();
+        equivalent.session_id = Some("another-session".into());
+        equivalent.created_at = 99;
+        let (replayed, created) = store.admit(equivalent).unwrap();
+        assert!(!created);
+        assert_eq!(replayed.workload_id, first.workload_id);
+
+        let mutations: Vec<(&str, Box<dyn Fn(&mut CreateWorkload)>)> = vec![
+            (
+                "definition_id",
+                Box::new(|r| r.definition_id = "other".into()),
+            ),
+            (
+                "definition_revision",
+                Box::new(|r| r.definition_revision = "r2".into()),
+            ),
+            (
+                "environment",
+                Box::new(|r| r.environment = "staging".into()),
+            ),
+            (
+                "input",
+                Box::new(|r| r.input = serde_json::json!({"a": 1, "b": 3})),
+            ),
+            (
+                "capability_snapshot",
+                Box::new(|r| {
+                    r.capability_snapshot.insert("sms:send".into());
+                }),
+            ),
+            (
+                "connection_scopes",
+                Box::new(|r| {
+                    r.connection_scopes.insert("crm".into());
+                }),
+            ),
+            ("priority", Box::new(|r| r.priority = 5)),
+            (
+                "retry_policy",
+                Box::new(|r| r.retry_policy.max_attempts = 9),
+            ),
+            (
+                "retry_policy",
+                Box::new(|r| {
+                    r.retry_policy.retry_on.remove("worker_failure");
+                }),
+            ),
+            (
+                "timeouts",
+                Box::new(|r| r.timeouts.execution_ms = Some(60_000)),
+            ),
+            (
+                "dependencies",
+                Box::new(|r| r.dependencies.pop().map(|_| ()).unwrap()),
+            ),
+            ("not_before", Box::new(|r| r.not_before = Some(5_000))),
+        ];
+        for (field, mutate) in mutations {
+            let mut request = original.clone();
+            mutate(&mut request);
+            let error = store.admit(request).unwrap_err();
+            assert_eq!(error.code, "IDEMPOTENCY_KEY_REUSED", "{field}");
+            assert_eq!(
+                error.details.unwrap()["fields"],
+                serde_json::json!([field]),
+                "{field}"
+            );
+        }
+        // Several at once are all named.
+        let mut request = original.clone();
+        request.priority = 1;
+        request.environment = "staging".into();
+        let fields = store.admit(request).unwrap_err().details.unwrap()["fields"].clone();
+        assert_eq!(fields, serde_json::json!(["environment", "priority"]));
+
+        // After a restart: the same answers from the committed record.
+        drop(store);
+        let mut reopened = WorkloadStore::load(&path).unwrap();
+        assert!(!reopened.admit(original.clone()).unwrap().1);
+        let mut changed = original.clone();
+        changed.retry_policy.initial_delay_ms = 5;
+        assert_eq!(
+            reopened.admit(changed).unwrap_err().code,
+            "IDEMPOTENCY_KEY_REUSED"
+        );
+        assert_eq!(reopened.workloads.len(), 1, "nothing was created");
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn concurrent_conflicting_admissions_admit_one_and_refuse_the_other() {
+        for round in 0..10 {
+            let path = std::env::temp_dir().join(format!(
+                "workload-admission-race-{round}-{}.log",
+                std::process::id()
+            ));
+            let _ = fs::remove_file(&path);
+            let db = FeltDb::open(&path).unwrap();
+            let stores = [
+                WorkloadStore::from_db(db.clone()).unwrap(),
+                WorkloadStore::from_db(db.clone()).unwrap(),
+            ];
+            let barrier = std::sync::Barrier::new(2);
+            let outcomes = std::thread::scope(|scope| {
+                stores
+                    .into_iter()
+                    .enumerate()
+                    .map(|(i, mut store)| {
+                        let barrier = &barrier;
+                        scope.spawn(move || {
+                            let mut request = create();
+                            request.priority = i as i32;
+                            barrier.wait();
+                            store
+                                .admit(request)
+                                .map(|(w, created)| (w.priority, created))
+                        })
+                    })
+                    .collect::<Vec<_>>()
+                    .into_iter()
+                    .map(|handle| handle.join().unwrap())
+                    .collect::<Vec<_>>()
+            });
+            let admitted = outcomes
+                .iter()
+                .filter(|o| matches!(o, Ok((_, true))))
+                .count();
+            let refused = outcomes
+                .iter()
+                .filter(|o| {
+                    o.as_ref()
+                        .is_err_and(|e| e.code == "IDEMPOTENCY_KEY_REUSED")
+                })
+                .count();
+            assert_eq!((admitted, refused), (1, 1), "round {round}: {outcomes:?}");
+            let _ = fs::remove_file(path);
+        }
+    }
+
+    fn claimed(
+        name: &str,
+        policy: impl FnOnce(&mut CreateWorkload),
+    ) -> (PathBuf, WorkloadStore, String, WorkloadClaim) {
+        let (path, mut store) = store(name);
+        let mut request = create();
+        policy(&mut request);
+        let id = store.create(request).unwrap().workload_id;
+        let claim = store.claim(&id, "a", &capabilities(), 11, 10_000).unwrap();
+        (path, store, id, claim)
+    }
+
+    #[test]
+    fn a_worker_reports_failure_from_claimed_fenced_and_by_policy() {
+        // Could not start: a known failure before any effect, retried by kind.
+        let (path, mut store, id, claim) = claimed("fail-claimed", |_| {});
+        let state = store
+            .fail_with_effect(
+                &id,
+                &claim.claim_id,
+                "a",
+                claim.fencing_token,
+                "transient_connection",
+                "no route",
+                None,
+                12,
+            )
+            .unwrap();
+        assert_eq!(state, WorkloadState::Ready);
+        let w = store.get(&id).unwrap();
+        let failed = w
+            .execution_history
+            .iter()
+            .find(|e| e.to == WorkloadState::Failed)
+            .unwrap();
+        assert_eq!(
+            failed.evidence["effect"], "none",
+            "never started, so no effect"
+        );
+        assert_eq!(w.attempts[0].status, AttemptStatus::Failed);
+        // The identical report again (lost response) replays and commits nothing.
+        let version = version_of(&store, &id);
+        assert_eq!(
+            store
+                .fail_with_effect(
+                    &id,
+                    &claim.claim_id,
+                    "a",
+                    claim.fencing_token,
+                    "transient_connection",
+                    "no route",
+                    None,
+                    13
+                )
+                .unwrap(),
+            WorkloadState::Ready
+        );
+        assert_eq!(version_of(&store, &id), version);
+        // A different report from the same attempt is answered by its fence.
+        assert_eq!(
+            store
+                .fail_with_effect(
+                    &id,
+                    &claim.claim_id,
+                    "a",
+                    claim.fencing_token,
+                    "timeout",
+                    "other",
+                    None,
+                    13
+                )
+                .unwrap_err()
+                .code,
+            "WORKLOAD_CLAIM_INVALID"
+        );
+        // So is a completion: the failure stands.
+        assert_eq!(
+            store
+                .complete(
+                    &id,
+                    &claim.claim_id,
+                    "a",
+                    claim.fencing_token,
+                    "r",
+                    Value::Null,
+                    vec![],
+                    1,
+                    13
+                )
+                .unwrap_err()
+                .code,
+            "WORKLOAD_CLAIM_INVALID"
+        );
+        // Another worker's claim fences the old attempt out.
+        let next = store.claim(&id, "b", &capabilities(), 14, 10_000).unwrap();
+        assert_eq!(
+            store
+                .fail_with_effect(
+                    &id,
+                    &claim.claim_id,
+                    "a",
+                    claim.fencing_token,
+                    "timeout",
+                    "late",
+                    None,
+                    15
+                )
+                .unwrap_err()
+                .code,
+            "WORKLOAD_STALE_FENCE"
+        );
+        assert_eq!(
+            store
+                .get(&id)
+                .unwrap()
+                .current_claim
+                .as_ref()
+                .unwrap()
+                .claim_id,
+            next.claim_id
+        );
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn failure_policy_decides_retry_and_an_unknown_effect_is_decided_like_a_lost_worker() {
+        // A kind the policy does not retry dead-letters.
+        let (path, mut store, id, claim) = claimed("fail-kind", |_| {});
+        store
+            .start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
+            .unwrap();
+        assert_eq!(
+            store
+                .fail_with_effect(
+                    &id,
+                    &claim.claim_id,
+                    "a",
+                    claim.fencing_token,
+                    "validation",
+                    "bad input",
+                    Some("none"),
+                    13
+                )
+                .unwrap(),
+            WorkloadState::DeadLetter
+        );
+        assert_eq!(
+            store
+                .get(&id)
+                .unwrap()
+                .execution_history
+                .last()
+                .unwrap()
+                .evidence["decision"],
+            "failure_kind_not_retryable"
+        );
+        let _ = fs::remove_file(path);
+
+        // A retryable kind with an unknown effect, under a policy that does not
+        // re-run lost work: not retried automatically.
+        let (path, mut store, id, claim) = claimed("fail-unknown", |r| {
+            r.retry_policy.retry_on = ["timeout".into(), "transient_connection".into()].into();
+        });
+        store
+            .start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
+            .unwrap();
+        assert_eq!(
+            store
+                .fail_with_effect(
+                    &id,
+                    &claim.claim_id,
+                    "a",
+                    claim.fencing_token,
+                    "timeout",
+                    "charge sent, no answer",
+                    Some("unknown"),
+                    13
+                )
+                .unwrap(),
+            WorkloadState::DeadLetter
+        );
+        let w = store.get(&id).unwrap();
+        assert_eq!(
+            w.execution_history.last().unwrap().evidence["decision"],
+            "effect_unknown_not_retryable"
+        );
+        // An operator can still re-arm it, with a reason.
+        store
+            .retry_with_reason(&id, "operator", 14, Some("verified: not charged"))
+            .unwrap();
+        assert_eq!(store.get(&id).unwrap().state, WorkloadState::Ready);
+        let _ = fs::remove_file(path);
+
+        // The same report under a policy that re-runs lost work is retried.
+        let (path, mut store, id, claim) = claimed("fail-unknown-retry", |_| {});
+        store
+            .start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
+            .unwrap();
+        assert_eq!(
+            store
+                .fail_with_effect(
+                    &id,
+                    &claim.claim_id,
+                    "a",
+                    claim.fencing_token,
+                    "validation",
+                    "x",
+                    Some("unknown"),
+                    13
+                )
+                .unwrap(),
+            WorkloadState::Ready
+        );
+        // An invalid effect is refused before anything changes.
+        assert_eq!(
+            store
+                .fail_with_effect(&id, "c", "a", 9, "x", "y", Some("maybe"), 14)
+                .unwrap_err()
+                .code,
+            "WORKLOAD_INVALID"
+        );
+        let _ = fs::remove_file(path);
+
+        // The attempt budget bounds retries.
+        let (path, mut store, id, mut claim) =
+            claimed("fail-budget", |r| r.retry_policy.max_attempts = 2);
+        assert_eq!(
+            store
+                .fail(
+                    &id,
+                    &claim.claim_id,
+                    "a",
+                    claim.fencing_token,
+                    "timeout",
+                    "x",
+                    12
+                )
+                .unwrap(),
+            WorkloadState::Ready
+        );
+        let next = store.get(&id).unwrap().next_attempt_at.unwrap();
+        claim = store
+            .claim(&id, "a", &capabilities(), next, 10_000)
+            .unwrap();
+        assert_eq!(
+            store
+                .fail(
+                    &id,
+                    &claim.claim_id,
+                    "a",
+                    claim.fencing_token,
+                    "timeout",
+                    "x",
+                    next
+                )
+                .unwrap(),
+            WorkloadState::DeadLetter
+        );
+        assert_eq!(
+            store
+                .get(&id)
+                .unwrap()
+                .execution_history
+                .last()
+                .unwrap()
+                .evidence["decision"],
+            "attempt_budget_exhausted"
+        );
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn failure_racing_completion_commits_exactly_one_outcome() {
+        for completion_first in [true, false] {
+            let (path, mut store, id, claim) = claimed(&format!("race-{completion_first}"), |_| {});
+            store
+                .start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
+                .unwrap();
+            let complete = |s: &mut WorkloadStore| {
+                s.complete(
+                    &id,
+                    &claim.claim_id,
+                    "a",
+                    claim.fencing_token,
+                    "r",
+                    Value::Null,
+                    vec![],
+                    1,
+                    13,
+                )
+                .map(|_| ())
+            };
+            let fail = |s: &mut WorkloadStore| {
+                s.fail(
+                    &id,
+                    &claim.claim_id,
+                    "a",
+                    claim.fencing_token,
+                    "timeout",
+                    "x",
+                    13,
+                )
+                .map(|_| ())
+            };
+            let (first, second) = if completion_first {
+                (complete(&mut store), fail(&mut store))
+            } else {
+                (fail(&mut store), complete(&mut store))
+            };
+            assert!(first.is_ok());
+            assert_eq!(second.unwrap_err().code, "WORKLOAD_CLAIM_INVALID");
+            let w = store.get(&id).unwrap();
+            assert_eq!(
+                w.state,
+                if completion_first {
+                    WorkloadState::Succeeded
+                } else {
+                    WorkloadState::Ready
+                }
+            );
+            assert_eq!(w.attempts.len(), 1);
+            let _ = fs::remove_file(path);
+        }
+    }
+
+    #[test]
+    fn deadlines_are_enforced_from_committed_timestamps_and_race_safely() {
+        // execution: a heartbeat inside the deadline keeps the attempt; after it,
+        // every worker call is refused and the attempt is timed out, once.
+        let (path, mut store, id, claim) =
+            claimed("deadline-exec", |r| r.timeouts.execution_ms = Some(5_000));
+        store
+            .start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
+            .unwrap();
+        store
+            .heartbeat(&id, &claim.claim_id, "a", claim.fencing_token, 16, 10_000)
+            .unwrap();
+        assert!(store.enforce_deadlines(16).unwrap().is_empty());
+        let refused = store
+            .complete(
+                &id,
+                &claim.claim_id,
+                "a",
+                claim.fencing_token,
+                "late",
+                Value::Null,
+                vec![],
+                1,
+                17,
+            )
+            .unwrap_err();
+        assert_eq!(refused.code, "WORKLOAD_TIMED_OUT");
+        assert_eq!(refused.details.unwrap()["deadline"], "execution");
+        let w = store.get(&id).unwrap().clone();
+        assert_eq!(
+            w.state,
+            WorkloadState::Ready,
+            "timeout is retryable by default"
+        );
+        assert_eq!(w.attempts[0].status, AttemptStatus::TimedOut);
+        assert!(w.result.is_none(), "a late completion never lands");
+        assert!(
+            store.enforce_deadlines(30).unwrap().is_empty(),
+            "timed out once"
+        );
+        let _ = fs::remove_file(path);
+
+        // heartbeat: the gap since the last heartbeat; a heartbeat that commits
+        // first moves the deadline, the recovery pass after it changes nothing.
+        let (path, mut store, id, claim) = claimed("deadline-heartbeat", |r| {
+            r.timeouts.heartbeat_ms = Some(3_000)
+        });
+        store
+            .start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
+            .unwrap();
+        store
+            .heartbeat(&id, &claim.claim_id, "a", claim.fencing_token, 13, 10_000)
+            .unwrap();
+        assert!(store.enforce_deadlines(15).unwrap().is_empty());
+        assert_eq!(store.enforce_deadlines(16).unwrap(), vec![id.clone()]);
+        assert_eq!(
+            store
+                .heartbeat(&id, &claim.claim_id, "a", claim.fencing_token, 16, 10_000)
+                .unwrap_err()
+                .code,
+            "WORKLOAD_CLAIM_INVALID",
+            "the timed-out claim is gone"
+        );
+        let _ = fs::remove_file(path);
+
+        // claim: claimed but never started; not retried when the policy says so.
+        let (path, mut store, id, claim) = claimed("deadline-claim", |r| {
+            r.timeouts.claim_ms = Some(2_000);
+            r.retry_policy.retry_on = ["worker_failure".into()].into();
+        });
+        let refused = store
+            .start(&id, &claim.claim_id, "a", claim.fencing_token, 13)
+            .unwrap_err();
+        assert_eq!(refused.code, "WORKLOAD_TIMED_OUT");
+        let w = store.get(&id).unwrap();
+        assert_eq!(w.state, WorkloadState::DeadLetter);
+        assert_eq!(
+            w.execution_history.last().unwrap().evidence["decision"],
+            "timeout_not_retryable"
+        );
+        let _ = fs::remove_file(path);
+
+        // queue and overall bound the work: dead-lettered, never retried, and a
+        // queued workload's previous attempt is not rewritten.
+        let (path, mut store) = super::tests::store("deadline-queue");
+        let mut request = create();
+        request.timeouts.queue_ms = Some(4_000);
+        let id = store.create(request).unwrap().workload_id;
+        assert_eq!(
+            store
+                .claim(&id, "a", &capabilities(), 14, 10_000)
+                .unwrap_err()
+                .code,
+            "WORKLOAD_TIMED_OUT"
+        );
+        let w = store.get(&id).unwrap();
+        assert_eq!(w.state, WorkloadState::DeadLetter);
+        assert_eq!(
+            w.execution_history.last().unwrap().evidence["decision"],
+            "deadline_not_retryable"
+        );
+        assert!(w.attempts.is_empty());
+        let _ = fs::remove_file(path);
+
+        let (path, mut store, id, claim) =
+            claimed("deadline-overall", |r| r.timeouts.overall_ms = Some(10_000));
+        store
+            .start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
+            .unwrap();
+        // Restart: the deadline comes from the committed record.
+        drop(store);
+        let mut store = WorkloadStore::load(&path).unwrap();
+        assert!(store.enforce_deadlines(19).unwrap().is_empty());
+        assert_eq!(store.enforce_deadlines(20).unwrap(), vec![id.clone()]);
+        assert_eq!(store.get(&id).unwrap().state, WorkloadState::DeadLetter);
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn deadlines_are_versioned_and_zero_is_refused() {
+        let (path, mut store) = store("deadline-version");
+        let mut request = create();
+        request.timeouts.queue_ms = Some(0);
+        assert_eq!(store.create(request).unwrap_err().code, "WORKLOAD_INVALID");
+        // A record admitted before deadlines were enforced keeps its contract.
+        let mut request = create();
+        request.timeouts.queue_ms = Some(1_000);
+        let id = store.create(request).unwrap().workload_id;
+        assert_eq!(
+            store.get(&id).unwrap().contract_version,
+            DEADLINE_CONTRACT_VERSION
+        );
+        let mut legacy = store.get(&id).unwrap().clone();
+        legacy.contract_version = 1;
+        assert_eq!(deadline_breach(&legacy, 1_000_000), None);
+        assert!(deadline_breach(store.get(&id).unwrap(), 1_000_000).is_some());
+        let _ = fs::remove_file(path);
+    }
 }