@feltdb/core 0.12.2 → 0.13.0

package/dist/create/server-source/crates/feltdb/src/workload.rs

@@ -329,16 +329,9 @@ impl WorkloadStore {
             document_versions.insert(workload.workload_id.clone(), version);
             workloads.insert(workload.workload_id.clone(), workload);
         }
-        let mut idempotency = BTreeMap::new();
+        let idempotency = idempotency_index(&workloads, idempotency_scope);
         let mut result_ids = BTreeMap::new();
         for workload in workloads.values() {
-            idempotency.insert(
-                format!(
-                    "{}:{}:{}",
-                    workload.tenant_id, workload.application_id, workload.idempotency_key
-                ),
-                workload.workload_id.clone(),
-            );
             if let Some(result) = &workload.result {
                 result_ids.insert(result.result_id.clone(), workload.workload_id.clone());
             }
@@ -382,7 +375,7 @@ impl WorkloadStore {
         }
         // The maps are verification evidence, not migrated authorities.
         let rebuilt = Self::from_db(store.db.clone())?;
-        if legacy.idempotency != rebuilt.idempotency
+        if legacy.idempotency != idempotency_index(&rebuilt.workloads, legacy_idempotency_scope)
             || legacy.result_ids != rebuilt.result_ids
             || legacy.event_sequence != derived_event_sequence(&rebuilt.workloads)
         {
@@ -454,6 +447,21 @@ impl WorkloadStore {
         Ok(())
     }
     pub fn create(&mut self, input: CreateWorkload) -> Result<Workload, WorkloadError> {
+        self.admit(input).map(|(workload, _)| workload)
+    }
+
+    /// Create a workload, or return the one this requester already created
+    /// under the same key.
+    ///
+    /// The key is scoped to the tenant, the application **and the requester**,
+    /// so one requester's key can never return, reveal or block another's
+    /// workload. A repeated key is answered with the original workload only when
+    /// the request is the same — definition, revision, environment, input and
+    /// capability snapshot; otherwise it is refused with
+    /// `IDEMPOTENCY_KEY_REUSED` and nothing is created. The requester's
+    /// authorization snapshot is checked before any replay. The boolean is
+    /// `true` when this call created the workload.
+    pub fn admit(&mut self, input: CreateWorkload) -> Result<(Workload, bool), WorkloadError> {
         ensure_safe(&input.input)?;
         if input.idempotency_key.trim().is_empty() {
             return Err(WorkloadError::new(
@@ -461,13 +469,6 @@ impl WorkloadStore {
                 "idempotency key is required",
             ));
         }
-        let idempotency = format!(
-            "{}:{}:{}",
-            input.tenant_id, input.application_id, input.idempotency_key
-        );
-        if let Some(id) = self.idempotency.get(&idempotency) {
-            return Ok(self.workloads[id].clone());
-        }
         if input.authorization_snapshot.grant.subject != input.requested_by
             || input.authorization_snapshot.grant.tenant_id != input.tenant_id
             || input.authorization_snapshot.grant.application_id != input.application_id
@@ -477,11 +478,22 @@ impl WorkloadStore {
                 "authorization snapshot does not bind the requester",
             ));
         }
+        let idempotency = idempotency_scope(
+            &input.tenant_id,
+            &input.application_id,
+            &input.requested_by,
+            &input.idempotency_key,
+        );
+        if let Some(id) = self.idempotency.get(&idempotency) {
+            let existing = self.workloads[id].clone();
+            return replay(existing, &input);
+        }
         // The scoped idempotency identity is the record identity. Competing
         // admissions therefore contend on one canonical require-absent CAS
         // even when their payloads differ; no separate durable uniqueness
         // index is needed.
         let id = format!("workload_{}", &hex(&Sha256::digest(&idempotency))[..24]);
+        let request = input.clone();
         let input_hash = hash(&input.input);
         let mut workload = Workload {
             contract_version: 1,
@@ -535,14 +547,23 @@ impl WorkloadStore {
             input.created_at,
             evidence,
         )?;
+        let key = work_key(&workload);
         self.idempotency.insert(idempotency.clone(), id.clone());
         self.workloads.insert(id.clone(), workload.clone());
         if let Err(error) = self.persist_workload(&id, "workload.created") {
             self.workloads.remove(&id);
             self.idempotency.remove(&idempotency);
+            // Another writer of the same store (a second process, or a replica)
+            // admitted this identity first. Converge on what it committed.
+            if error.code == "WORKLOAD_CONFLICT" {
+                if let Some(existing) = self.reload_committed(&key)? {
+                    self.idempotency.insert(idempotency, id);
+                    return replay(existing, &request);
+                }
+            }
             return Err(error);
         }
-        Ok(workload)
+        Ok((workload, true))
     }
     pub fn get(&self, id: &str) -> Result<&Workload, WorkloadError> {
         self.workloads
@@ -573,11 +594,47 @@ impl WorkloadStore {
         let result = action(self, &mut edited)?;
         self.workloads.insert(id.into(), edited);
         if let Err(error) = self.persist_workload(id, "workload.mutated") {
-            self.workloads.insert(id.into(), workload);
+            self.workloads.insert(id.into(), workload.clone());
+            // Another writer of the same store committed this workload first.
+            // Nothing of this edit was committed; adopt what was, so this
+            // projection does not keep acting on a version that no longer
+            // exists (every later guarded commit from it would conflict).
+            if error.code == "WORKLOAD_CONFLICT" {
+                self.reload_committed(&work_key(&workload))?;
+            }
             return Err(error);
         }
         Ok(result)
     }
+    #[cfg(test)]
+    pub(crate) fn db_for_tests(&self) -> FeltDb {
+        self.db.clone()
+    }
+    /// Replace this projection's copy of a workload with the committed row.
+    fn reload_committed(&mut self, key: &str) -> Result<Option<Workload>, WorkloadError> {
+        let committed = self
+            .db
+            .query_collection(DURABLE_WORK_COLLECTION, Some(1), |row| row.key == key)
+            .map_err(io)?;
+        let Some(row) = committed.into_iter().next() else {
+            return Ok(None);
+        };
+        let version = row
+            .value
+            .get("__version")
+            .and_then(Value::as_u64)
+            .unwrap_or(1);
+        let workload: Workload = serde_json::from_value(row.value).map_err(io)?;
+        if let Some(result) = &workload.result {
+            self.result_ids
+                .insert(result.result_id.clone(), workload.workload_id.clone());
+        }
+        self.document_versions
+            .insert(workload.workload_id.clone(), version);
+        self.workloads
+            .insert(workload.workload_id.clone(), workload.clone());
+        Ok(Some(workload))
+    }
     fn transition(
         &mut self,
         w: &mut Workload,
@@ -618,6 +675,27 @@ impl WorkloadStore {
         now: i64,
         lease_ms: u64,
     ) -> Result<WorkloadClaim, WorkloadError> {
+        // A Ready workload that has already used its budget (an operator
+        // re-arm beyond it, or a record left by unbounded recovery before it
+        // was bounded) is dead-lettered here rather than claimed again.
+        let exhausted = self
+            .get(id)
+            .is_ok_and(|w| w.state == WorkloadState::Ready && w.attempt_count >= attempt_budget(w));
+        if exhausted {
+            self.edit(id, |store, w| {
+                let evidence = [
+                    ("reason".into(), "attempt_budget_exhausted".into()),
+                    ("attempts_used".into(), w.attempt_count.to_string()),
+                    ("attempt_budget".into(), attempt_budget(w).to_string()),
+                ]
+                .into();
+                store.transition(w, WorkloadState::DeadLetter, "rust.claim", now, evidence)
+            })?;
+            return Err(WorkloadError::new(
+                "WORKLOAD_ATTEMPTS_EXHAUSTED",
+                "workload has used its attempt budget",
+            ));
+        }
         self.edit(id, |store, w| {
             if w.state != WorkloadState::Ready {
                 return Err(WorkloadError::new(
@@ -912,8 +990,8 @@ impl WorkloadStore {
                 now,
                 [("failure_kind".into(), kind.into())].into(),
             )?;
-            let retryable = w.retry_policy.retry_on.contains(kind)
-                && w.attempt_count < w.retry_policy.max_attempts;
+            let retryable =
+                w.retry_policy.retry_on.contains(kind) && w.attempt_count < attempt_budget(w);
             if retryable {
                 w.retry_count += 1;
                 let multiplier = if w.retry_policy.backoff == BackoffPolicy::Exponential {
@@ -1024,8 +1102,7 @@ impl WorkloadStore {
             }
             w.current_claim = None;
             store.transition(w, WorkloadState::TimedOut, actor, now, BTreeMap::new())?;
-            if w.attempt_count < w.retry_policy.max_attempts
-                && w.retry_policy.retry_on.contains("timeout")
+            if w.attempt_count < attempt_budget(w) && w.retry_policy.retry_on.contains("timeout")
             {
                 w.retry_count += 1;
                 w.next_attempt_at = Some(now + (w.retry_policy.initial_delay_ms / 1000) as i64);
@@ -1043,6 +1120,21 @@ impl WorkloadStore {
         })
     }
     pub fn retry(&mut self, id: &str, actor: &str, now: i64) -> Result<(), WorkloadError> {
+        self.retry_with_reason(id, actor, now, None)
+    }
+    /// An operator's re-arm of failed, dead-lettered or timed-out work.
+    ///
+    /// Each re-arm grants one more attempt, as it always has, and is recorded
+    /// in the history with its actor and reason. A workload may be re-armed at
+    /// most `retry_policy.max_attempts` times, so operator retries cannot
+    /// become an unbounded loop.
+    pub fn retry_with_reason(
+        &mut self,
+        id: &str,
+        actor: &str,
+        now: i64,
+        reason: Option<&str>,
+    ) -> Result<(), WorkloadError> {
         self.edit(id, |store, w| {
             if !matches!(
                 w.state,
@@ -1053,9 +1145,31 @@ impl WorkloadStore {
                     "workload is not retryable",
                 ));
             }
+            let previous = operator_retries(w);
+            if previous >= w.retry_policy.max_attempts {
+                return Err(WorkloadError::new(
+                    "WORKLOAD_RETRY_EXHAUSTED",
+                    "workload has used every operator retry its policy allows",
+                ));
+            }
             w.next_attempt_at = Some(now);
             w.current_claim = None;
-            store.transition(w, WorkloadState::Ready, actor, now, BTreeMap::new())
+            let mut evidence: BTreeMap<String, String> = [
+                ("retry".into(), "operator".into()),
+                ("operator_retry".into(), (previous + 1).to_string()),
+                (
+                    "attempt_budget".into(),
+                    w.retry_policy
+                        .max_attempts
+                        .saturating_add(previous + 1)
+                        .to_string(),
+                ),
+            ]
+            .into();
+            if let Some(reason) = reason.filter(|reason| !reason.trim().is_empty()) {
+                evidence.insert("reason".into(), reason.chars().take(256).collect());
+            }
+            store.transition(w, WorkloadState::Ready, actor, now, evidence)
         })
     }
     pub fn recover_expired(&mut self, now: i64) -> Result<Vec<String>, WorkloadError> {
@@ -1076,41 +1190,71 @@ impl WorkloadStore {
             })
             .map(|w| w.workload_id.clone())
             .collect::<Vec<_>>();
-        for id in &ids {
-            self.edit(id, |store, w| {
+        let mut recovered = Vec::with_capacity(ids.len());
+        for id in ids {
+            let outcome = self.edit(&id, |store, w| {
+                let lost = w.current_claim.take();
                 if let Some(attempt) = w.attempts.last_mut() {
                     attempt.status = AttemptStatus::WorkerLost;
                     attempt.ended_at = Some(now);
+                    attempt.error = Some("worker_lost: claim lease expired".into());
+                }
+                let mut evidence: BTreeMap<String, String> = [
+                    ("recovery_reason".into(), "lease_expired".into()),
+                    ("attempts_used".into(), w.attempt_count.to_string()),
+                    ("attempt_budget".into(), attempt_budget(w).to_string()),
+                ]
+                .into();
+                if let Some(claim) = &lost {
+                    evidence.insert("claim_id".into(), claim.claim_id.clone());
+                    evidence.insert("worker_id".into(), claim.worker_id.clone());
+                    evidence.insert("lease_expired_at".into(), claim.expires_at.to_string());
+                }
+                if let Some(attempt) = w.attempts.last() {
+                    evidence.insert("attempt_id".into(), attempt.attempt_id.clone());
                 }
-                w.current_claim = None;
                 if w.state == WorkloadState::CancellationRequested {
-                    store.transition(
-                        w,
-                        WorkloadState::Cancelled,
-                        "rust.recovery",
-                        now,
-                        BTreeMap::new(),
-                    )?;
+                    store.transition(w, WorkloadState::Cancelled, "rust.recovery", now, evidence)?;
                 } else {
                     store.transition(
                         w,
                         WorkloadState::Recovering,
                         "rust.recovery",
                         now,
-                        BTreeMap::new(),
+                        evidence,
                     )?;
+                    // A lost worker is the policy's `worker_failure`. Whether
+                    // the lost attempt had effects is unknown to FeltDB, so a
+                    // retry re-executes it: at-least-once, never exactly-once.
+                    let (to, decision) = if !w.retry_policy.retry_on.contains("worker_failure") {
+                        (WorkloadState::DeadLetter, "worker_failure_not_retryable")
+                    } else if w.attempt_count >= attempt_budget(w) {
+                        (WorkloadState::DeadLetter, "attempt_budget_exhausted")
+                    } else {
+                        (WorkloadState::Ready, "retry")
+                    };
                     store.transition(
                         w,
-                        WorkloadState::Ready,
+                        to,
                         "rust.recovery",
                         now,
-                        BTreeMap::new(),
+                        [("decision".into(), decision.into())].into(),
                     )?;
                 }
                 Ok(())
-            })?;
+            });
+            match outcome {
+                Ok(()) => recovered.push(id),
+                // Another writer of this store committed the workload first —
+                // in a race, its own recovery. Nothing of this attempt was
+                // committed and `edit` has adopted the committed workload, so
+                // the attempt is recovered once, by that writer. If it still
+                // needs recovery, the next pass sees it.
+                Err(error) if error.code == "WORKLOAD_CONFLICT" => {}
+                Err(error) => return Err(error),
+            }
         }
-        Ok(ids)
+        Ok(recovered)
     }
 }
 
@@ -1211,6 +1355,8 @@ fn legal(from: WorkloadState, to: WorkloadState) -> bool {
             | (Checkpointed, TimedOut)
             | (Ready, TimedOut)
             | (TimedOut, DeadLetter)
+            | (Recovering, DeadLetter)
+            | (Ready, DeadLetter)
     )
 }
 pub fn can_transition(from: WorkloadState, to: WorkloadState) -> bool {
@@ -1259,6 +1405,69 @@ fn ensure_safe(value: &Value) -> Result<(), WorkloadError> {
 fn sensitive_text(v: &str) -> bool {
     v.contains("BEGIN PRIVATE KEY") || v.starts_with("sk-") || v.starts_with("fdb_live_")
 }
+/// The identity a caller's idempotency key is unique within: tenant,
+/// application and requester. Encoded as a JSON array, so no choice of key or
+/// identifier can make two different scopes coincide.
+fn idempotency_scope(tenant: &str, application: &str, requester: &Subject, key: &str) -> String {
+    serde_json::json!([tenant, application, requester.kind(), requester.id(), key]).to_string()
+}
+/// The scope before keys were bound to their requester, kept only to verify
+/// legacy store files during migration.
+fn legacy_idempotency_scope(
+    tenant: &str,
+    application: &str,
+    _requester: &Subject,
+    key: &str,
+) -> String {
+    format!("{tenant}:{application}:{key}")
+}
+fn idempotency_index(
+    workloads: &BTreeMap<String, Workload>,
+    scope: fn(&str, &str, &Subject, &str) -> String,
+) -> BTreeMap<String, String> {
+    workloads
+        .values()
+        .map(|w| {
+            (
+                scope(&w.tenant_id, &w.application_id, &w.requested_by, &w.idempotency_key),
+                w.workload_id.clone(),
+            )
+        })
+        .collect()
+}
+/// Answer a repeated key: the original workload for the same request, a
+/// refusal for a different one. Never the original for a different request.
+fn replay(existing: Workload, input: &CreateWorkload) -> Result<(Workload, bool), WorkloadError> {
+    let same = existing.requested_by == input.requested_by
+        && existing.definition_id == input.definition_id
+        && existing.definition_revision == input.definition_revision
+        && existing.environment == input.environment
+        && existing.input_hash == hash(&input.input)
+        && existing.capability_snapshot == input.capability_snapshot;
+    if same {
+        Ok((existing, false))
+    } else {
+        Err(WorkloadError::new(
+            "IDEMPOTENCY_KEY_REUSED",
+            "idempotency key was already used for a different request",
+        ))
+    }
+}
+/// Operator re-arms recorded in the workload's own history.
+fn operator_retries(w: &Workload) -> u32 {
+    w.execution_history
+        .iter()
+        .filter(|event| event.evidence.get("retry").map(String::as_str) == Some("operator"))
+        .count() as u32
+}
+/// How many claims the workload may have in total: its policy's
+/// `max_attempts`, plus one for each operator re-arm (which is what an operator
+/// retry has always granted).
+fn attempt_budget(w: &Workload) -> u32 {
+    w.retry_policy
+        .max_attempts
+        .saturating_add(operator_retries(w))
+}
 fn work_key(workload: &Workload) -> String {
     format!(
         "{}/{}/{}",
@@ -1312,8 +1521,12 @@ mod tests {
     };
     use std::{fs, path::PathBuf};
     fn create() -> CreateWorkload {
+        create_as("a1")
+    }
+    /// The fixture request, made by agent `agent`.
+    fn create_as(agent: &str) -> CreateWorkload {
         let signer = GrantSigner::new("k", vec![1; 32]).unwrap();
-        let subject = Subject::Agent("a1".into());
+        let subject = Subject::Agent(agent.into());
         let grant = Grant {
             contract_version: 1,
             id: "g".into(),
@@ -1355,6 +1568,339 @@ mod tests {
             created_at: 10,
         }
     }
+    fn capabilities() -> BTreeSet<String> {
+        ["email:send".into()].into()
+    }
+
+    #[test]
+    fn idempotency_keys_are_bound_to_their_requester_and_request() {
+        let (path, mut store) = store("requester-scoped-keys");
+        let (original, created) = store.admit(create_as("a1")).unwrap();
+        assert!(created);
+
+        // The same requester and request: the original, not a second workload.
+        let (replayed, created) = store.admit(create_as("a1")).unwrap();
+        assert!(!created);
+        assert_eq!(replayed.workload_id, original.workload_id);
+
+        // The same key for a different request is refused and creates nothing.
+        let mut different = create_as("a1");
+        different.input = serde_json::json!({"id": 2});
+        assert_eq!(store.admit(different).unwrap_err().code, "IDEMPOTENCY_KEY_REUSED");
+        let mut other_definition = create_as("a1");
+        other_definition.definition_id = "other".into();
+        assert_eq!(
+            store.admit(other_definition).unwrap_err().code,
+            "IDEMPOTENCY_KEY_REUSED"
+        );
+
+        // Another requester's identical key is its own workload: it neither
+        // returns, reveals nor blocks the first requester's.
+        let (theirs, created) = store.admit(create_as("a2")).unwrap();
+        assert!(created);
+        assert_ne!(theirs.workload_id, original.workload_id);
+        assert_eq!(store.workloads.len(), 2);
+
+        // A snapshot that does not bind the requester is refused before any
+        // replay, so a key cannot be used to fetch a workload.
+        let mut unbound = create_as("a1");
+        unbound.requested_by = Subject::Agent("a2".into());
+        assert_eq!(store.admit(unbound).unwrap_err().code, "WORKLOAD_UNAUTHORIZED");
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn lease_recovery_records_the_lost_worker_and_is_bounded_by_the_attempt_budget() {
+        let (path, mut store) = store("bounded-recovery");
+        let workload = store.create(create()).unwrap();
+        let id = workload.workload_id.clone();
+        for attempt in 1..=3u32 {
+            let now = 100 * attempt as i64;
+            let claim = store.claim(&id, "worker-a", &capabilities(), now, 10_000).unwrap();
+            assert_eq!(store.recover_expired(claim.expires_at).unwrap(), vec![id.clone()]);
+            let recovered = store.get(&id).unwrap();
+            let lost = recovered.attempts.last().unwrap();
+            assert_eq!(lost.status, AttemptStatus::WorkerLost);
+            assert_eq!(lost.claim_id, claim.claim_id);
+            let recovering = recovered
+                .execution_history
+                .iter()
+                .rev()
+                .find(|event| event.to == WorkloadState::Recovering)
+                .unwrap();
+            assert_eq!(recovering.evidence["recovery_reason"], "lease_expired");
+            assert_eq!(recovering.evidence["worker_id"], "worker-a");
+            assert_eq!(recovering.evidence["claim_id"], claim.claim_id);
+            assert_eq!(recovering.evidence["attempts_used"], attempt.to_string());
+            let decision = &recovered.execution_history.last().unwrap();
+            if attempt < 3 {
+                assert_eq!(recovered.state, WorkloadState::Ready);
+                assert_eq!(decision.evidence["decision"], "retry");
+            } else {
+                assert_eq!(recovered.state, WorkloadState::DeadLetter);
+                assert_eq!(decision.evidence["decision"], "attempt_budget_exhausted");
+            }
+        }
+        assert_eq!(store.get(&id).unwrap().attempt_count, 3);
+        assert_eq!(
+            store.claim(&id, "worker-b", &capabilities(), 1_000, 10_000).unwrap_err().code,
+            "WORKLOAD_NOT_READY"
+        );
+        let _ = fs::remove_file(path);
+    }
+
+    /// The recovery history of `id` as committed, read by a fresh projection.
+    fn committed_workload(db: &FeltDb, id: &str) -> Workload {
+        WorkloadStore::from_db(db.clone()).unwrap().get(id).unwrap().clone()
+    }
+
+    fn recoveries(workload: &Workload) -> usize {
+        workload
+            .execution_history
+            .iter()
+            .filter(|event| event.to == WorkloadState::Recovering)
+            .count()
+    }
+
+    /// Two writers of one store recover the same expired workload at once.
+    fn race_recovery(db: &FeltDb, now: i64) -> (WorkloadStore, WorkloadStore, Vec<Vec<String>>) {
+        let stores = [
+            WorkloadStore::from_db(db.clone()).unwrap(),
+            WorkloadStore::from_db(db.clone()).unwrap(),
+        ];
+        let barrier = std::sync::Barrier::new(2);
+        let results = std::thread::scope(|scope| {
+            let handles = stores
+                .into_iter()
+                .map(|mut store| {
+                    let barrier = &barrier;
+                    scope.spawn(move || {
+                        barrier.wait();
+                        let recovered = store.recover_expired(now).unwrap();
+                        (store, recovered)
+                    })
+                })
+                .collect::<Vec<_>>();
+            handles
+                .into_iter()
+                .map(|handle| handle.join().unwrap())
+                .collect::<Vec<_>>()
+        });
+        let mut results = results.into_iter();
+        let (first, first_recovered) = results.next().unwrap();
+        let (second, second_recovered) = results.next().unwrap();
+        (first, second, vec![first_recovered, second_recovered])
+    }
+
+    #[test]
+    fn concurrent_recovery_commits_one_transition_and_the_loser_converges() {
+        for round in 0..20 {
+            let (path, mut store) = store(&format!("concurrent-recovery-{round}"));
+            let db = store.db.clone();
+            let id = store.create(create()).unwrap().workload_id;
+            let claim = store.claim(&id, "worker-a", &capabilities(), 100, 10_000).unwrap();
+            // A writer that loaded the workload before either recovery.
+            let mut stale = WorkloadStore::from_db(db.clone()).unwrap();
+
+            let (mut first, mut second, recovered) = race_recovery(&db, claim.expires_at);
+            // Exactly one writer committed the recovery; the other reported none.
+            let mut outcomes = recovered.clone();
+            outcomes.sort();
+            assert_eq!(outcomes, vec![vec![], vec![id.clone()]], "round {round}");
+
+            let committed = committed_workload(&db, &id);
+            assert_eq!(recoveries(&committed), 1, "one recovery transition");
+            assert_eq!(committed.state, WorkloadState::Ready);
+            assert_eq!(committed.attempt_count, 1);
+            assert_eq!(committed.attempts.len(), 1);
+            assert_eq!(committed.attempts[0].status, AttemptStatus::WorkerLost);
+            assert!(committed.current_claim.is_none());
+            let recovering = committed
+                .execution_history
+                .iter()
+                .find(|event| event.to == WorkloadState::Recovering)
+                .unwrap();
+            assert_eq!(recovering.evidence["claim_id"], claim.claim_id);
+            assert_eq!(recovering.evidence["attempts_used"], "1");
+
+            // The losing writer adopted the committed workload instead of
+            // keeping its own, uncommitted view.
+            let expected = serde_json::to_value(&committed).unwrap();
+            assert_eq!(serde_json::to_value(first.get(&id).unwrap()).unwrap(), expected);
+            assert_eq!(serde_json::to_value(second.get(&id).unwrap()).unwrap(), expected);
+            // And neither recovers it again, or commits anything more.
+            assert!(first.recover_expired(claim.expires_at + 1).unwrap().is_empty());
+            assert!(second.recover_expired(claim.expires_at + 1).unwrap().is_empty());
+            assert_eq!(
+                serde_json::to_value(committed_workload(&db, &id)).unwrap(),
+                expected,
+                "no further commit"
+            );
+
+            // The lost claimant cannot complete, even through a projection
+            // that still shows its claim as current and unexpired: the guarded
+            // commit refuses the stale version.
+            let refused = stale
+                .start(&id, &claim.claim_id, "worker-a", claim.fencing_token, claim.expires_at - 1)
+                .unwrap_err();
+            assert_eq!(refused.code, "WORKLOAD_CONFLICT");
+            assert_eq!(stale.get(&id).unwrap().state, WorkloadState::Ready);
+            let refused = stale
+                .complete(
+                    &id,
+                    &claim.claim_id,
+                    "worker-a",
+                    claim.fencing_token,
+                    "late-result",
+                    serde_json::json!({"late": true}),
+                    vec![],
+                    1,
+                    claim.expires_at - 1,
+                )
+                .unwrap_err();
+            assert_eq!(refused.code, "WORKLOAD_CLAIM_INVALID");
+            // Through a current projection its fence no longer matches.
+            let refused = first
+                .heartbeat(&id, &claim.claim_id, "worker-a", claim.fencing_token, claim.expires_at - 1, 10_000)
+                .unwrap_err();
+            assert_eq!(refused.code, "WORKLOAD_CLAIM_INVALID");
+            assert_eq!(serde_json::to_value(committed_workload(&db, &id)).unwrap(), expected);
+            drop((store, first, second, stale, db));
+            let _ = fs::remove_file(path);
+        }
+    }
+
+    #[test]
+    fn concurrent_recovery_never_exceeds_the_attempt_budget() {
+        let (path, mut store) = store("concurrent-recovery-budget");
+        let db = store.db.clone();
+        let id = store.create(create()).unwrap().workload_id;
+        for attempt in 1..=3i64 {
+            // Each attempt is claimed by an up-to-date writer and lost.
+            let mut claimer = WorkloadStore::from_db(db.clone()).unwrap();
+            let claim = claimer
+                .claim(&id, "worker-a", &capabilities(), 100 * attempt, 10_000)
+                .unwrap();
+            let (_, _, recovered) = race_recovery(&db, claim.expires_at);
+            assert_eq!(recovered.concat(), vec![id.clone()], "attempt {attempt}");
+            let committed = committed_workload(&db, &id);
+            assert_eq!(recoveries(&committed), attempt as usize);
+            assert_eq!(committed.attempt_count, attempt as u32);
+        }
+        let committed = committed_workload(&db, &id);
+        assert_eq!(committed.state, WorkloadState::DeadLetter);
+        assert_eq!(
+            committed.execution_history.last().unwrap().evidence["decision"],
+            "attempt_budget_exhausted"
+        );
+        assert!(committed
+            .attempts
+            .iter()
+            .all(|attempt| attempt.status == AttemptStatus::WorkerLost));
+        let refused = WorkloadStore::from_db(db.clone())
+            .unwrap()
+            .claim(&id, "worker-b", &capabilities(), 10_000, 10_000)
+            .unwrap_err();
+        assert_eq!(refused.code, "WORKLOAD_NOT_READY");
+        assert_eq!(committed_workload(&db, &id).attempt_count, 3);
+        drop((store, db));
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn a_policy_that_does_not_retry_worker_failure_dead_letters_a_lost_attempt() {
+        let (path, mut store) = store("no-worker-failure-retry");
+        let mut request = create();
+        request.retry_policy.retry_on = ["timeout".into()].into();
+        let id = store.create(request).unwrap().workload_id;
+        let claim = store.claim(&id, "worker-a", &capabilities(), 100, 10_000).unwrap();
+        store.recover_expired(claim.expires_at).unwrap();
+        let workload = store.get(&id).unwrap();
+        assert_eq!(workload.state, WorkloadState::DeadLetter);
+        assert_eq!(
+            workload.execution_history.last().unwrap().evidence["decision"],
+            "worker_failure_not_retryable"
+        );
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn claiming_exhausted_ready_work_dead_letters_it_instead() {
+        let (path, mut store) = store("exhausted-claim");
+        let id = store.create(create()).unwrap().workload_id;
+        // What unbounded recovery used to leave behind: Ready, budget spent.
+        let mut legacy = store.get(&id).unwrap().clone();
+        legacy.attempt_count = 7;
+        store.workloads.insert(id.clone(), legacy);
+        store.persist_workload(&id, "fixture.legacy").unwrap();
+        let refused = store.claim(&id, "worker-a", &capabilities(), 100, 10_000).unwrap_err();
+        assert_eq!(refused.code, "WORKLOAD_ATTEMPTS_EXHAUSTED");
+        let workload = store.get(&id).unwrap();
+        assert_eq!(workload.state, WorkloadState::DeadLetter);
+        assert_eq!(
+            workload.execution_history.last().unwrap().evidence["reason"],
+            "attempt_budget_exhausted"
+        );
+        // Durable: a restart sees the same outcome.
+        let reopened = WorkloadStore::from_db(store.db.clone()).unwrap();
+        assert_eq!(reopened.get(&id).unwrap().state, WorkloadState::DeadLetter);
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn operator_retry_grants_one_attempt_is_recorded_and_is_bounded() {
+        let (path, mut store) = store("operator-retry");
+        let id = store.create(create()).unwrap().workload_id;
+        // Claim and lose leases until the workload stops being claimable.
+        fn exhaust(store: &mut WorkloadStore, id: &str, now: &mut i64) {
+            loop {
+                *now += 100;
+                match store.claim(id, "worker-a", &capabilities(), *now, 10_000) {
+                    Ok(claim) => {
+                        store.recover_expired(claim.expires_at).unwrap();
+                    }
+                    Err(error) => {
+                        assert!(
+                            matches!(
+                                error.code.as_str(),
+                                "WORKLOAD_NOT_READY" | "WORKLOAD_ATTEMPTS_EXHAUSTED"
+                            ),
+                            "{error:?}"
+                        );
+                        return;
+                    }
+                }
+            }
+        }
+        let mut now = 100;
+        exhaust(&mut store, &id, &mut now);
+        assert_eq!(store.get(&id).unwrap().attempt_count, 3);
+        for retry in 1..=3u32 {
+            now += 100;
+            store
+                .retry_with_reason(&id, "operator-1", now, Some("dependency restored"))
+                .unwrap();
+            let event = store.get(&id).unwrap().execution_history.last().unwrap().clone();
+            assert_eq!(event.actor, "operator-1");
+            assert_eq!(event.evidence["retry"], "operator");
+            assert_eq!(event.evidence["reason"], "dependency restored");
+            assert_eq!(event.evidence["operator_retry"], retry.to_string());
+            exhaust(&mut store, &id, &mut now);
+            assert_eq!(
+                store.get(&id).unwrap().attempt_count,
+                3 + retry,
+                "each operator retry grants exactly one attempt"
+            );
+            assert_eq!(store.get(&id).unwrap().state, WorkloadState::DeadLetter);
+        }
+        assert_eq!(
+            store.retry(&id, "operator-1", now + 100).unwrap_err().code,
+            "WORKLOAD_RETRY_EXHAUSTED"
+        );
+        assert_eq!(store.get(&id).unwrap().state, WorkloadState::DeadLetter);
+        let _ = fs::remove_file(path);
+    }
+
     fn store(name: &str) -> (PathBuf, WorkloadStore) {
         let path =
             std::env::temp_dir().join(format!("workload-{name}-{}.json", std::process::id()));
@@ -1367,7 +1913,8 @@ mod tests {
         let workload = source.create(create()).unwrap();
         let legacy = LegacyWorkloadStore {
             workloads: source.workloads.clone(),
-            idempotency: source.idempotency.clone(),
+            // What a pre-requester-scope release wrote.
+            idempotency: idempotency_index(&source.workloads, legacy_idempotency_scope),
             result_ids: source.result_ids.clone(),
             event_sequence: derived_event_sequence(&source.workloads),
         };
@@ -1458,7 +2005,10 @@ mod tests {
         winner.claim(&id, "worker-a", &caps, 11, 1_000).unwrap();
         let conflict = loser.claim(&id, "worker-b", &caps, 11, 1_000).unwrap_err();
         assert_eq!(conflict.code, "WORKLOAD_CONFLICT");
-        assert_eq!(loser.get(&id).unwrap().state, WorkloadState::Ready);
+        // The loser committed nothing and now projects the winner's claim.
+        let projected = loser.get(&id).unwrap();
+        assert_eq!(projected.state, WorkloadState::Claimed);
+        assert_eq!(projected.current_claim.as_ref().unwrap().worker_id, "worker-a");
 
         let row = db
             .list_collection(DURABLE_WORK_COLLECTION)
@@ -1481,10 +2031,9 @@ mod tests {
             .claim(&id, "worker-d", &caps, 13, 10_000)
             .unwrap_err();
         assert_eq!(conflict.code, "WORKLOAD_CONFLICT");
-        assert_eq!(
-            restarted_loser.get(&id).unwrap().state,
-            WorkloadState::Ready
-        );
+        let projected = restarted_loser.get(&id).unwrap();
+        assert_eq!(projected.state, WorkloadState::Claimed);
+        assert_eq!(projected.current_claim.as_ref().unwrap().worker_id, "worker-c");
         drop((recovery, restarted_winner, restarted_loser, restarted_db));
         let _ = fs::remove_file(path);
     }
@@ -1499,22 +2048,103 @@ mod tests {
         let mut first = WorkloadStore::from_db(db.clone()).unwrap();
         let mut second = WorkloadStore::from_db(db.clone()).unwrap();
         let accepted = first.create(create()).unwrap();
+        // A second writer of the same store loses the require-absent CAS and
+        // converges on what was committed: a different request is refused as a
+        // reused key, never answered with the original.
         let mut competing = create();
         competing.input = serde_json::json!({"id": 2});
         let conflict = second.create(competing).unwrap_err();
-        assert_eq!(conflict.code, "WORKLOAD_CONFLICT");
-        assert!(
-            second.workloads.is_empty(),
-            "failed projection was rolled back"
+        assert_eq!(conflict.code, "IDEMPOTENCY_KEY_REUSED");
+        assert_eq!(
+            second.workloads.keys().collect::<Vec<_>>(),
+            vec![&accepted.workload_id],
+            "only the committed workload is projected"
         );
-
-        drop((first, second, db));
+        // The same request from a third, equally stale writer converges on it.
+        let mut third = WorkloadStore::from_db(db.clone()).unwrap();
+        third.workloads.clear();
+        third.idempotency.clear();
+        third.document_versions.clear();
+        let (converged, created) = third.admit(create()).unwrap();
+        assert!(!created);
+        assert_eq!(converged.workload_id, accepted.workload_id);
+
+        drop((first, second, third, db));
         let restarted = WorkloadStore::from_db(FeltDb::open(&path).unwrap()).unwrap();
         assert_eq!(restarted.workloads.len(), 1);
-        assert_eq!(restarted.idempotency["t:app:one"], accepted.workload_id);
+        assert_eq!(
+            restarted.idempotency[&idempotency_scope("t", "app", &accepted.requested_by, "one")],
+            accepted.workload_id
+        );
         let _ = fs::remove_file(path);
     }
 
+    #[test]
+    fn idempotency_scopes_cannot_be_made_to_coincide() {
+        let (path, mut store) = store("scope-separators");
+        let mut first = create_as("x:k1");
+        first.idempotency_key = "k2".into();
+        let mut second = create_as("x");
+        second.idempotency_key = "k1:k2".into();
+        let first = store.create(first).unwrap();
+        let second = store.create(second).unwrap();
+        assert_ne!(first.workload_id, second.workload_id);
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn concurrent_admissions_by_competing_writers_converge_or_conflict() {
+        for round in 0..20 {
+            let (path, seed) = store(&format!("concurrent-admission-{round}"));
+            let db = seed.db.clone();
+            let admit_concurrently = |requests: [CreateWorkload; 2]| {
+                let barrier = std::sync::Barrier::new(2);
+                std::thread::scope(|scope| {
+                    requests
+                        .into_iter()
+                        .map(|request| {
+                            let mut writer = WorkloadStore::from_db(db.clone()).unwrap();
+                            let barrier = &barrier;
+                            scope.spawn(move || {
+                                barrier.wait();
+                                writer.admit(request)
+                            })
+                        })
+                        .collect::<Vec<_>>()
+                        .into_iter()
+                        .map(|handle| handle.join().unwrap())
+                        .collect::<Vec<_>>()
+                })
+            };
+            // The same request: both writers answer with one workload, and
+            // exactly one of them created it.
+            let same = admit_concurrently([create(), create()]);
+            let (first, first_created) = same[0].as_ref().unwrap().clone();
+            let (second, second_created) = same[1].as_ref().unwrap().clone();
+            assert_eq!(first.workload_id, second.workload_id);
+            assert!(first_created ^ second_created, "round {round}");
+
+            // Different requests under one key: one is admitted, the other is
+            // refused as a reused key and never answered with the winner.
+            let mut left = create();
+            left.idempotency_key = "contested".into();
+            let mut right = left.clone();
+            right.input = serde_json::json!({"id": 2});
+            let contested = admit_concurrently([left, right]);
+            let admitted = contested.iter().filter(|outcome| outcome.is_ok()).count();
+            assert_eq!(admitted, 1, "round {round}");
+            assert!(contested.iter().any(|outcome| outcome
+                .as_ref()
+                .is_err_and(|error| error.code == "IDEMPOTENCY_KEY_REUSED")));
+            let (winner, _) = contested.iter().find_map(|o| o.as_ref().ok()).unwrap();
+            let stored = WorkloadStore::from_db(db.clone()).unwrap();
+            assert_eq!(stored.workloads.len(), 2);
+            assert_eq!(stored.get(&winner.workload_id).unwrap().input, winner.input);
+            drop((seed, stored, db));
+            let _ = fs::remove_file(path);
+        }
+    }
+
     #[test]
     fn pre_versioned_canonical_rows_upgrade_through_the_standard_cas_contract() {
         let (path, mut store) = store("pre-versioned-upgrade");
@@ -1589,7 +2219,9 @@ mod tests {
         store.document_versions.clear();
         let rebuilt = WorkloadStore::from_db(store.db.clone()).unwrap();
         assert_eq!(
-            rebuilt.idempotency.get("t:app:one"),
+            rebuilt
+                .idempotency
+                .get(&idempotency_scope("t", "app", &workload.requested_by, "one")),
             Some(&workload.workload_id)
         );
         assert_eq!(