@feltdb/core 0.12.1 → 0.13.0

package/dist/create/server-source/crates/feltdb/src/worker_mesh.rs

@@ -486,6 +486,48 @@ impl WorkerMeshStore {
         }
         self.persist()
     }
+    /// Recover work whose claim lease expired, and release those claims from
+    /// the workers that held them so their capacity is not held by work they
+    /// no longer own. Recovery itself — `WorkerLost`, the attempt budget, the
+    /// decision — is `WorkloadStore::recover_expired`. Worker liveness is not
+    /// judged here; that is `detect_failures`.
+    pub fn recover_expired_work(
+        &mut self,
+        now: i64,
+        workloads: &mut WorkloadStore,
+    ) -> Result<Vec<String>, WorkerMeshError> {
+        let expired: Vec<(String, String, String)> = workloads
+            .workloads
+            .values()
+            .filter_map(|w| w.current_claim.as_ref().map(|claim| (w, claim)))
+            .filter(|(_, claim)| claim.expires_at <= now)
+            .map(|(w, claim)| {
+                (
+                    w.workload_id.clone(),
+                    claim.worker_id.clone(),
+                    claim.claim_id.clone(),
+                )
+            })
+            .collect();
+        let recovered = workloads.recover_expired(now)?;
+        for (workload_id, worker_id, claim_id) in expired {
+            // Release a claim only once the committed workload no longer holds
+            // it, whichever writer of the store recovered it.
+            let still_current = workloads
+                .workloads
+                .get(&workload_id)
+                .and_then(|w| w.current_claim.as_ref())
+                .is_some_and(|claim| claim.claim_id == claim_id);
+            let held = self
+                .workers
+                .get(&worker_id)
+                .is_some_and(|worker| worker.claims.contains(&claim_id));
+            if held && !still_current {
+                self.release_claim(&worker_id, &claim_id)?;
+            }
+        }
+        Ok(recovered)
+    }
     pub fn detect_failures(
         &mut self,
         now: i64,
@@ -507,7 +549,7 @@ impl WorkerMeshStore {
                 worker.claims.clear();
             }
         }
-        workloads.recover_expired(now)?;
+        self.recover_expired_work(now, workloads)?;
         self.persist()?;
         Ok(failed)
     }
@@ -852,6 +894,167 @@ mod tests {
         let _ = fs::remove_file(w);
     }
     #[test]
+    fn a_claim_recovered_by_another_writer_is_still_released() {
+        let (m, wpath, mut mesh, mut workloads, grant) = setup("release-raced");
+        let worker = register(&mut mesh, grant.clone());
+        let heartbeat = |timestamp| WorkerHeartbeat {
+            worker_id: worker.worker_id.clone(),
+            version: "1".into(),
+            capability_version: 1,
+            capacity: WorkerCapacity {
+                concurrency: 1,
+                active: 0,
+            },
+            health: WorkerHealth::Healthy,
+            timestamp,
+            running_claims: BTreeSet::new(),
+        };
+        mesh.heartbeat(&worker.worker_id, heartbeat(11), 2).unwrap();
+        let subject = grant.grant.subject.clone();
+        let create = |key: &str| CreateWorkload {
+            tenant_id: "t".into(),
+            application_id: "a".into(),
+            environment: "production".into(),
+            definition_id: "extract".into(),
+            definition_revision: "r".into(),
+            trigger: "manual".into(),
+            input: serde_json::json!({"file": key}),
+            requested_by: subject.clone(),
+            session_id: None,
+            authorization_snapshot: grant.clone(),
+            capability_snapshot: ["documents.extract".into()].into(),
+            connection_scopes: BTreeSet::new(),
+            priority: 0,
+            idempotency_key: key.into(),
+            retry_policy: WorkloadRetryPolicy::default(),
+            timeouts: WorkloadTimeouts::default(),
+            dependencies: vec![],
+            not_before: None,
+            created_at: 11,
+        };
+        let first = workloads.create(create("first")).unwrap();
+        let second = workloads.create(create("second")).unwrap();
+        mesh.place_and_claim(
+            &mut workloads,
+            &first.workload_id,
+            &worker.worker_id,
+            12,
+            1000,
+        )
+        .unwrap();
+        assert_eq!(mesh.workers[&worker.worker_id].capacity.active, 1);
+        // Another writer of the same store recovers the work first.
+        let mut other = WorkloadStore::from_db(workloads.db_for_tests()).unwrap();
+        assert_eq!(
+            other.recover_expired(14).unwrap(),
+            vec![first.workload_id.clone()]
+        );
+        // This node's pass loses the guarded commit, adopts the committed
+        // workload, does not report it as its own recovery, and still frees the
+        // worker, whose claim is no longer the workload's.
+        assert!(mesh
+            .recover_expired_work(14, &mut workloads)
+            .unwrap()
+            .is_empty());
+        let recovered = workloads.get(&first.workload_id).unwrap();
+        assert_eq!(recovered.state, crate::workload::WorkloadState::Ready);
+        assert_eq!(
+            recovered
+                .execution_history
+                .iter()
+                .filter(|event| event.to == crate::workload::WorkloadState::Recovering)
+                .count(),
+            1
+        );
+        assert!(mesh.workers[&worker.worker_id].claims.is_empty());
+        assert_eq!(mesh.workers[&worker.worker_id].capacity.active, 0);
+        let _ = second;
+        let _ = fs::remove_file(m);
+        let _ = fs::remove_file(wpath);
+    }
+    #[test]
+    fn recovering_an_expired_claim_releases_it_from_a_live_worker() {
+        let (m, wpath, mut mesh, mut workloads, grant) = setup("release-expired");
+        let worker = register(&mut mesh, grant.clone());
+        let heartbeat = |timestamp| WorkerHeartbeat {
+            worker_id: worker.worker_id.clone(),
+            version: "1".into(),
+            capability_version: 1,
+            capacity: WorkerCapacity {
+                concurrency: 1,
+                active: 0,
+            },
+            health: WorkerHealth::Healthy,
+            timestamp,
+            running_claims: BTreeSet::new(),
+        };
+        mesh.heartbeat(&worker.worker_id, heartbeat(11), 2).unwrap();
+        let subject = grant.grant.subject.clone();
+        let create = |key: &str| CreateWorkload {
+            tenant_id: "t".into(),
+            application_id: "a".into(),
+            environment: "production".into(),
+            definition_id: "extract".into(),
+            definition_revision: "r".into(),
+            trigger: "manual".into(),
+            input: serde_json::json!({"file": key}),
+            requested_by: subject.clone(),
+            session_id: None,
+            authorization_snapshot: grant.clone(),
+            capability_snapshot: ["documents.extract".into()].into(),
+            connection_scopes: BTreeSet::new(),
+            priority: 0,
+            idempotency_key: key.into(),
+            retry_policy: WorkloadRetryPolicy::default(),
+            timeouts: WorkloadTimeouts::default(),
+            dependencies: vec![],
+            not_before: None,
+            created_at: 11,
+        };
+        let first = workloads.create(create("first")).unwrap();
+        let second = workloads.create(create("second")).unwrap();
+        mesh.place_and_claim(
+            &mut workloads,
+            &first.workload_id,
+            &worker.worker_id,
+            12,
+            1000,
+        )
+        .unwrap();
+        assert_eq!(mesh.workers[&worker.worker_id].capacity.active, 1);
+        // The lease expired at 13. The server's recovery is claim recovery: it
+        // recovers the work and frees the worker, and does not judge liveness.
+        let recovered = mesh.recover_expired_work(14, &mut workloads).unwrap();
+        assert_eq!(recovered, vec![first.workload_id.clone()]);
+        let live = &mesh.workers[&worker.worker_id];
+        assert!(live.claims.is_empty());
+        assert_eq!(live.capacity.active, 0);
+        assert_eq!(live.lifecycle, WorkerLifecycle::Ready);
+        assert_eq!(
+            workloads.get(&first.workload_id).unwrap().state,
+            crate::workload::WorkloadState::Ready
+        );
+        // Its capacity is usable again.
+        mesh.place_and_claim(
+            &mut workloads,
+            &second.workload_id,
+            &worker.worker_id,
+            15,
+            10_000,
+        )
+        .unwrap();
+        // Long past its heartbeat timeout, claim recovery still leaves the
+        // worker's lifecycle alone; only `detect_failures` marks it offline.
+        mesh.recover_expired_work(1_000, &mut workloads).unwrap();
+        assert_ne!(
+            mesh.workers[&worker.worker_id].lifecycle,
+            WorkerLifecycle::Offline
+        );
+        assert!(mesh.workers[&worker.worker_id].claims.is_empty());
+        let _ = fs::remove_file(m);
+        let _ = fs::remove_file(wpath);
+    }
+    #[test]
     fn placement_claim_fencing_failure_and_recovery() {
         let (m, wpath, mut mesh, mut workloads, grant) = setup("claim");
         let worker = register(&mut mesh, grant.clone());