@feltdb/core 0.13.0 → 0.15.0

package/dist/create/server-source/crates/feltdb/src/worker_mesh.rs

@@ -491,11 +491,51 @@ impl WorkerMeshStore {
     /// no longer own. Recovery itself — `WorkerLost`, the attempt budget, the
     /// decision — is `WorkloadStore::recover_expired`. Worker liveness is not
     /// judged here; that is `detect_failures`.
+    /// Time out work past a deadline ([`WorkloadStore::enforce_deadlines`]) and
+    /// release the claims the committed workloads no longer hold, as
+    /// [`Self::recover_expired_work`] does for expired leases.
+    pub fn enforce_deadlines(
+        &mut self,
+        now: i64,
+        workloads: &mut WorkloadStore,
+    ) -> Result<Vec<String>, WorkerMeshError> {
+        // The claims this node's projection knows, listed before the store is
+        // re-read: a claim another writer already ended must still be released
+        // here, so the list cannot come from the refreshed projection.
+        let claimed: Vec<(String, String, String)> = workloads
+            .workloads
+            .values()
+            .filter_map(|w| {
+                w.current_claim.as_ref().map(|claim| {
+                    (
+                        w.workload_id.clone(),
+                        claim.worker_id.clone(),
+                        claim.claim_id.clone(),
+                    )
+                })
+            })
+            .collect();
+        let expired = workloads.enforce_deadlines(now)?;
+        for (workload_id, worker_id, claim_id) in claimed {
+            let still_current = workloads
+                .workloads
+                .get(&workload_id)
+                .and_then(|w| w.current_claim.as_ref())
+                .is_some_and(|claim| claim.claim_id == claim_id);
+            // Releasing is idempotent: a claim the worker no longer holds is a no-op.
+            if !still_current && self.workers.contains_key(&worker_id) {
+                self.release_claim(&worker_id, &claim_id)?;
+            }
+        }
+        Ok(expired)
+    }
     pub fn recover_expired_work(
         &mut self,
         now: i64,
         workloads: &mut WorkloadStore,
     ) -> Result<Vec<String>, WorkerMeshError> {
+        // Listed before `recover_expired` re-reads the store, for the same
+        // reason as in `enforce_deadlines`.
         let expired: Vec<(String, String, String)> = workloads
             .workloads
             .values()
@@ -645,7 +685,10 @@ impl WorkerMeshStore {
             .workers
             .get_mut(worker_id)
             .ok_or_else(|| WorkerMeshError::new("WORKER_NOT_FOUND", "worker not found"))?;
-        worker.claims.remove(claim_id);
+        // Releasing is idempotent: only a claim the worker still holds frees capacity.
+        if !worker.claims.remove(claim_id) {
+            return Ok(());
+        }
         worker.capacity.active = worker.capacity.active.saturating_sub(1);
         if worker.lifecycle == WorkerLifecycle::Draining && worker.claims.is_empty() {
             worker.lifecycle = WorkerLifecycle::Offline
@@ -894,6 +937,99 @@ mod tests {
         let _ = fs::remove_file(w);
     }
     #[test]
+    fn releasing_a_claim_twice_frees_capacity_once() {
+        let (m, wpath, mut mesh, mut workloads, grant) = setup("release-twice");
+        let worker = mesh
+            .register(WorkerRegistration {
+                tenant_id: "t".into(),
+                application_id: "a".into(),
+                name: "edge".into(),
+                runtime: WorkerRuntimeKind::Customer,
+                version: "1".into(),
+                capabilities: ["documents.extract".into()].into(),
+                workload_kinds: BTreeSet::new(),
+                resource_profile: Default::default(),
+                labels: BTreeMap::new(),
+                capacity: WorkerCapacity {
+                    concurrency: 2,
+                    active: 0,
+                },
+                registered_by: Subject::Service("registrar".into()),
+                authorization_grant: grant.clone(),
+                registered_at: 10,
+            })
+            .unwrap();
+        mesh.heartbeat(
+            &worker.worker_id,
+            WorkerHeartbeat {
+                worker_id: worker.worker_id.clone(),
+                version: "1".into(),
+                capability_version: 1,
+                capacity: WorkerCapacity {
+                    concurrency: 2,
+                    active: 0,
+                },
+                health: WorkerHealth::Healthy,
+                timestamp: 11,
+                running_claims: BTreeSet::new(),
+            },
+            2,
+        )
+        .unwrap();
+        let subject = grant.grant.subject.clone();
+        let create = |key: &str| CreateWorkload {
+            tenant_id: "t".into(),
+            application_id: "a".into(),
+            environment: "production".into(),
+            definition_id: "extract".into(),
+            definition_revision: "r".into(),
+            trigger: "manual".into(),
+            input: serde_json::json!({"file": key}),
+            requested_by: subject.clone(),
+            session_id: None,
+            authorization_snapshot: grant.clone(),
+            capability_snapshot: ["documents.extract".into()].into(),
+            connection_scopes: BTreeSet::new(),
+            priority: 0,
+            idempotency_key: key.into(),
+            retry_policy: WorkloadRetryPolicy::default(),
+            timeouts: WorkloadTimeouts::default(),
+            dependencies: vec![],
+            not_before: None,
+            created_at: 11,
+        };
+        let mut claims = vec![];
+        for key in ["first", "second"] {
+            let work = workloads.create(create(key)).unwrap();
+            mesh.place_and_claim(
+                &mut workloads,
+                &work.workload_id,
+                &worker.worker_id,
+                12,
+                10_000,
+            )
+            .unwrap();
+            claims.push(
+                workloads
+                    .get(&work.workload_id)
+                    .unwrap()
+                    .current_claim
+                    .clone()
+                    .unwrap()
+                    .claim_id,
+            );
+        }
+        assert_eq!(mesh.workers[&worker.worker_id].capacity.active, 2);
+        // A completion retried after a lost response releases the same claim again.
+        mesh.release_claim(&worker.worker_id, &claims[0]).unwrap();
+        mesh.release_claim(&worker.worker_id, &claims[0]).unwrap();
+        let state = &mesh.workers[&worker.worker_id];
+        assert_eq!(state.claims.len(), 1);
+        assert_eq!(state.capacity.active, 1, "the second claim is still held");
+        let _ = fs::remove_file(m);
+        let _ = fs::remove_file(wpath);
+    }
+    #[test]
     fn a_claim_recovered_by_another_writer_is_still_released() {
         let (m, wpath, mut mesh, mut workloads, grant) = setup("release-raced");
         let worker = register(&mut mesh, grant.clone());