@feltdb/core 0.12.2 → 0.15.0

package/dist/create/server-source/crates/feltdb/src/state_contract.rs

@@ -1061,6 +1061,13 @@ impl StateFailure {
     }
 }
 
+fn idempotency_conflict() -> StateFailure {
+    StateFailure::new(
+        "IDEMPOTENCY_CONFLICT",
+        "transaction ID was already committed with a different canonical payload",
+    )
+}
+
 fn collection<'a>(
     schema: &'a StateSchema,
     name: &str,
@@ -2478,53 +2485,114 @@ fn execute_transaction_with_policies(
         transaction_id
     );
     let payload_hash = format!("sha256:{:x}", Sha256::digest(&canonical));
-    if db
-        .has_applied_transaction(&storage_transaction_id)
-        .map_err(StateFailure::storage)?
-    {
-        if db
-            .applied_transaction_payload_hash(&storage_transaction_id)
-            .map_err(StateFailure::storage)?
-            .as_deref()
-            != Some(payload_hash.as_str())
-        {
-            return Err(StateFailure::new(
-                "IDEMPOTENCY_CONFLICT",
-                "transaction ID was already committed with a different canonical payload",
-            ));
-        }
-        let state = db.sequence().map_err(StateFailure::storage)?;
-        let cursor = db
-            .operation_versions()
-            .map_err(StateFailure::storage)?
-            .into_iter()
-            .collect();
-        let audit = TransactionAudit {
-            transaction_id: transaction_id.clone(),
-            subject: request.authorization.subject.clone(),
-            tenant: request.tenant_id.clone(),
-            application: request.application_id.clone(),
-            revision: request.revision_id.clone(),
-            schema_version: request.schema_version,
-            state_before: state,
-            state_after: state,
-            operations: request.operations.len(),
-            authorization_decisions: vec!["state:write allowed".into()],
-            causal_parent: request.causal_parent,
-            result: "duplicate".into(),
-        };
-        return Ok(TransactionResult {
-            transaction_id,
-            base_revision: db.current_revision().map_err(StateFailure::storage)?,
-            commit_revision: db.current_revision().map_err(StateFailure::storage)?,
-            status: crate::TransactionStatus::Committed,
-            state_before: state,
-            state_after: state,
-            causal_cursor: cursor,
-            duplicate: true,
-            audit,
-        });
+    // A retried id is answered before anything is evaluated against current
+    // state, with the original commit: its revisions and positions, not the
+    // store's current ones. The same answer the commit lock gives a retry that
+    // races past this check.
+    if let Some(result) = committed_result(
+        db,
+        request,
+        &transaction_id,
+        &storage_transaction_id,
+        &payload_hash,
+    )? {
+        return Ok(result);
     }
+    match commit_new_transaction(
+        db,
+        schema,
+        request,
+        uniform_write_policy,
+        collection_write_policies,
+        &transaction_id,
+        &storage_transaction_id,
+        &payload_hash,
+    ) {
+        Ok(result) => Ok(result),
+        // The request was evaluated against state read before the commit
+        // lock. An identical request that committed in between makes this one
+        // its duplicate, and the duplicate -- the original commit -- is the
+        // answer, not the refusal that later state produced (a create that
+        // now "exists", a fence that now "changed").
+        Err(failure) => match committed_result(
+            db,
+            request,
+            &transaction_id,
+            &storage_transaction_id,
+            &payload_hash,
+        )? {
+            Some(result) => Ok(result),
+            None => Err(failure),
+        },
+    }
+}
+
+/// A retried id, answered with the original commit: its revisions and
+/// positions, never the store's current ones. `IDEMPOTENCY_CONFLICT` when the id
+/// committed with a different payload; `None` when it has not committed.
+fn committed_result(
+    db: &FeltDb,
+    request: &TransactionRequest,
+    transaction_id: &str,
+    storage_transaction_id: &str,
+    payload_hash: &str,
+) -> Result<Option<TransactionResult>, StateFailure> {
+    let Some(original) = db
+        .committed_transaction(storage_transaction_id, Some(payload_hash))
+        .map_err(|error| match error {
+            crate::FlowError::TransactionIdPayloadMismatch { .. } => idempotency_conflict(),
+            other => StateFailure::storage(other),
+        })?
+    else {
+        return Ok(None);
+    };
+    let cursor = db
+        .operation_versions()
+        .map_err(StateFailure::storage)?
+        .into_iter()
+        .collect();
+    let audit = TransactionAudit {
+        transaction_id: transaction_id.to_string(),
+        subject: request.authorization.subject.clone(),
+        tenant: request.tenant_id.clone(),
+        application: request.application_id.clone(),
+        revision: request.revision_id.clone(),
+        schema_version: request.schema_version,
+        state_before: original.state_before,
+        state_after: original.state_after,
+        operations: request.operations.len(),
+        authorization_decisions: vec!["state:write allowed".into()],
+        causal_parent: request.causal_parent,
+        result: "duplicate".into(),
+    };
+    Ok(Some(TransactionResult {
+        transaction_id: transaction_id.to_string(),
+        base_revision: original.base_revision,
+        commit_revision: original.commit_revision,
+        status: original.status,
+        state_before: original.state_before,
+        state_after: original.state_after,
+        causal_cursor: cursor,
+        duplicate: true,
+        audit,
+    }))
+}
+
+/// Evaluate and commit a transaction id that has not committed.
+#[allow(clippy::too_many_arguments)]
+fn commit_new_transaction(
+    db: &FeltDb,
+    schema: &StateSchema,
+    request: &TransactionRequest,
+    uniform_write_policy: Option<&PolicySubject>,
+    collection_write_policies: Option<&BTreeMap<String, PolicySubject>>,
+    transaction_id: &str,
+    storage_transaction_id: &str,
+    payload_hash: &str,
+) -> Result<TransactionResult, StateFailure> {
+    let transaction_id = transaction_id.to_string();
+    let storage_transaction_id = storage_transaction_id.to_string();
+    let payload_hash = payload_hash.to_string();
     let namespace = request
         .state_namespace
         .as_deref()
@@ -2753,7 +2821,19 @@ fn execute_transaction_with_policies(
         let value = match operation.kind {
             TransactionOperationKind::Insert => {
                 if existing_value.is_some() {
-                    return Err(StateFailure::new("CONFLICT", "record already exists"));
+                    // A create is conditional on absence: the same structured
+                    // refusal a `requireAbsent` create gets on every route.
+                    let refusal = crate::PreconditionFailure::Present {
+                        collection: capability.clone(),
+                        key: key.clone(),
+                    };
+                    let mut failure =
+                        StateFailure::new("PRECONDITION_FAILED", &refusal.to_string());
+                    failure.transaction = Some(transaction_id.clone());
+                    failure.resource = Some(format!("{}:{}", operation.collection, key));
+                    failure.failure =
+                        Some(refusal.public_json(&crate::ReadLayout::Namespace(namespace.clone())));
+                    return Err(failure);
                 }
                 let mut value = operation.value.clone();
                 match definition {
@@ -2924,12 +3004,18 @@ fn execute_transaction_with_policies(
             // "this deployment is broken", so it is carried through as
             // structure rather than flattened into the message.
             if let FlowError::PreconditionFailed(refusal) = &error {
+                let layout = crate::ReadLayout::Namespace(namespace.clone());
+                let public = refusal.public_json(&layout);
                 let mut failure =
                     StateFailure::new("PRECONDITION_FAILED", &error.to_string());
                 failure.transaction = Some(transaction_id.clone());
                 failure.causal_position = db.sequence().ok();
-                failure.resource = Some(refusal.key().to_string());
-                failure.failure = serde_json::to_value(refusal.as_ref()).ok();
+                failure.resource = public
+                    .get("key")
+                    .and_then(Value::as_str)
+                    .map(str::to_owned)
+                    .or_else(|| Some(refusal.key().to_string()));
+                failure.failure = Some(public);
                 if let PreconditionFailure::Version {
                     expected, actual, ..
                 } = refusal.as_ref()
@@ -2949,6 +3035,21 @@ fn execute_transaction_with_policies(
                 failure.actual = Some(Value::from(*actual));
                 return failure;
             }
+            if let FlowError::TransactionIdPayloadMismatch { .. } = &error {
+                // Two different payloads raced past the check above under one
+                // id; the commit lock admitted the first. Same refusal.
+                let mut failure = idempotency_conflict();
+                failure.transaction = Some(transaction_id.clone());
+                return failure;
+            }
+            if let FlowError::Io(_) | FlowError::Serde(_) = &error {
+                // The commit could not be made durable. Never a lost race and
+                // never a success: the outcome must be treated as unknown and
+                // reconciled by retrying with the same transaction id.
+                let mut failure = StateFailure::new("DURABILITY_FAILURE", &error.to_string());
+                failure.transaction = Some(transaction_id.clone());
+                return failure;
+            }
             let text = error.to_string();
             if text.contains("INVALID_READ_SET:") {
                 // A malformed or foreign basis is a caller error, never a lost race.
@@ -3262,11 +3363,19 @@ mod tests {
         let mut stale = request;
         stale.transaction_id = Some("stale".into());
         stale.causal_parent = Some(0);
+        // A create of a record that exists is refused as a structured presence
+        // precondition, as a `requireAbsent` create is on every other route.
+        let refused = execute_transaction(&db, &schema, &stale, None).unwrap_err();
+        assert_eq!(refused.code, "PRECONDITION_FAILED");
+        let failure = refused.failure.expect("structured failure");
+        assert_eq!(failure["predicate"], "present");
+        assert_eq!(failure["collection"], "incidents");
+        assert_eq!(failure["id"], "one");
+        assert_eq!(failure["key"], "incidents:one");
         assert_eq!(
-            execute_transaction(&db, &schema, &stale, None)
-                .unwrap_err()
-                .code,
-            "CONFLICT"
+            db.export_snapshot().unwrap().rows.len(),
+            2,
+            "nothing written"
         );
     }
     /// A conditional write through the canonical contract has to mean the same

package/dist/create/server-source/crates/feltdb/src/worker_mesh.rs

@@ -486,6 +486,88 @@ impl WorkerMeshStore {
         }
         self.persist()
     }
+    /// Recover work whose claim lease expired, and release those claims from
+    /// the workers that held them so their capacity is not held by work they
+    /// no longer own. Recovery itself — `WorkerLost`, the attempt budget, the
+    /// decision — is `WorkloadStore::recover_expired`. Worker liveness is not
+    /// judged here; that is `detect_failures`.
+    /// Time out work past a deadline ([`WorkloadStore::enforce_deadlines`]) and
+    /// release the claims the committed workloads no longer hold, as
+    /// [`Self::recover_expired_work`] does for expired leases.
+    pub fn enforce_deadlines(
+        &mut self,
+        now: i64,
+        workloads: &mut WorkloadStore,
+    ) -> Result<Vec<String>, WorkerMeshError> {
+        // The claims this node's projection knows, listed before the store is
+        // re-read: a claim another writer already ended must still be released
+        // here, so the list cannot come from the refreshed projection.
+        let claimed: Vec<(String, String, String)> = workloads
+            .workloads
+            .values()
+            .filter_map(|w| {
+                w.current_claim.as_ref().map(|claim| {
+                    (
+                        w.workload_id.clone(),
+                        claim.worker_id.clone(),
+                        claim.claim_id.clone(),
+                    )
+                })
+            })
+            .collect();
+        let expired = workloads.enforce_deadlines(now)?;
+        for (workload_id, worker_id, claim_id) in claimed {
+            let still_current = workloads
+                .workloads
+                .get(&workload_id)
+                .and_then(|w| w.current_claim.as_ref())
+                .is_some_and(|claim| claim.claim_id == claim_id);
+            // Releasing is idempotent: a claim the worker no longer holds is a no-op.
+            if !still_current && self.workers.contains_key(&worker_id) {
+                self.release_claim(&worker_id, &claim_id)?;
+            }
+        }
+        Ok(expired)
+    }
+    pub fn recover_expired_work(
+        &mut self,
+        now: i64,
+        workloads: &mut WorkloadStore,
+    ) -> Result<Vec<String>, WorkerMeshError> {
+        // Listed before `recover_expired` re-reads the store, for the same
+        // reason as in `enforce_deadlines`.
+        let expired: Vec<(String, String, String)> = workloads
+            .workloads
+            .values()
+            .filter_map(|w| w.current_claim.as_ref().map(|claim| (w, claim)))
+            .filter(|(_, claim)| claim.expires_at <= now)
+            .map(|(w, claim)| {
+                (
+                    w.workload_id.clone(),
+                    claim.worker_id.clone(),
+                    claim.claim_id.clone(),
+                )
+            })
+            .collect();
+        let recovered = workloads.recover_expired(now)?;
+        for (workload_id, worker_id, claim_id) in expired {
+            // Release a claim only once the committed workload no longer holds
+            // it, whichever writer of the store recovered it.
+            let still_current = workloads
+                .workloads
+                .get(&workload_id)
+                .and_then(|w| w.current_claim.as_ref())
+                .is_some_and(|claim| claim.claim_id == claim_id);
+            let held = self
+                .workers
+                .get(&worker_id)
+                .is_some_and(|worker| worker.claims.contains(&claim_id));
+            if held && !still_current {
+                self.release_claim(&worker_id, &claim_id)?;
+            }
+        }
+        Ok(recovered)
+    }
     pub fn detect_failures(
         &mut self,
         now: i64,
@@ -507,7 +589,7 @@ impl WorkerMeshStore {
                 worker.claims.clear();
             }
         }
-        workloads.recover_expired(now)?;
+        self.recover_expired_work(now, workloads)?;
         self.persist()?;
         Ok(failed)
     }
@@ -603,7 +685,10 @@ impl WorkerMeshStore {
             .workers
             .get_mut(worker_id)
             .ok_or_else(|| WorkerMeshError::new("WORKER_NOT_FOUND", "worker not found"))?;
-        worker.claims.remove(claim_id);
+        // Releasing is idempotent: only a claim the worker still holds frees capacity.
+        if !worker.claims.remove(claim_id) {
+            return Ok(());
+        }
         worker.capacity.active = worker.capacity.active.saturating_sub(1);
         if worker.lifecycle == WorkerLifecycle::Draining && worker.claims.is_empty() {
             worker.lifecycle = WorkerLifecycle::Offline
@@ -852,6 +937,260 @@ mod tests {
         let _ = fs::remove_file(w);
     }
     #[test]
+    fn releasing_a_claim_twice_frees_capacity_once() {
+        let (m, wpath, mut mesh, mut workloads, grant) = setup("release-twice");
+        let worker = mesh
+            .register(WorkerRegistration {
+                tenant_id: "t".into(),
+                application_id: "a".into(),
+                name: "edge".into(),
+                runtime: WorkerRuntimeKind::Customer,
+                version: "1".into(),
+                capabilities: ["documents.extract".into()].into(),
+                workload_kinds: BTreeSet::new(),
+                resource_profile: Default::default(),
+                labels: BTreeMap::new(),
+                capacity: WorkerCapacity {
+                    concurrency: 2,
+                    active: 0,
+                },
+                registered_by: Subject::Service("registrar".into()),
+                authorization_grant: grant.clone(),
+                registered_at: 10,
+            })
+            .unwrap();
+        mesh.heartbeat(
+            &worker.worker_id,
+            WorkerHeartbeat {
+                worker_id: worker.worker_id.clone(),
+                version: "1".into(),
+                capability_version: 1,
+                capacity: WorkerCapacity {
+                    concurrency: 2,
+                    active: 0,
+                },
+                health: WorkerHealth::Healthy,
+                timestamp: 11,
+                running_claims: BTreeSet::new(),
+            },
+            2,
+        )
+        .unwrap();
+        let subject = grant.grant.subject.clone();
+        let create = |key: &str| CreateWorkload {
+            tenant_id: "t".into(),
+            application_id: "a".into(),
+            environment: "production".into(),
+            definition_id: "extract".into(),
+            definition_revision: "r".into(),
+            trigger: "manual".into(),
+            input: serde_json::json!({"file": key}),
+            requested_by: subject.clone(),
+            session_id: None,
+            authorization_snapshot: grant.clone(),
+            capability_snapshot: ["documents.extract".into()].into(),
+            connection_scopes: BTreeSet::new(),
+            priority: 0,
+            idempotency_key: key.into(),
+            retry_policy: WorkloadRetryPolicy::default(),
+            timeouts: WorkloadTimeouts::default(),
+            dependencies: vec![],
+            not_before: None,
+            created_at: 11,
+        };
+        let mut claims = vec![];
+        for key in ["first", "second"] {
+            let work = workloads.create(create(key)).unwrap();
+            mesh.place_and_claim(
+                &mut workloads,
+                &work.workload_id,
+                &worker.worker_id,
+                12,
+                10_000,
+            )
+            .unwrap();
+            claims.push(
+                workloads
+                    .get(&work.workload_id)
+                    .unwrap()
+                    .current_claim
+                    .clone()
+                    .unwrap()
+                    .claim_id,
+            );
+        }
+        assert_eq!(mesh.workers[&worker.worker_id].capacity.active, 2);
+        // A completion retried after a lost response releases the same claim again.
+        mesh.release_claim(&worker.worker_id, &claims[0]).unwrap();
+        mesh.release_claim(&worker.worker_id, &claims[0]).unwrap();
+        let state = &mesh.workers[&worker.worker_id];
+        assert_eq!(state.claims.len(), 1);
+        assert_eq!(state.capacity.active, 1, "the second claim is still held");
+        let _ = fs::remove_file(m);
+        let _ = fs::remove_file(wpath);
+    }
+    #[test]
+    fn a_claim_recovered_by_another_writer_is_still_released() {
+        let (m, wpath, mut mesh, mut workloads, grant) = setup("release-raced");
+        let worker = register(&mut mesh, grant.clone());
+        let heartbeat = |timestamp| WorkerHeartbeat {
+            worker_id: worker.worker_id.clone(),
+            version: "1".into(),
+            capability_version: 1,
+            capacity: WorkerCapacity {
+                concurrency: 1,
+                active: 0,
+            },
+            health: WorkerHealth::Healthy,
+            timestamp,
+            running_claims: BTreeSet::new(),
+        };
+        mesh.heartbeat(&worker.worker_id, heartbeat(11), 2).unwrap();
+        let subject = grant.grant.subject.clone();
+        let create = |key: &str| CreateWorkload {
+            tenant_id: "t".into(),
+            application_id: "a".into(),
+            environment: "production".into(),
+            definition_id: "extract".into(),
+            definition_revision: "r".into(),
+            trigger: "manual".into(),
+            input: serde_json::json!({"file": key}),
+            requested_by: subject.clone(),
+            session_id: None,
+            authorization_snapshot: grant.clone(),
+            capability_snapshot: ["documents.extract".into()].into(),
+            connection_scopes: BTreeSet::new(),
+            priority: 0,
+            idempotency_key: key.into(),
+            retry_policy: WorkloadRetryPolicy::default(),
+            timeouts: WorkloadTimeouts::default(),
+            dependencies: vec![],
+            not_before: None,
+            created_at: 11,
+        };
+        let first = workloads.create(create("first")).unwrap();
+        let second = workloads.create(create("second")).unwrap();
+        mesh.place_and_claim(
+            &mut workloads,
+            &first.workload_id,
+            &worker.worker_id,
+            12,
+            1000,
+        )
+        .unwrap();
+        assert_eq!(mesh.workers[&worker.worker_id].capacity.active, 1);
+        // Another writer of the same store recovers the work first.
+        let mut other = WorkloadStore::from_db(workloads.db_for_tests()).unwrap();
+        assert_eq!(
+            other.recover_expired(14).unwrap(),
+            vec![first.workload_id.clone()]
+        );
+        // This node's pass loses the guarded commit, adopts the committed
+        // workload, does not report it as its own recovery, and still frees the
+        // worker, whose claim is no longer the workload's.
+        assert!(mesh
+            .recover_expired_work(14, &mut workloads)
+            .unwrap()
+            .is_empty());
+        let recovered = workloads.get(&first.workload_id).unwrap();
+        assert_eq!(recovered.state, crate::workload::WorkloadState::Ready);
+        assert_eq!(
+            recovered
+                .execution_history
+                .iter()
+                .filter(|event| event.to == crate::workload::WorkloadState::Recovering)
+                .count(),
+            1
+        );
+        assert!(mesh.workers[&worker.worker_id].claims.is_empty());
+        assert_eq!(mesh.workers[&worker.worker_id].capacity.active, 0);
+        let _ = second;
+        let _ = fs::remove_file(m);
+        let _ = fs::remove_file(wpath);
+    }
+    #[test]
+    fn recovering_an_expired_claim_releases_it_from_a_live_worker() {
+        let (m, wpath, mut mesh, mut workloads, grant) = setup("release-expired");
+        let worker = register(&mut mesh, grant.clone());
+        let heartbeat = |timestamp| WorkerHeartbeat {
+            worker_id: worker.worker_id.clone(),
+            version: "1".into(),
+            capability_version: 1,
+            capacity: WorkerCapacity {
+                concurrency: 1,
+                active: 0,
+            },
+            health: WorkerHealth::Healthy,
+            timestamp,
+            running_claims: BTreeSet::new(),
+        };
+        mesh.heartbeat(&worker.worker_id, heartbeat(11), 2).unwrap();
+        let subject = grant.grant.subject.clone();
+        let create = |key: &str| CreateWorkload {
+            tenant_id: "t".into(),
+            application_id: "a".into(),
+            environment: "production".into(),
+            definition_id: "extract".into(),
+            definition_revision: "r".into(),
+            trigger: "manual".into(),
+            input: serde_json::json!({"file": key}),
+            requested_by: subject.clone(),
+            session_id: None,
+            authorization_snapshot: grant.clone(),
+            capability_snapshot: ["documents.extract".into()].into(),
+            connection_scopes: BTreeSet::new(),
+            priority: 0,
+            idempotency_key: key.into(),
+            retry_policy: WorkloadRetryPolicy::default(),
+            timeouts: WorkloadTimeouts::default(),
+            dependencies: vec![],
+            not_before: None,
+            created_at: 11,
+        };
+        let first = workloads.create(create("first")).unwrap();
+        let second = workloads.create(create("second")).unwrap();
+        mesh.place_and_claim(
+            &mut workloads,
+            &first.workload_id,
+            &worker.worker_id,
+            12,
+            1000,
+        )
+        .unwrap();
+        assert_eq!(mesh.workers[&worker.worker_id].capacity.active, 1);
+        // The lease expired at 13. The server's recovery is claim recovery: it
+        // recovers the work and frees the worker, and does not judge liveness.
+        let recovered = mesh.recover_expired_work(14, &mut workloads).unwrap();
+        assert_eq!(recovered, vec![first.workload_id.clone()]);
+        let live = &mesh.workers[&worker.worker_id];
+        assert!(live.claims.is_empty());
+        assert_eq!(live.capacity.active, 0);
+        assert_eq!(live.lifecycle, WorkerLifecycle::Ready);
+        assert_eq!(
+            workloads.get(&first.workload_id).unwrap().state,
+            crate::workload::WorkloadState::Ready
+        );
+        // Its capacity is usable again.
+        mesh.place_and_claim(
+            &mut workloads,
+            &second.workload_id,
+            &worker.worker_id,
+            15,
+            10_000,
+        )
+        .unwrap();
+        // Long past its heartbeat timeout, claim recovery still leaves the
+        // worker's lifecycle alone; only `detect_failures` marks it offline.
+        mesh.recover_expired_work(1_000, &mut workloads).unwrap();
+        assert_ne!(
+            mesh.workers[&worker.worker_id].lifecycle,
+            WorkerLifecycle::Offline
+        );
+        assert!(mesh.workers[&worker.worker_id].claims.is_empty());
+        let _ = fs::remove_file(m);
+        let _ = fs::remove_file(wpath);
+    }
+    #[test]
     fn placement_claim_fencing_failure_and_recovery() {
         let (m, wpath, mut mesh, mut workloads, grant) = setup("claim");
         let worker = register(&mut mesh, grant.clone());