@feltdb/core 0.8.4 → 0.8.5

package/dist/create/server-source/crates/feltdb/src/worker_mesh.rs

@@ -891,6 +891,7 @@ mod tests {
                 retry_policy: WorkloadRetryPolicy::default(),
                 timeouts: WorkloadTimeouts::default(),
                 dependencies: vec![],
+                not_before: None,
                 created_at: 11,
             })
             .unwrap();

package/dist/create/server-source/crates/feltdb/src/workload.rs

@@ -190,6 +190,14 @@ pub struct Workload {
     pub timeouts: WorkloadTimeouts,
     pub attempt_count: u32,
     pub retry_count: u32,
+    /// The instant, in unix seconds, before which this workload must not be
+    /// claimed. `None` means "eligible as soon as it is `Ready`".
+    ///
+    /// This is the single temporal eligibility concept in the workload model.
+    /// It is what retry backoff sets after a failure, what a timeout sets before
+    /// re-arming, and what [`CreateWorkload::not_before`] sets at creation, so a
+    /// workload that is not yet executable is always not-yet-executable for the
+    /// same reason and is refused by the same check in [`WorkloadStore::claim`].
     pub next_attempt_at: Option<i64>,
     pub last_failure: Option<String>,
     pub current_claim: Option<WorkloadClaim>,
@@ -197,6 +205,19 @@ pub struct Workload {
     #[serde(default)]
     pub dependencies: Vec<WorkloadDependency>,
 }
+
+impl Workload {
+    /// Has this workload's eligibility instant arrived?
+    ///
+    /// Eligibility is only half of claimability — the state must also be
+    /// `Ready`, and [`WorkloadStore::claim`] checks both. This answers the
+    /// temporal half alone so a scheduler can skip a workload that is merely
+    /// early without attempting a claim it knows will be refused.
+    pub fn is_eligible_at(&self, now: i64) -> bool {
+        self.next_attempt_at.is_none_or(|instant| instant <= now)
+    }
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct CreateWorkload {
     pub tenant_id: String,
@@ -220,6 +241,20 @@ pub struct CreateWorkload {
     pub timeouts: WorkloadTimeouts,
     #[serde(default)]
     pub dependencies: Vec<WorkloadDependency>,
+    /// The instant, in unix seconds, before which this workload must not be
+    /// claimed. Omitted or `None` means it is claimable as soon as it is
+    /// `Ready`, which is what every workload did before this field existed.
+    ///
+    /// A workload created with `not_before` is durable immediately and
+    /// executable later: it reaches `Ready`, survives restart, and is refused to
+    /// every worker until the instant arrives. It is *not* a schedule — there is
+    /// no recurrence, no expression, and no component that re-arms it. Creating
+    /// the next occurrence is the caller's business.
+    ///
+    /// A `not_before` already in the past is not an error. It simply describes a
+    /// workload that is eligible now, which is the same thing as omitting it.
+    #[serde(default)]
+    pub not_before: Option<i64>,
     pub created_at: i64,
 }
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
@@ -338,18 +373,28 @@ impl WorkloadStore {
             timeouts: input.timeouts,
             attempt_count: 0,
             retry_count: 0,
-            next_attempt_at: None,
+            // Creation-time eligibility and retry backoff are the same thing, so
+            // they are the same field. A workload with `not_before` is created
+            // exactly as any other and is simply not yet claimable.
+            next_attempt_at: input.not_before,
             last_failure: None,
             current_claim: None,
             fencing_token: 0,
             dependencies: input.dependencies,
         };
+        // The durable history records the eligibility the workload was created
+        // with, so "why was this Ready and unclaimed for an hour" is answerable
+        // from the workload itself rather than from the caller's intent.
+        let evidence = match workload.next_attempt_at {
+            Some(instant) => [("not_before".into(), instant.to_string())].into(),
+            None => BTreeMap::new(),
+        };
         self.transition(
             &mut workload,
             WorkloadState::Ready,
             "rust",
             input.created_at,
-            BTreeMap::new(),
+            evidence,
         )?;
         self.idempotency.insert(idempotency, id.clone());
         self.workloads.insert(id, workload.clone());
@@ -425,12 +470,26 @@ impl WorkloadStore {
         lease_ms: u64,
     ) -> Result<WorkloadClaim, WorkloadError> {
         self.edit(id, |store, w| {
-            if w.state != WorkloadState::Ready || w.next_attempt_at.is_some_and(|v| v > now) {
+            if w.state != WorkloadState::Ready {
                 return Err(WorkloadError::new(
                     "WORKLOAD_NOT_READY",
                     "workload is not eligible",
                 ));
             }
+            // Separated from the state check because the two mean opposite
+            // things to a worker: a workload in the wrong state may never become
+            // claimable, while one that is merely early certainly will, at an
+            // instant the error names. Collapsing them into one code left a
+            // polling worker unable to tell "come back" from "give up".
+            if !w.is_eligible_at(now) {
+                return Err(WorkloadError::new(
+                    "WORKLOAD_NOT_YET_ELIGIBLE",
+                    format!(
+                        "workload is not claimable until {}",
+                        w.next_attempt_at.unwrap_or(now)
+                    ),
+                ));
+            }
             if !w.capability_snapshot.is_subset(capabilities) {
                 return Err(WorkloadError::new(
                     "WORKLOAD_CAPABILITY_DENIED",
@@ -943,7 +1002,14 @@ impl WorkScheduler for LocalDurableScheduler {
             .list(tenant, application)
             .into_iter()
             .filter(|w| {
-                w.state == WorkloadState::Ready && w.capability_snapshot.is_subset(capabilities)
+                // Eligibility is part of selection, not a reason to fail. A
+                // workload that is merely early is skipped so the scheduler can
+                // hand out work that is ready now; without this it would pick an
+                // ineligible workload, have `store.claim` refuse it, and return
+                // an error while claimable work sat behind it.
+                w.state == WorkloadState::Ready
+                    && w.is_eligible_at(now)
+                    && w.capability_snapshot.is_subset(capabilities)
             })
             .find(|w| {
                 w.dependencies.iter().all(|d| {
@@ -1102,6 +1168,7 @@ mod tests {
             retry_policy: Default::default(),
             timeouts: Default::default(),
             dependencies: vec![],
+            not_before: None,
             created_at: 10,
         }
     }
@@ -1307,4 +1374,445 @@ mod tests {
         assert_ne!(first.workload_id, child.workload_id);
         let _ = fs::remove_file(path);
     }
+
+    // ------------------------------------------------------------------
+    // Temporal work eligibility.
+    //
+    // The invariant under test: a workload may exist durably before it is
+    // executable. Every test below drives the real store — creating, claiming,
+    // reloading from disk — rather than asserting that a field or a function
+    // name exists.
+    // ------------------------------------------------------------------
+
+    /// A store on a path unique to this test, cleared first so a recycled
+    /// process id cannot hand a test a previous run's file.
+    fn temporal_store(name: &str) -> (PathBuf, WorkloadStore) {
+        let path = std::env::temp_dir().join(format!(
+            "workload-temporal-{name}-{}-{}.json",
+            std::process::id(),
+            rand::random::<u64>()
+        ));
+        let _ = fs::remove_file(&path);
+        (path.clone(), WorkloadStore::load(&path).unwrap())
+    }
+
+    fn scheduled(not_before: i64) -> CreateWorkload {
+        let mut input = create();
+        input.not_before = Some(not_before);
+        input
+    }
+
+    fn caps() -> BTreeSet<String> {
+        ["email:send".into()].into()
+    }
+
+    #[test]
+    fn scheduled_work_is_not_claimable_before_not_before() {
+        let (path, mut store) = temporal_store("before");
+        let workload = store.create(scheduled(2_000)).unwrap();
+
+        // Durable and Ready immediately: existing before it is executable is
+        // the whole point.
+        assert_eq!(workload.state, WorkloadState::Ready);
+        assert_eq!(workload.next_attempt_at, Some(2_000));
+
+        let refused = store
+            .claim(&workload.workload_id, "worker", &caps(), 1_999, 10_000)
+            .unwrap_err();
+        assert_eq!(refused.code, "WORKLOAD_NOT_YET_ELIGIBLE");
+        assert!(
+            refused.message.contains("2000"),
+            "the refusal names the instant the worker should return at: {}",
+            refused.message
+        );
+
+        // Refusing a claim is not a state change. Nothing executed.
+        let after = store.get(&workload.workload_id).unwrap();
+        assert_eq!(after.state, WorkloadState::Ready);
+        assert_eq!(after.attempt_count, 0);
+        assert!(after.attempts.is_empty());
+        assert!(after.current_claim.is_none());
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn scheduled_work_becomes_claimable_after_not_before() {
+        let (path, mut store) = temporal_store("after");
+        let workload = store.create(scheduled(2_000)).unwrap();
+
+        // The boundary belongs to the eligible side: at exactly `not_before`
+        // the workload is claimable, so a worker polling on the second does not
+        // have to wait a further tick.
+        let claim = store
+            .claim(&workload.workload_id, "worker", &caps(), 2_000, 10_000)
+            .expect("eligible at the instant itself");
+        assert_eq!(claim.worker_id, "worker");
+        assert_eq!(
+            store.get(&workload.workload_id).unwrap().state,
+            WorkloadState::Claimed
+        );
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn scheduled_work_survives_restart_before_eligibility() {
+        let (path, mut store) = temporal_store("restart-before");
+        let id = store.create(scheduled(2_000)).unwrap().workload_id;
+        drop(store);
+
+        // Restart the authority. The workload is still here, still Ready, still
+        // carrying its eligibility, and still refused.
+        let mut reopened = WorkloadStore::load(&path).unwrap();
+        let recovered = reopened.get(&id).unwrap();
+        assert_eq!(recovered.state, WorkloadState::Ready);
+        assert_eq!(recovered.next_attempt_at, Some(2_000));
+
+        assert_eq!(
+            reopened
+                .claim(&id, "worker", &caps(), 1_999, 10_000)
+                .unwrap_err()
+                .code,
+            "WORKLOAD_NOT_YET_ELIGIBLE"
+        );
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn scheduled_work_executes_after_restart() {
+        let (path, mut store) = temporal_store("restart-after");
+        let id = store.create(scheduled(2_000)).unwrap().workload_id;
+        drop(store);
+
+        let mut reopened = WorkloadStore::load(&path).unwrap();
+        let claim = reopened
+            .claim(&id, "worker", &caps(), 2_001, 10_000)
+            .expect("claimable after restart once eligible");
+        reopened
+            .start(&id, &claim.claim_id, "worker", claim.fencing_token, 2_002)
+            .unwrap();
+        reopened
+            .complete(
+                &id,
+                &claim.claim_id,
+                "worker",
+                claim.fencing_token,
+                "result-scheduled",
+                serde_json::json!({"ok": true}),
+                vec![],
+                1,
+                2_003,
+            )
+            .unwrap();
+
+        // And the outcome is durable across a second restart.
+        let again = WorkloadStore::load(&path).unwrap();
+        let finished = again.get(&id).unwrap();
+        assert_eq!(finished.state, WorkloadState::Succeeded);
+        assert_eq!(
+            finished.result.as_ref().map(|r| r.result_id.as_str()),
+            Some("result-scheduled")
+        );
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn immediate_work_behavior_is_unchanged() {
+        let (path, mut store) = temporal_store("immediate");
+        // `create()` sets no `not_before`, which is what every caller did
+        // before this field existed.
+        let workload = store.create(create()).unwrap();
+        assert_eq!(workload.next_attempt_at, None);
+        assert!(workload.is_eligible_at(workload.created_at));
+
+        // Claimable at its own creation instant, with no waiting.
+        store
+            .claim(
+                &workload.workload_id,
+                "worker",
+                &caps(),
+                workload.created_at,
+                10_000,
+            )
+            .expect("work with no eligibility instant is claimable immediately");
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn multiple_workers_claim_temporally_eligible_work_once() {
+        let (path, mut store) = temporal_store("contention");
+        let id = store.create(scheduled(2_000)).unwrap().workload_id;
+
+        // Both workers are polling and both see eligibility open at the same
+        // instant. Exactly one wins.
+        let winner = store
+            .claim(&id, "worker-a", &caps(), 2_000, 10_000)
+            .unwrap();
+        let loser = store
+            .claim(&id, "worker-b", &caps(), 2_000, 10_000)
+            .unwrap_err();
+        assert_eq!(
+            loser.code, "WORKLOAD_NOT_READY",
+            "the second worker is refused on state, not on time: the workload is claimed"
+        );
+
+        // And the loser cannot complete work it never owned.
+        assert_eq!(
+            store
+                .complete(
+                    &id,
+                    &winner.claim_id,
+                    "worker-b",
+                    winner.fencing_token,
+                    "duplicate",
+                    Value::Null,
+                    vec![],
+                    1,
+                    2_001,
+                )
+                .unwrap_err()
+                .code,
+            "WORKLOAD_STALE_FENCE"
+        );
+        assert_eq!(store.get(&id).unwrap().attempt_count, 1);
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn retry_sets_next_eligibility_without_duplicate_execution() {
+        let (path, mut store) = temporal_store("retry");
+        let id = store.create(scheduled(2_000)).unwrap().workload_id;
+
+        let first = store
+            .claim(&id, "worker-a", &caps(), 2_000, 10_000)
+            .unwrap();
+        store
+            .start(&id, &first.claim_id, "worker-a", first.fencing_token, 2_001)
+            .unwrap();
+        let state = store
+            .fail(
+                &id,
+                &first.claim_id,
+                "worker-a",
+                first.fencing_token,
+                "timeout",
+                "upstream did not answer",
+                2_002,
+            )
+            .unwrap();
+
+        // Retry re-arms the same eligibility field creation used, so there is
+        // one temporal concept rather than two competing ones.
+        assert_eq!(state, WorkloadState::Ready);
+        let backing_off = store.get(&id).unwrap();
+        let next = backing_off.next_attempt_at.expect("retry set eligibility");
+        assert!(
+            next > 2_002,
+            "backoff pushed eligibility into the future: {next}"
+        );
+
+        assert_eq!(
+            store
+                .claim(&id, "worker-b", &caps(), next - 1, 10_000)
+                .unwrap_err()
+                .code,
+            "WORKLOAD_NOT_YET_ELIGIBLE"
+        );
+
+        let second = store.claim(&id, "worker-b", &caps(), next, 10_000).unwrap();
+        assert!(second.fencing_token > first.fencing_token);
+
+        // The first attempt cannot land a result after the retry was handed on.
+        assert_eq!(
+            store
+                .complete(
+                    &id,
+                    &first.claim_id,
+                    "worker-a",
+                    first.fencing_token,
+                    "stale",
+                    Value::Null,
+                    vec![],
+                    1,
+                    next,
+                )
+                .unwrap_err()
+                .code,
+            "WORKLOAD_STALE_FENCE"
+        );
+        assert_eq!(store.get(&id).unwrap().attempt_count, 2);
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn cancellation_before_not_before_prevents_execution() {
+        let (path, mut store) = temporal_store("cancel");
+        let id = store.create(scheduled(2_000)).unwrap().workload_id;
+
+        // Cancelling work that is durable but not yet executable is the whole
+        // reason to be able to see it before it runs.
+        assert_eq!(
+            store.cancel(&id, "operator", 1_500).unwrap(),
+            WorkloadState::Cancelled
+        );
+
+        // Eligibility arriving does not resurrect it.
+        assert_eq!(
+            store
+                .claim(&id, "worker", &caps(), 2_001, 10_000)
+                .unwrap_err()
+                .code,
+            "WORKLOAD_NOT_READY"
+        );
+
+        let reopened = WorkloadStore::load(&path).unwrap();
+        let cancelled = reopened.get(&id).unwrap();
+        assert_eq!(cancelled.state, WorkloadState::Cancelled);
+        assert!(cancelled.attempts.is_empty(), "nothing ever executed");
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn a_worker_dying_after_a_claim_returns_the_work_and_it_executes_once() {
+        let (path, mut store) = temporal_store("worker-death");
+        let id = store.create(create()).unwrap().workload_id;
+
+        // Worker A claims a short lease, starts, and dies without completing.
+        let dead = store.claim(&id, "worker-a", &caps(), 10, 1_000).unwrap();
+        store
+            .start(&id, &dead.claim_id, "worker-a", dead.fencing_token, 10)
+            .unwrap();
+        assert_eq!(
+            store.get(&id).unwrap().state,
+            WorkloadState::Running,
+            "the work was in flight when its worker died"
+        );
+
+        // The lease expiring is the only thing that can notice.
+        let recovered = store.recover_expired(12).unwrap();
+        assert_eq!(recovered, vec![id.clone()]);
+        let back = store.get(&id).unwrap();
+        assert_eq!(back.state, WorkloadState::Ready, "returned to the pool");
+        assert_eq!(
+            back.attempts.last().map(|attempt| attempt.status),
+            Some(AttemptStatus::WorkerLost),
+            "and the dead attempt is recorded as lost rather than erased"
+        );
+
+        // Worker B picks it up and finishes it.
+        let live = store.claim(&id, "worker-b", &caps(), 12, 10_000).unwrap();
+        assert!(live.fencing_token > dead.fencing_token);
+        store
+            .start(&id, &live.claim_id, "worker-b", live.fencing_token, 13)
+            .unwrap();
+        store
+            .complete(
+                &id,
+                &live.claim_id,
+                "worker-b",
+                live.fencing_token,
+                "result-recovered",
+                serde_json::json!({"ok": true}),
+                vec![],
+                1,
+                14,
+            )
+            .unwrap();
+
+        // Exactly one execution succeeded, and the dead worker cannot add another.
+        let finished = store.get(&id).unwrap();
+        assert_eq!(finished.state, WorkloadState::Succeeded);
+        assert_eq!(
+            finished
+                .attempts
+                .iter()
+                .filter(|attempt| attempt.status == AttemptStatus::Succeeded)
+                .count(),
+            1
+        );
+        // The dead worker cannot add a second result. Its claim was released by
+        // the recovery, and completion released the successor's, so it is
+        // refused for having no claim at all rather than a stale one.
+        assert_eq!(
+            store
+                .complete(
+                    &id,
+                    &dead.claim_id,
+                    "worker-a",
+                    dead.fencing_token,
+                    "resurrected",
+                    Value::Null,
+                    vec![],
+                    1,
+                    15,
+                )
+                .unwrap_err()
+                .code,
+            "WORKLOAD_CLAIM_INVALID"
+        );
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn a_recovered_workload_refuses_its_dead_worker_while_the_successor_still_holds_it() {
+        let (path, mut store) = temporal_store("worker-death-inflight");
+        let id = store.create(create()).unwrap().workload_id;
+
+        let dead = store.claim(&id, "worker-a", &caps(), 10, 1_000).unwrap();
+        store
+            .start(&id, &dead.claim_id, "worker-a", dead.fencing_token, 10)
+            .unwrap();
+        store.recover_expired(12).unwrap();
+        let live = store.claim(&id, "worker-b", &caps(), 12, 10_000).unwrap();
+
+        // While the successor is mid-flight, the resurrected worker is refused
+        // on the fence itself — the case a lease exists to decide.
+        assert_eq!(
+            store
+                .complete(
+                    &id,
+                    &dead.claim_id,
+                    "worker-a",
+                    dead.fencing_token,
+                    "resurrected",
+                    Value::Null,
+                    vec![],
+                    1,
+                    13,
+                )
+                .unwrap_err()
+                .code,
+            "WORKLOAD_STALE_FENCE"
+        );
+        assert_eq!(store.get(&id).unwrap().state, WorkloadState::Claimed);
+        assert!(live.fencing_token > dead.fencing_token);
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn the_scheduler_skips_ineligible_work_rather_than_failing_on_it() {
+        let (path, mut store) = temporal_store("scheduler");
+        let mut early = create();
+        early.idempotency_key = "early".into();
+        early.not_before = Some(9_000);
+        let early = store.create(early).unwrap();
+
+        let mut ready = create();
+        ready.idempotency_key = "ready".into();
+        let ready = store.create(ready).unwrap();
+
+        // The ineligible workload sorts first by id in some orderings and not
+        // others, so what matters is that the scheduler never returns it and
+        // never fails because of it.
+        let mut scheduler = LocalDurableScheduler::default();
+        let claim = scheduler
+            .claim(&mut store, "t", "app", "worker", &caps(), 1_000, 10_000)
+            .expect("an ineligible workload must not fail the whole pass")
+            .expect("the eligible workload is still handed out");
+        assert_eq!(claim.workload_id, ready.workload_id);
+        assert_eq!(
+            store.get(&early.workload_id).unwrap().state,
+            WorkloadState::Ready
+        );
+        let _ = fs::remove_file(path);
+    }
 }

package/dist/create/server-source/crates/feltdb/src/workload_diagnostics.rs

@@ -231,10 +231,9 @@ impl Phase {
             // Nesting them under `AuditWrite` or `HttpHandler` would let a
             // reader compute a share of a span they do not run inside, and
             // that share could exceed one hundred per cent.
-            Phase::AuditOpen
-            | Phase::AuditAppend
-            | Phase::AuditGroupSync
-            | Phase::AuditClose => None,
+            Phase::AuditOpen | Phase::AuditAppend | Phase::AuditGroupSync | Phase::AuditClose => {
+                None
+            }
         }
     }
 }