@feltdb/core 0.9.0 → 0.10.0

package/dist/create/server-source/crates/feltdb/src/workload.rs

@@ -1,12 +1,16 @@
 //! Durable workload authority. Workers execute; this state machine decides.
-use crate::authorization::{SignedGrant, Subject};
+use crate::{
+    authorization::{SignedGrant, Subject},
+    AtomicMutation, FeltDb, FlowError, RecordPrecondition,
+};
+use rand::{distributions::Alphanumeric, Rng};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use sha2::{Digest, Sha256};
 use std::collections::{BTreeMap, BTreeSet};
-use std::fs;
-use std::path::{Path, PathBuf};
+use std::path::Path;
 pub const WORKLOAD_CONTRACT_VERSION: u32 = 1;
+pub const DURABLE_WORK_COLLECTION: &str = "system_durable_work";
 
 #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
 #[serde(rename_all = "SCREAMING_SNAKE_CASE")]
@@ -174,6 +178,8 @@ pub struct Workload {
     pub input: Value,
     pub input_hash: String,
     pub requested_by: Subject,
+    #[serde(default)]
+    pub session_id: Option<String>,
     pub authorization_snapshot: SignedGrant,
     pub capability_snapshot: BTreeSet<String>,
     pub connection_scopes: BTreeSet<String>,
@@ -228,6 +234,8 @@ pub struct CreateWorkload {
     pub trigger: String,
     pub input: Value,
     pub requested_by: Subject,
+    #[serde(default)]
+    pub session_id: Option<String>,
     pub authorization_snapshot: SignedGrant,
     pub capability_snapshot: BTreeSet<String>,
     #[serde(default)]
@@ -277,12 +285,21 @@ impl std::fmt::Display for WorkloadError {
 }
 impl std::error::Error for WorkloadError {}
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Clone)]
 pub struct WorkloadStore {
-    #[serde(skip)]
-    path: PathBuf,
-    #[serde(default)]
+    db: FeltDb,
     pub workloads: BTreeMap<String, Workload>,
+    idempotency: BTreeMap<String, String>,
+    result_ids: BTreeMap<String, String>,
+    /// Rebuildable cache of FeltDB-owned document versions. This is never
+    /// serialized and never advances independently of a canonical commit.
+    document_versions: BTreeMap<String, u64>,
+}
+
+#[derive(Serialize, Deserialize)]
+struct LegacyWorkloadStore {
+    #[serde(default)]
+    workloads: BTreeMap<String, Workload>,
     #[serde(default)]
     idempotency: BTreeMap<String, String>,
     #[serde(default)]
@@ -290,29 +307,151 @@ pub struct WorkloadStore {
     #[serde(default)]
     event_sequence: u64,
 }
+
 impl WorkloadStore {
     pub fn load(path: impl AsRef<Path>) -> Result<Self, WorkloadError> {
-        let path = path.as_ref().to_path_buf();
-        if !path.exists() {
-            return Ok(Self {
-                path,
-                workloads: BTreeMap::new(),
-                idempotency: BTreeMap::new(),
-                result_ids: BTreeMap::new(),
-                event_sequence: 0,
-            });
+        Self::from_db(FeltDb::open(path).map_err(io)?)
+    }
+
+    pub fn from_db(db: FeltDb) -> Result<Self, WorkloadError> {
+        let mut workloads = BTreeMap::new();
+        let mut document_versions = BTreeMap::new();
+        for row in db.list_collection(DURABLE_WORK_COLLECTION).map_err(io)? {
+            let version = row
+                .value
+                .get("__version")
+                .and_then(Value::as_u64)
+                // Canonical FeltDB CAS defines a pre-versioned document as
+                // version 1. The next guarded mutation writes version 2, which
+                // upgrades rows created before Work adopted RecordPrecondition.
+                .unwrap_or(1);
+            let workload: Workload = serde_json::from_value(row.value).map_err(io)?;
+            document_versions.insert(workload.workload_id.clone(), version);
+            workloads.insert(workload.workload_id.clone(), workload);
+        }
+        let mut idempotency = BTreeMap::new();
+        let mut result_ids = BTreeMap::new();
+        for workload in workloads.values() {
+            idempotency.insert(
+                format!(
+                    "{}:{}:{}",
+                    workload.tenant_id, workload.application_id, workload.idempotency_key
+                ),
+                workload.workload_id.clone(),
+            );
+            if let Some(result) = &workload.result {
+                result_ids.insert(result.result_id.clone(), workload.workload_id.clone());
+            }
         }
-        let mut value: Self = serde_json::from_slice(&fs::read(&path).map_err(io)?).map_err(io)?;
-        value.path = path;
-        Ok(value)
+        Ok(Self {
+            db,
+            workloads,
+            idempotency,
+            result_ids,
+            document_versions,
+        })
     }
-    fn persist(&self) -> Result<(), WorkloadError> {
-        if let Some(parent) = self.path.parent() {
-            fs::create_dir_all(parent).map_err(io)?;
+
+    pub fn migrate_legacy_file(db: FeltDb, path: &Path) -> Result<Self, WorkloadError> {
+        let bytes = std::fs::read(path).map_err(io)?;
+        let legacy: LegacyWorkloadStore = serde_json::from_slice(&bytes).map_err(io)?;
+        let mut store = Self::from_db(db)?;
+        for (id, workload) in legacy.workloads {
+            if id != workload.workload_id {
+                return Err(WorkloadError::new(
+                    "WORKLOAD_MIGRATION_CONFLICT",
+                    format!(
+                        "legacy key {id} does not match workload identity {}",
+                        workload.workload_id
+                    ),
+                ));
+            }
+            if let Some(existing) = store.workloads.get(&id) {
+                if serde_json::to_value(existing).map_err(io)?
+                    != serde_json::to_value(&workload).map_err(io)?
+                {
+                    return Err(WorkloadError::new(
+                        "WORKLOAD_MIGRATION_CONFLICT",
+                        format!("canonical workload {id} differs from legacy source"),
+                    ));
+                }
+                continue;
+            }
+            store.workloads.insert(id.clone(), workload);
+            store.persist_workload(&id, "workload_migration")?;
         }
-        let temp = self.path.with_extension("tmp");
-        fs::write(&temp, serde_json::to_vec_pretty(self).map_err(io)?).map_err(io)?;
-        fs::rename(temp, &self.path).map_err(io)
+        // The maps are verification evidence, not migrated authorities.
+        let rebuilt = Self::from_db(store.db.clone())?;
+        if legacy.idempotency != rebuilt.idempotency
+            || legacy.result_ids != rebuilt.result_ids
+            || legacy.event_sequence != derived_event_sequence(&rebuilt.workloads)
+        {
+            return Err(WorkloadError::new(
+                "WORKLOAD_MIGRATION_CONFLICT",
+                "legacy derived indexes do not match canonical work aggregates",
+            ));
+        }
+        Ok(rebuilt)
+    }
+
+    fn persist_workload(&mut self, id: &str, action: &str) -> Result<(), WorkloadError> {
+        let workload = self
+            .workloads
+            .get(id)
+            .ok_or_else(|| WorkloadError::new("WORKLOAD_NOT_FOUND", "workload not found"))?;
+        let key = work_key(workload);
+        let expected_version = self.document_versions.get(id).copied();
+        let transaction_id = runtime_id("work_tx");
+        let aggregate_digest = hash(&serde_json::to_value(workload).map_err(io)?);
+        let commit = self
+            .db
+            .apply_atomic_transaction_guarded(
+                &transaction_id,
+                None,
+                None,
+                &[],
+                &[RecordPrecondition {
+                    capability: DURABLE_WORK_COLLECTION.into(),
+                    key: key.clone(),
+                    require_absent: expected_version.is_none(),
+                    expected_version,
+                    expected_epoch: None,
+                    expected_lease_id: None,
+                }],
+                &[AtomicMutation {
+                    capability: DURABLE_WORK_COLLECTION.into(),
+                    key,
+                    value: Some(serde_json::to_value(workload).map_err(io)?),
+                }],
+                Some(serde_json::json!({
+                    "kind": action,
+                    "subject": workload.requested_by.id(),
+                    "principal_id": workload.requested_by.id(),
+                    "agent_id": (workload.requested_by.kind() == "agent").then(|| workload.requested_by.id()),
+                    "session_id": workload.session_id,
+                    "delegator_id": workload.authorization_snapshot.grant.issuer.id(),
+                    "authorization_decision_id": workload.authorization_snapshot.grant.id,
+                    "tenant": workload.tenant_id,
+                    "application": workload.application_id,
+                    "work_id": workload.workload_id,
+                    "operation_id": transaction_id,
+                    "attempt_id": workload.attempts.last().map(|attempt| attempt.attempt_id.as_str()),
+                    "worker_id": workload.current_claim.as_ref().map(|claim| claim.worker_id.as_str()),
+                    "result_id": workload.result.as_ref().map(|result| result.result_id.as_str()),
+                    "source_kind": (action == "workload_migration").then_some("WorkloadStore"),
+                    "source_digest": aggregate_digest,
+                    "migration_outcome": (action == "workload_migration").then_some("imported"),
+                })),
+            )
+            .map_err(workload_storage_error)?;
+        let version = commit
+            .rows
+            .first()
+            .and_then(|row| row.value.get("__version"))
+            .and_then(Value::as_u64)
+            .ok_or_else(|| io("canonical work commit has no document version"))?;
+        self.document_versions.insert(id.into(), version);
+        Ok(())
     }
     pub fn create(&mut self, input: CreateWorkload) -> Result<Workload, WorkloadError> {
         ensure_safe(&input.input)?;
@@ -338,12 +477,11 @@ impl WorkloadStore {
                 "authorization snapshot does not bind the requester",
             ));
         }
-        let seed = format!(
-            "{idempotency}:{}:{}",
-            input.definition_revision,
-            hash(&input.input)
-        );
-        let id = format!("workload_{}", &hex(&Sha256::digest(seed))[..24]);
+        // The scoped idempotency identity is the record identity. Competing
+        // admissions therefore contend on one canonical require-absent CAS
+        // even when their payloads differ; no separate durable uniqueness
+        // index is needed.
+        let id = format!("workload_{}", &hex(&Sha256::digest(&idempotency))[..24]);
         let input_hash = hash(&input.input);
         let mut workload = Workload {
             contract_version: 1,
@@ -357,6 +495,7 @@ impl WorkloadStore {
             input: input.input,
             input_hash,
             requested_by: input.requested_by,
+            session_id: input.session_id,
             authorization_snapshot: input.authorization_snapshot,
             capability_snapshot: input.capability_snapshot,
             connection_scopes: input.connection_scopes,
@@ -396,9 +535,13 @@ impl WorkloadStore {
             input.created_at,
             evidence,
         )?;
-        self.idempotency.insert(idempotency, id.clone());
-        self.workloads.insert(id, workload.clone());
-        self.persist()?;
+        self.idempotency.insert(idempotency.clone(), id.clone());
+        self.workloads.insert(id.clone(), workload.clone());
+        if let Err(error) = self.persist_workload(&id, "workload.created") {
+            self.workloads.remove(&id);
+            self.idempotency.remove(&idempotency);
+            return Err(error);
+        }
         Ok(workload)
     }
     pub fn get(&self, id: &str) -> Result<&Workload, WorkloadError> {
@@ -421,16 +564,19 @@ impl WorkloadStore {
         id: &str,
         action: impl FnOnce(&mut Self, &mut Workload) -> Result<R, WorkloadError>,
     ) -> Result<R, WorkloadError> {
-        let mut workload = self
+        let workload = self
             .workloads
-            .remove(id)
+            .get(id)
+            .cloned()
             .ok_or_else(|| WorkloadError::new("WORKLOAD_NOT_FOUND", "workload not found"))?;
-        let result = action(self, &mut workload);
-        self.workloads.insert(id.into(), workload);
-        if result.is_ok() {
-            self.persist()?;
+        let mut edited = workload.clone();
+        let result = action(self, &mut edited)?;
+        self.workloads.insert(id.into(), edited);
+        if let Err(error) = self.persist_workload(id, "workload.mutated") {
+            self.workloads.insert(id.into(), workload);
+            return Err(error);
         }
-        result
+        Ok(result)
     }
     fn transition(
         &mut self,
@@ -449,9 +595,12 @@ impl WorkloadStore {
         let from = w.state;
         w.state = to;
         w.updated_at = at;
-        self.event_sequence += 1;
+        let sequence = w
+            .execution_history
+            .last()
+            .map_or(1, |event| event.sequence + 1);
         w.execution_history.push(WorkloadEvent {
-            sequence: self.event_sequence,
+            sequence,
             event_type: event_name(to).into(),
             from: Some(from),
             to,
@@ -730,7 +879,6 @@ impl WorkloadStore {
             Ok(result)
         })?;
         self.result_ids.insert(result_id.into(), id.into());
-        self.persist()?;
         Ok(result)
     }
     pub fn fail(
@@ -1111,9 +1259,42 @@ fn ensure_safe(value: &Value) -> Result<(), WorkloadError> {
 fn sensitive_text(v: &str) -> bool {
     v.contains("BEGIN PRIVATE KEY") || v.starts_with("sk-") || v.starts_with("fdb_live_")
 }
+fn work_key(workload: &Workload) -> String {
+    format!(
+        "{}/{}/{}",
+        workload.tenant_id, workload.application_id, workload.workload_id
+    )
+}
+fn derived_event_sequence(workloads: &BTreeMap<String, Workload>) -> u64 {
+    workloads
+        .values()
+        .flat_map(|workload| workload.execution_history.iter())
+        .map(|event| event.sequence)
+        .max()
+        .unwrap_or_default()
+}
+fn runtime_id(prefix: &str) -> String {
+    let value: String = rand::thread_rng()
+        .sample_iter(&Alphanumeric)
+        .take(24)
+        .map(char::from)
+        .collect();
+    format!("{prefix}_{value}")
+}
 fn io(e: impl std::fmt::Display) -> WorkloadError {
     WorkloadError::new("WORKLOAD_STORAGE_FAILURE", e.to_string())
 }
+fn workload_storage_error(error: FlowError) -> WorkloadError {
+    match error {
+        FlowError::PreconditionFailed(_) | FlowError::RevisionConflict { .. } => {
+            WorkloadError::new("WORKLOAD_CONFLICT", error.to_string())
+        }
+        FlowError::CapabilityError(ref message) if message.starts_with("PRECONDITION_FAILED:") => {
+            WorkloadError::new("WORKLOAD_CONFLICT", error.to_string())
+        }
+        _ => io(error),
+    }
+}
 fn hash(value: &Value) -> String {
     hex(&Sha256::digest(
         serde_json::to_vec(value).unwrap_or_default(),
@@ -1129,6 +1310,7 @@ mod tests {
     use crate::authorization::{
         Grant, GrantConstraints, GrantSigner, GrantStatus, ResourceScope, ResourceScopeKind,
     };
+    use std::{fs, path::PathBuf};
     fn create() -> CreateWorkload {
         let signer = GrantSigner::new("k", vec![1; 32]).unwrap();
         let subject = Subject::Agent("a1".into());
@@ -1160,6 +1342,7 @@ mod tests {
             trigger: "manual".into(),
             input: serde_json::json!({"id":1}),
             requested_by: subject,
+            session_id: Some("session-1".into()),
             authorization_snapshot: signer.sign(grant).unwrap(),
             capability_snapshot: ["email:send".into()].into(),
             connection_scopes: BTreeSet::new(),
@@ -1179,6 +1362,259 @@ mod tests {
         (path, store)
     }
     #[test]
+    fn legacy_migration_is_idempotent_conflict_detecting_and_source_independent() {
+        let (source_db, mut source) = store("legacy-source-builder");
+        let workload = source.create(create()).unwrap();
+        let legacy = LegacyWorkloadStore {
+            workloads: source.workloads.clone(),
+            idempotency: source.idempotency.clone(),
+            result_ids: source.result_ids.clone(),
+            event_sequence: derived_event_sequence(&source.workloads),
+        };
+        let legacy_path = source_db.with_extension("legacy-workloads.json");
+        fs::write(&legacy_path, serde_json::to_vec(&legacy).unwrap()).unwrap();
+        let canonical_path = source_db.with_extension("canonical.log");
+        let db = FeltDb::open(&canonical_path).unwrap();
+        let migrated = WorkloadStore::migrate_legacy_file(db.clone(), &legacy_path).unwrap();
+        assert_eq!(
+            migrated.get(&workload.workload_id).unwrap().input_hash,
+            workload.input_hash
+        );
+        WorkloadStore::migrate_legacy_file(db.clone(), &legacy_path).unwrap();
+        fs::remove_file(&legacy_path).unwrap();
+        let reopened = WorkloadStore::from_db(FeltDb::open(&canonical_path).unwrap()).unwrap();
+        assert_eq!(
+            reopened.get(&workload.workload_id).unwrap().workload_id,
+            workload.workload_id
+        );
+
+        let mut conflicting = legacy;
+        conflicting
+            .workloads
+            .get_mut(&workload.workload_id)
+            .unwrap()
+            .priority += 1;
+        fs::write(&legacy_path, serde_json::to_vec(&conflicting).unwrap()).unwrap();
+        assert_eq!(
+            WorkloadStore::migrate_legacy_file(db, &legacy_path)
+                .err()
+                .unwrap()
+                .code,
+            "WORKLOAD_MIGRATION_CONFLICT"
+        );
+        let _ = fs::remove_file(source_db);
+        let _ = fs::remove_file(canonical_path);
+        let _ = fs::remove_file(legacy_path);
+    }
+
+    #[test]
+    fn canonical_history_reconstructs_work_identity_provenance() {
+        let (path, mut store) = store("provenance");
+        let workload = store.create(create()).unwrap();
+        let transaction = store
+            .db
+            .committed_transactions_after(0, usize::MAX)
+            .unwrap()
+            .into_iter()
+            .last()
+            .unwrap();
+        let audit = transaction.audit.unwrap();
+        assert_eq!(audit["work_id"], workload.workload_id);
+        assert_eq!(audit["session_id"], workload.session_id.as_deref().unwrap());
+        assert_eq!(audit["agent_id"], workload.requested_by.id());
+        assert_eq!(
+            audit["delegator_id"],
+            workload.authorization_snapshot.grant.issuer.id()
+        );
+        assert!(transaction.commit_revision > transaction.base_revision);
+        assert_eq!(transaction.work_id.as_deref(), Some(workload.workload_id.as_str()));
+        assert_eq!(transaction.agent_id.as_deref(), Some(workload.requested_by.id()));
+        assert_eq!(transaction.session_id, workload.session_id);
+        assert_eq!(
+            transaction.delegator_id.as_deref(),
+            Some(workload.authorization_snapshot.grant.issuer.id())
+        );
+        assert_eq!(transaction.operation_ids.len(), 1);
+        assert_ne!(transaction.operation_ids[0], transaction.transaction_id);
+        drop(store);
+        let reopened = WorkloadStore::load(&path).unwrap();
+        assert_eq!(
+            reopened.get(&workload.workload_id).unwrap().workload_id,
+            workload.workload_id
+        );
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn competing_writers_use_canonical_document_cas_before_and_after_restart() {
+        let (path, mut seed) = store("canonical-cas");
+        let workload = seed.create(create()).unwrap();
+        let id = workload.workload_id.clone();
+        let db = seed.db.clone();
+        let caps = ["email:send".into()].into();
+
+        let mut winner = WorkloadStore::from_db(db.clone()).unwrap();
+        let mut loser = WorkloadStore::from_db(db.clone()).unwrap();
+        winner.claim(&id, "worker-a", &caps, 11, 1_000).unwrap();
+        let conflict = loser.claim(&id, "worker-b", &caps, 11, 1_000).unwrap_err();
+        assert_eq!(conflict.code, "WORKLOAD_CONFLICT");
+        assert_eq!(loser.get(&id).unwrap().state, WorkloadState::Ready);
+
+        let row = db
+            .list_collection(DURABLE_WORK_COLLECTION)
+            .unwrap()
+            .into_iter()
+            .find(|row| row.key.ends_with(&id))
+            .unwrap();
+        assert_eq!(row.value["__version"], 2);
+
+        drop((seed, winner, loser, db));
+        let restarted_db = FeltDb::open(&path).unwrap();
+        let mut recovery = WorkloadStore::from_db(restarted_db.clone()).unwrap();
+        recovery.recover_expired(13).unwrap();
+        let mut restarted_winner = WorkloadStore::from_db(restarted_db.clone()).unwrap();
+        let mut restarted_loser = WorkloadStore::from_db(restarted_db.clone()).unwrap();
+        restarted_winner
+            .claim(&id, "worker-c", &caps, 13, 10_000)
+            .unwrap();
+        let conflict = restarted_loser
+            .claim(&id, "worker-d", &caps, 13, 10_000)
+            .unwrap_err();
+        assert_eq!(conflict.code, "WORKLOAD_CONFLICT");
+        assert_eq!(
+            restarted_loser.get(&id).unwrap().state,
+            WorkloadState::Ready
+        );
+        drop((recovery, restarted_winner, restarted_loser, restarted_db));
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn competing_admissions_share_one_canonical_idempotency_identity() {
+        let path = std::env::temp_dir().join(format!(
+            "workload-idempotency-cas-{}.log",
+            std::process::id()
+        ));
+        let db = FeltDb::open(&path).unwrap();
+        let mut first = WorkloadStore::from_db(db.clone()).unwrap();
+        let mut second = WorkloadStore::from_db(db.clone()).unwrap();
+        let accepted = first.create(create()).unwrap();
+        let mut competing = create();
+        competing.input = serde_json::json!({"id": 2});
+        let conflict = second.create(competing).unwrap_err();
+        assert_eq!(conflict.code, "WORKLOAD_CONFLICT");
+        assert!(
+            second.workloads.is_empty(),
+            "failed projection was rolled back"
+        );
+
+        drop((first, second, db));
+        let restarted = WorkloadStore::from_db(FeltDb::open(&path).unwrap()).unwrap();
+        assert_eq!(restarted.workloads.len(), 1);
+        assert_eq!(restarted.idempotency["t:app:one"], accepted.workload_id);
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn pre_versioned_canonical_rows_upgrade_through_the_standard_cas_contract() {
+        let (path, mut store) = store("pre-versioned-upgrade");
+        let workload = store.create(create()).unwrap();
+        let db = store.db.clone();
+        db.apply_atomic_transaction(
+            "fixture:pre-versioned-work",
+            None,
+            &[],
+            &[AtomicMutation {
+                capability: DURABLE_WORK_COLLECTION.into(),
+                key: work_key(&workload),
+                value: Some(serde_json::to_value(&workload).unwrap()),
+            }],
+            None,
+        )
+        .unwrap();
+
+        let mut upgraded = WorkloadStore::from_db(db.clone()).unwrap();
+        upgraded
+            .claim(
+                &workload.workload_id,
+                "worker",
+                &["email:send".into()].into(),
+                11,
+                10_000,
+            )
+            .unwrap();
+        let row = db
+            .list_collection(DURABLE_WORK_COLLECTION)
+            .unwrap()
+            .into_iter()
+            .find(|row| row.key.ends_with(&workload.workload_id))
+            .unwrap();
+        assert_eq!(row.value["__version"], 2);
+        let _ = fs::remove_file(path);
+    }
+
+    #[test]
+    fn derived_indexes_rebuild_exclusively_from_canonical_work() {
+        let (path, mut store) = store("derived-index-rebuild");
+        let workload = store.create(create()).unwrap();
+        let caps = ["email:send".into()].into();
+        let claim = store
+            .claim(&workload.workload_id, "worker", &caps, 11, 10_000)
+            .unwrap();
+        store
+            .start(
+                &workload.workload_id,
+                &claim.claim_id,
+                "worker",
+                claim.fencing_token,
+                12,
+            )
+            .unwrap();
+        store
+            .complete(
+                &workload.workload_id,
+                &claim.claim_id,
+                "worker",
+                claim.fencing_token,
+                "result-rebuilt",
+                serde_json::json!({"ok": true}),
+                vec![],
+                1,
+                13,
+            )
+            .unwrap();
+
+        store.idempotency.clear();
+        store.result_ids.clear();
+        store.document_versions.clear();
+        let rebuilt = WorkloadStore::from_db(store.db.clone()).unwrap();
+        assert_eq!(
+            rebuilt.idempotency.get("t:app:one"),
+            Some(&workload.workload_id)
+        );
+        assert_eq!(
+            rebuilt.result_ids.get("result-rebuilt"),
+            Some(&workload.workload_id)
+        );
+        assert!(rebuilt.document_versions[&workload.workload_id] > 1);
+        assert_eq!(
+            rebuilt
+                .get(&workload.workload_id)
+                .unwrap()
+                .execution_history
+                .iter()
+                .map(|event| event.sequence)
+                .collect::<Vec<_>>(),
+            vec![1, 2, 3, 4]
+        );
+        let duplicate = WorkloadStore::from_db(store.db.clone())
+            .unwrap()
+            .create(create())
+            .unwrap();
+        assert_eq!(duplicate.workload_id, workload.workload_id);
+        let _ = fs::remove_file(path);
+    }
+    #[test]
     fn idempotent_create_and_legal_lifecycle() {
         let (path, mut store) = store("lifecycle");
         let one = store.create(create()).unwrap();