@feltdb/core 0.15.0 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (247) hide show
  1. package/dist/agent-state.d.ts +20 -0
  2. package/dist/agent-state.d.ts.map +1 -1
  3. package/dist/agent-state.js +36 -0
  4. package/dist/build-identity.json +12 -4
  5. package/dist/cli/commands.js +187 -4
  6. package/dist/cli/publish-engine.js +116 -16
  7. package/dist/collection.d.ts.map +1 -1
  8. package/dist/collection.js +3 -0
  9. package/dist/create/build-identity.json +12 -4
  10. package/dist/create/managed-account.js +93 -35
  11. package/dist/create/package-versions.js +1 -1
  12. package/dist/create/server-source/Cargo.lock +6 -509
  13. package/dist/create/server-source/Cargo.toml +2 -13
  14. package/dist/create/server-source/crates/feltdb-client/Cargo.toml +1 -1
  15. package/dist/create/server-source/crates/feltdb-server/Cargo.toml +1 -1
  16. package/dist/create/server-source/crates/feltdb-server/src/main.rs +255 -12
  17. package/dist/create/server-source/crates/feltdb-wasm/Cargo.toml +1 -1
  18. package/dist/db.d.ts +11 -1
  19. package/dist/db.d.ts.map +1 -1
  20. package/dist/db.js +15 -2
  21. package/dist/error-codes.d.ts +5 -0
  22. package/dist/error-codes.d.ts.map +1 -1
  23. package/dist/error-codes.js +9 -1
  24. package/dist/http-db.d.ts +16 -0
  25. package/dist/http-db.d.ts.map +1 -1
  26. package/dist/http-db.js +36 -6
  27. package/dist/managed-recovery.d.ts +6 -0
  28. package/dist/managed-recovery.d.ts.map +1 -1
  29. package/dist/managed-recovery.js +9 -0
  30. package/dist/studio-app/assets/{feltdb_wasm-BjM1ylL3.js → feltdb_wasm-DtybUY0q.js} +1 -1
  31. package/dist/studio-app/assets/feltdb_wasm_bg-DsgIUwaB.wasm +0 -0
  32. package/dist/studio-app/assets/index-C4OffsZl.js +37 -0
  33. package/dist/studio-app/index.html +1 -1
  34. package/dist/wasm/feltdb_wasm_bg.wasm +0 -0
  35. package/dist/wasm/package.json +1 -1
  36. package/dist/workload.d.ts +55 -3
  37. package/dist/workload.d.ts.map +1 -1
  38. package/dist/workload.js +23 -0
  39. package/dist/workspace/workspace-connection.d.ts.map +1 -1
  40. package/dist/workspace/workspace-connection.js +14 -3
  41. package/package.json +1 -1
  42. package/dist/create/server-source/crates/feltdb/Cargo.lock +0 -175
  43. package/dist/create/server-source/crates/feltdb/Cargo.toml +0 -102
  44. package/dist/create/server-source/crates/feltdb/LICENSE +0 -21
  45. package/dist/create/server-source/crates/feltdb/README.md +0 -75
  46. package/dist/create/server-source/crates/feltdb/benches/baselines/gate-13-redux.json +0 -370
  47. package/dist/create/server-source/crates/feltdb/benches/gate13_baseline.rs +0 -589
  48. package/dist/create/server-source/crates/feltdb/benches/gate13_phase_7_1_release_economics.rs +0 -247
  49. package/dist/create/server-source/crates/feltdb/benches/gate_13_redux.rs +0 -440
  50. package/dist/create/server-source/crates/feltdb/benches/gate_13_regression_runner.rs +0 -275
  51. package/dist/create/server-source/crates/feltdb/benches/gate_14a_concurrent_writer_scaling.rs +0 -381
  52. package/dist/create/server-source/crates/feltdb/benches/gate_14a_production_admission_revalidation.rs +0 -467
  53. package/dist/create/server-source/crates/feltdb/benches/gate_14a_rc2_admission_contract.rs +0 -489
  54. package/dist/create/server-source/crates/feltdb/benches/gate_14a_rc_root_cause.rs +0 -281
  55. package/dist/create/server-source/crates/feltdb/benches/gate_14a_sync1_queued_prototype.rs +0 -606
  56. package/dist/create/server-source/crates/feltdb/benches/gate_14a_sync_economics.rs +0 -531
  57. package/dist/create/server-source/crates/feltdb/benches/gate_14b_causal_backlog_scaling.rs +0 -462
  58. package/dist/create/server-source/crates/feltdb/benches/gate_14c_replication_contract_test.rs +0 -505
  59. package/dist/create/server-source/crates/feltdb/benches/gate_14c_replication_scaling.rs +0 -484
  60. package/dist/create/server-source/crates/feltdb/benches/gate_14d_combined_dimension_scaling.rs +0 -758
  61. package/dist/create/server-source/crates/feltdb/benches/phase_7_1_2_optimization_benchmark.rs +0 -421
  62. package/dist/create/server-source/crates/feltdb/benches/phase_7_1_3_crossover_analysis.rs +0 -329
  63. package/dist/create/server-source/crates/feltdb/src/acceptance_tests.rs +0 -1705
  64. package/dist/create/server-source/crates/feltdb/src/acquisition.rs +0 -286
  65. package/dist/create/server-source/crates/feltdb/src/admission.rs +0 -185
  66. package/dist/create/server-source/crates/feltdb/src/admission_contract_tests.rs +0 -507
  67. package/dist/create/server-source/crates/feltdb/src/adversarial_transport.rs +0 -539
  68. package/dist/create/server-source/crates/feltdb/src/agent_state.rs +0 -320
  69. package/dist/create/server-source/crates/feltdb/src/agent_transition_contract.rs +0 -1545
  70. package/dist/create/server-source/crates/feltdb/src/analytics.rs +0 -521
  71. package/dist/create/server-source/crates/feltdb/src/application.rs +0 -2674
  72. package/dist/create/server-source/crates/feltdb/src/application_runtime.rs +0 -1070
  73. package/dist/create/server-source/crates/feltdb/src/authority.rs +0 -479
  74. package/dist/create/server-source/crates/feltdb/src/authority_failover.rs +0 -847
  75. package/dist/create/server-source/crates/feltdb/src/authorization.rs +0 -1030
  76. package/dist/create/server-source/crates/feltdb/src/authorization_security_tests.rs +0 -1122
  77. package/dist/create/server-source/crates/feltdb/src/bin/feltdb-authority-client.rs +0 -32
  78. package/dist/create/server-source/crates/feltdb/src/bin/feltdb-authority.rs +0 -19
  79. package/dist/create/server-source/crates/feltdb/src/bin/feltdb_node.rs +0 -1474
  80. package/dist/create/server-source/crates/feltdb/src/capabilities/mod.rs +0 -8
  81. package/dist/create/server-source/crates/feltdb/src/capabilities/search.rs +0 -418
  82. package/dist/create/server-source/crates/feltdb/src/capabilities/vector.rs +0 -482
  83. package/dist/create/server-source/crates/feltdb/src/capability.rs +0 -903
  84. package/dist/create/server-source/crates/feltdb/src/cardinality_diagnostics.rs +0 -263
  85. package/dist/create/server-source/crates/feltdb/src/cardinality_endpoint.rs +0 -77
  86. package/dist/create/server-source/crates/feltdb/src/causal_backlog_bound.rs +0 -496
  87. package/dist/create/server-source/crates/feltdb/src/causal_dependency_barrier.rs +0 -2574
  88. package/dist/create/server-source/crates/feltdb/src/causal_dependency_barrier_phase_7_1.rs +0 -212
  89. package/dist/create/server-source/crates/feltdb/src/concurrency_fuzzing.rs +0 -422
  90. package/dist/create/server-source/crates/feltdb/src/consistency_contract.rs +0 -445
  91. package/dist/create/server-source/crates/feltdb/src/content_distribution.rs +0 -465
  92. package/dist/create/server-source/crates/feltdb/src/convergence.rs +0 -634
  93. package/dist/create/server-source/crates/feltdb/src/crash_atomic_boundary.rs +0 -427
  94. package/dist/create/server-source/crates/feltdb/src/crash_injection.rs +0 -376
  95. package/dist/create/server-source/crates/feltdb/src/crash_recovery_tests.rs +0 -365
  96. package/dist/create/server-source/crates/feltdb/src/cron.rs +0 -294
  97. package/dist/create/server-source/crates/feltdb/src/dedup_bound_investigation.rs +0 -483
  98. package/dist/create/server-source/crates/feltdb/src/distributed_indexing.rs +0 -477
  99. package/dist/create/server-source/crates/feltdb/src/distributed_tests.rs +0 -1003
  100. package/dist/create/server-source/crates/feltdb/src/distributed_transactions.rs +0 -1334
  101. package/dist/create/server-source/crates/feltdb/src/durability_guarantees.rs +0 -368
  102. package/dist/create/server-source/crates/feltdb/src/durable_dedup_set.rs +0 -231
  103. package/dist/create/server-source/crates/feltdb/src/durable_operation_identity.rs +0 -482
  104. package/dist/create/server-source/crates/feltdb/src/durable_operation_log.rs +0 -203
  105. package/dist/create/server-source/crates/feltdb/src/durable_sync.rs +0 -317
  106. package/dist/create/server-source/crates/feltdb/src/equality_index.rs +0 -595
  107. package/dist/create/server-source/crates/feltdb/src/execution.rs +0 -426
  108. package/dist/create/server-source/crates/feltdb/src/in_process_transport.rs +0 -214
  109. package/dist/create/server-source/crates/feltdb/src/indexing.rs +0 -776
  110. package/dist/create/server-source/crates/feltdb/src/lib.rs +0 -7365
  111. package/dist/create/server-source/crates/feltdb/src/managed_cas_tests.rs +0 -234
  112. package/dist/create/server-source/crates/feltdb/src/materialization.rs +0 -184
  113. package/dist/create/server-source/crates/feltdb/src/metrics.rs +0 -266
  114. package/dist/create/server-source/crates/feltdb/src/multi_node_convergence.rs +0 -310
  115. package/dist/create/server-source/crates/feltdb/src/multi_operation_transaction.rs +0 -736
  116. package/dist/create/server-source/crates/feltdb/src/observability.rs +0 -379
  117. package/dist/create/server-source/crates/feltdb/src/operation.rs +0 -290
  118. package/dist/create/server-source/crates/feltdb/src/operation_algebra.rs +0 -439
  119. package/dist/create/server-source/crates/feltdb/src/operation_log.rs +0 -349
  120. package/dist/create/server-source/crates/feltdb/src/p1_application_atomicity.rs +0 -332
  121. package/dist/create/server-source/crates/feltdb/src/p1_atomicity_acceptance.rs +0 -718
  122. package/dist/create/server-source/crates/feltdb/src/partition_reconciliation.rs +0 -487
  123. package/dist/create/server-source/crates/feltdb/src/peer_registry.rs +0 -166
  124. package/dist/create/server-source/crates/feltdb/src/permutation_scheduler.rs +0 -289
  125. package/dist/create/server-source/crates/feltdb/src/persistence_reality.rs +0 -566
  126. package/dist/create/server-source/crates/feltdb/src/phase1b_acceptance.rs +0 -3391
  127. package/dist/create/server-source/crates/feltdb/src/phase1c1_acceptance.rs +0 -203
  128. package/dist/create/server-source/crates/feltdb/src/phase1c2_acceptance.rs +0 -261
  129. package/dist/create/server-source/crates/feltdb/src/phase1c3_acceptance.rs +0 -502
  130. package/dist/create/server-source/crates/feltdb/src/phase1c_atomicity_proof.rs +0 -216
  131. package/dist/create/server-source/crates/feltdb/src/phase5_integration.rs +0 -303
  132. package/dist/create/server-source/crates/feltdb/src/phase5_scenarios.rs +0 -323
  133. package/dist/create/server-source/crates/feltdb/src/phase6_adversarial_scenarios.rs +0 -531
  134. package/dist/create/server-source/crates/feltdb/src/phase6_convergence_validator.rs +0 -406
  135. package/dist/create/server-source/crates/feltdb/src/phase6_persistence.rs +0 -436
  136. package/dist/create/server-source/crates/feltdb/src/phase_1c_real_tcp.rs +0 -387
  137. package/dist/create/server-source/crates/feltdb/src/phase_2a_failures.rs +0 -378
  138. package/dist/create/server-source/crates/feltdb/src/phase_2b_network.rs +0 -359
  139. package/dist/create/server-source/crates/feltdb/src/phase_2c_cascading.rs +0 -372
  140. package/dist/create/server-source/crates/feltdb/src/phase_3_durability.rs +0 -404
  141. package/dist/create/server-source/crates/feltdb/src/phase_4_baseline.rs +0 -376
  142. package/dist/create/server-source/crates/feltdb/src/phase_5_soak.rs +0 -461
  143. package/dist/create/server-source/crates/feltdb/src/policy_evaluation.rs +0 -2125
  144. package/dist/create/server-source/crates/feltdb/src/production_api.rs +0 -418
  145. package/dist/create/server-source/crates/feltdb/src/provenance.rs +0 -207
  146. package/dist/create/server-source/crates/feltdb/src/query_execution_diagnostics.rs +0 -126
  147. package/dist/create/server-source/crates/feltdb/src/query_performance.rs +0 -218
  148. package/dist/create/server-source/crates/feltdb/src/read_basis.rs +0 -257
  149. package/dist/create/server-source/crates/feltdb/src/references.rs +0 -404
  150. package/dist/create/server-source/crates/feltdb/src/replay_fuzzing.rs +0 -401
  151. package/dist/create/server-source/crates/feltdb/src/replica_acknowledgements.rs +0 -501
  152. package/dist/create/server-source/crates/feltdb/src/replica_membership.rs +0 -680
  153. package/dist/create/server-source/crates/feltdb/src/replication_manager.rs +0 -163
  154. package/dist/create/server-source/crates/feltdb/src/replication_protocol.rs +0 -133
  155. package/dist/create/server-source/crates/feltdb/src/routing.rs +0 -409
  156. package/dist/create/server-source/crates/feltdb/src/semantic_decision.rs +0 -2029
  157. package/dist/create/server-source/crates/feltdb/src/semantic_query.rs +0 -599
  158. package/dist/create/server-source/crates/feltdb/src/sharding.rs +0 -549
  159. package/dist/create/server-source/crates/feltdb/src/state_conflict_contract.rs +0 -516
  160. package/dist/create/server-source/crates/feltdb/src/state_contract.rs +0 -6003
  161. package/dist/create/server-source/crates/feltdb/src/state_diff_contract.rs +0 -222
  162. package/dist/create/server-source/crates/feltdb/src/state_facade.rs +0 -320
  163. package/dist/create/server-source/crates/feltdb/src/state_hash.rs +0 -291
  164. package/dist/create/server-source/crates/feltdb/src/state_model.rs +0 -4158
  165. package/dist/create/server-source/crates/feltdb/src/state_transition_store.rs +0 -379
  166. package/dist/create/server-source/crates/feltdb/src/state_trigger.rs +0 -673
  167. package/dist/create/server-source/crates/feltdb/src/storage.rs +0 -273
  168. package/dist/create/server-source/crates/feltdb/src/submission.rs +0 -471
  169. package/dist/create/server-source/crates/feltdb/src/sync.rs +0 -495
  170. package/dist/create/server-source/crates/feltdb/src/sync_contract.rs +0 -822
  171. package/dist/create/server-source/crates/feltdb/src/tcp_transport.rs +0 -433
  172. package/dist/create/server-source/crates/feltdb/src/transaction_api.rs +0 -462
  173. package/dist/create/server-source/crates/feltdb/src/transaction_invariants.rs +0 -965
  174. package/dist/create/server-source/crates/feltdb/src/transaction_preconditions.rs +0 -1120
  175. package/dist/create/server-source/crates/feltdb/src/transactions.rs +0 -643
  176. package/dist/create/server-source/crates/feltdb/src/trigger.rs +0 -390
  177. package/dist/create/server-source/crates/feltdb/src/trigger_contract.rs +0 -749
  178. package/dist/create/server-source/crates/feltdb/src/versioned_object_substrate.rs +0 -708
  179. package/dist/create/server-source/crates/feltdb/src/worker_mesh.rs +0 -1269
  180. package/dist/create/server-source/crates/feltdb/src/workflow.rs +0 -713
  181. package/dist/create/server-source/crates/feltdb/src/workflow_acceptance_tests.rs +0 -510
  182. package/dist/create/server-source/crates/feltdb/src/workflow_integration.rs +0 -354
  183. package/dist/create/server-source/crates/feltdb/src/workflow_runtime.rs +0 -594
  184. package/dist/create/server-source/crates/feltdb/src/workload.rs +0 -4222
  185. package/dist/create/server-source/crates/feltdb/src/workload_diagnostics.rs +0 -456
  186. package/dist/create/server-source/crates/feltdb/tests/agent_state_native.rs +0 -1188
  187. package/dist/create/server-source/crates/feltdb/tests/agent_transition_contract_conformance.rs +0 -2884
  188. package/dist/create/server-source/crates/feltdb/tests/authority_process.rs +0 -297
  189. package/dist/create/server-source/crates/feltdb/tests/authority_transaction_identity.rs +0 -534
  190. package/dist/create/server-source/crates/feltdb/tests/bounded_read_contract.rs +0 -132
  191. package/dist/create/server-source/crates/feltdb/tests/branching_evidence.rs +0 -299
  192. package/dist/create/server-source/crates/feltdb/tests/common/mod.rs +0 -96
  193. package/dist/create/server-source/crates/feltdb/tests/compaction_stall_contract.rs +0 -272
  194. package/dist/create/server-source/crates/feltdb/tests/crash_durability_contract.rs +0 -467
  195. package/dist/create/server-source/crates/feltdb/tests/current_revision_authority_evidence.rs +0 -309
  196. package/dist/create/server-source/crates/feltdb/tests/durable_backup_contract.rs +0 -445
  197. package/dist/create/server-source/crates/feltdb/tests/durable_corruption_contract.rs +0 -518
  198. package/dist/create/server-source/crates/feltdb/tests/durable_format_compatibility.rs +0 -392
  199. package/dist/create/server-source/crates/feltdb/tests/feltdb_state_boundary_tests.rs +0 -850
  200. package/dist/create/server-source/crates/feltdb/tests/fixtures/agent_transition_corpus.json +0 -23340
  201. package/dist/create/server-source/crates/feltdb/tests/fixtures/state_conflict_contract_corpus.json +0 -1916
  202. package/dist/create/server-source/crates/feltdb/tests/fixtures/state_diff_contract_corpus.json +0 -1878
  203. package/dist/create/server-source/crates/feltdb/tests/fixtures/trigger_contract_corpus.json +0 -1862
  204. package/dist/create/server-source/crates/feltdb/tests/generalized_version_control_evidence.rs +0 -123
  205. package/dist/create/server-source/crates/feltdb/tests/generalized_version_control_substrate_completion.rs +0 -299
  206. package/dist/create/server-source/crates/feltdb/tests/local_process_ownership.rs +0 -69
  207. package/dist/create/server-source/crates/feltdb/tests/managed_incident_regression.rs +0 -191
  208. package/dist/create/server-source/crates/feltdb/tests/operational_health_contract.rs +0 -278
  209. package/dist/create/server-source/crates/feltdb/tests/pr34_query_collection.rs +0 -234
  210. package/dist/create/server-source/crates/feltdb/tests/pr35_equality_index.rs +0 -947
  211. package/dist/create/server-source/crates/feltdb/tests/pr7_self_authorization_proof.rs +0 -404
  212. package/dist/create/server-source/crates/feltdb/tests/pr8_vocabulary_assessment.rs +0 -916
  213. package/dist/create/server-source/crates/feltdb/tests/pr9_phase2_boundary_tests.rs +0 -1045
  214. package/dist/create/server-source/crates/feltdb/tests/pr9_phase3a_path_a_tests.rs +0 -347
  215. package/dist/create/server-source/crates/feltdb/tests/pr9_phase3c_authorized_mutations.rs +0 -361
  216. package/dist/create/server-source/crates/feltdb/tests/pr9_phase3c_role_based_authorization.rs +0 -330
  217. package/dist/create/server-source/crates/feltdb/tests/pr9_phase3c_simple_auth_delete.rs +0 -93
  218. package/dist/create/server-source/crates/feltdb/tests/pr9_phase3c_team_delete_role_authorization.rs +0 -622
  219. package/dist/create/server-source/crates/feltdb/tests/pr9_teams_role_based_access.rs +0 -517
  220. package/dist/create/server-source/crates/feltdb/tests/production_certification.rs +0 -1153
  221. package/dist/create/server-source/crates/feltdb/tests/production_contract.rs +0 -771
  222. package/dist/create/server-source/crates/feltdb/tests/production_readiness_contract.rs +0 -1383
  223. package/dist/create/server-source/crates/feltdb/tests/reconciliation_application.rs +0 -868
  224. package/dist/create/server-source/crates/feltdb/tests/reconciliation_wire_format_evidence.rs +0 -221
  225. package/dist/create/server-source/crates/feltdb/tests/replicated_history_contract.rs +0 -417
  226. package/dist/create/server-source/crates/feltdb/tests/resource_scoped_revisions.rs +0 -338
  227. package/dist/create/server-source/crates/feltdb/tests/revision_identity_contract.rs +0 -1039
  228. package/dist/create/server-source/crates/feltdb/tests/revision_model_decision.rs +0 -739
  229. package/dist/create/server-source/crates/feltdb/tests/revision_retention_boundary_evidence.rs +0 -452
  230. package/dist/create/server-source/crates/feltdb/tests/saas_authorization_integration.rs +0 -81
  231. package/dist/create/server-source/crates/feltdb/tests/saas_invitation_lifecycle.rs +0 -438
  232. package/dist/create/server-source/crates/feltdb/tests/semantic_decision.rs +0 -1165
  233. package/dist/create/server-source/crates/feltdb/tests/semantic_query.rs +0 -767
  234. package/dist/create/server-source/crates/feltdb/tests/semantic_query_contract.rs +0 -363
  235. package/dist/create/server-source/crates/feltdb/tests/state_conflict_contract_conformance.rs +0 -1799
  236. package/dist/create/server-source/crates/feltdb/tests/state_diff_contract_conformance.rs +0 -1316
  237. package/dist/create/server-source/crates/feltdb/tests/state_model_integration.rs +0 -358
  238. package/dist/create/server-source/crates/feltdb/tests/state_persistence_integration.rs +0 -365
  239. package/dist/create/server-source/crates/feltdb/tests/state_ref_authority.rs +0 -918
  240. package/dist/create/server-source/crates/feltdb/tests/state_store_boundary_evidence.rs +0 -299
  241. package/dist/create/server-source/crates/feltdb/tests/storage_independent_versioned_object_substrate.rs +0 -328
  242. package/dist/create/server-source/crates/feltdb/tests/sync_divergence_evidence.rs +0 -255
  243. package/dist/create/server-source/crates/feltdb/tests/three_way_input_boundary_evidence.rs +0 -249
  244. package/dist/create/server-source/crates/feltdb/tests/trigger_contract_conformance.rs +0 -994
  245. package/dist/create/server-source/crates/feltdb/tests/workload_envelope_contract.rs +0 -484
  246. package/dist/studio-app/assets/feltdb_wasm_bg-TseJn2Ax.wasm +0 -0
  247. package/dist/studio-app/assets/index-1wd0uKua.js +0 -30
@@ -1,4222 +0,0 @@
1
- //! Durable workload authority. Workers execute; this state machine decides.
2
- use crate::{
3
- authorization::{SignedGrant, Subject},
4
- AtomicMutation, FeltDb, FlowError, RecordPrecondition, StoredRow,
5
- };
6
- use rand::{distributions::Alphanumeric, Rng};
7
- use serde::{Deserialize, Serialize};
8
- use serde_json::Value;
9
- use sha2::{Digest, Sha256};
10
- use std::collections::{BTreeMap, BTreeSet};
11
- use std::path::Path;
12
- pub const WORKLOAD_CONTRACT_VERSION: u32 = 1;
13
- pub const DURABLE_WORK_COLLECTION: &str = "system_durable_work";
14
- /// Workloads admitted at this contract version or later have their `timeouts`
15
- /// enforced. Earlier records were admitted when timeouts were stored and
16
- /// ignored; they keep that behaviour, so an upgrade never times out work whose
17
- /// requester was told nothing would.
18
- pub const DEADLINE_CONTRACT_VERSION: u32 = 2;
19
-
20
- #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
21
- #[serde(rename_all = "SCREAMING_SNAKE_CASE")]
22
- pub enum WorkloadState {
23
- Created,
24
- Ready,
25
- Claimed,
26
- Running,
27
- Checkpointed,
28
- Recovering,
29
- CancellationRequested,
30
- Succeeded,
31
- Failed,
32
- Cancelled,
33
- TimedOut,
34
- DeadLetter,
35
- }
36
- #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
37
- #[serde(rename_all = "snake_case")]
38
- pub enum BackoffPolicy {
39
- Fixed,
40
- Exponential,
41
- }
42
- #[derive(Debug, Clone, Serialize, Deserialize)]
43
- pub struct WorkloadRetryPolicy {
44
- pub max_attempts: u32,
45
- pub initial_delay_ms: u64,
46
- pub max_delay_ms: u64,
47
- pub backoff: BackoffPolicy,
48
- #[serde(default)]
49
- pub retry_on: BTreeSet<String>,
50
- }
51
- impl Default for WorkloadRetryPolicy {
52
- fn default() -> Self {
53
- Self {
54
- max_attempts: 3,
55
- initial_delay_ms: 1000,
56
- max_delay_ms: 300_000,
57
- backoff: BackoffPolicy::Exponential,
58
- retry_on: [
59
- "timeout".into(),
60
- "worker_failure".into(),
61
- "transient_connection".into(),
62
- ]
63
- .into(),
64
- }
65
- }
66
- }
67
- #[derive(Debug, Clone, Serialize, Deserialize, Default)]
68
- pub struct WorkloadTimeouts {
69
- pub queue_ms: Option<u64>,
70
- pub claim_ms: Option<u64>,
71
- pub execution_ms: Option<u64>,
72
- pub heartbeat_ms: Option<u64>,
73
- pub overall_ms: Option<u64>,
74
- }
75
- #[derive(Debug, Clone, Serialize, Deserialize)]
76
- pub struct WorkloadDependency {
77
- pub workload_id: String,
78
- #[serde(default = "succeeded_state")]
79
- pub required_state: WorkloadState,
80
- }
81
- fn succeeded_state() -> WorkloadState {
82
- WorkloadState::Succeeded
83
- }
84
- #[derive(Debug, Clone, Serialize, Deserialize)]
85
- pub struct WorkloadClaim {
86
- pub claim_id: String,
87
- pub workload_id: String,
88
- pub worker_id: String,
89
- pub lease_id: String,
90
- pub fencing_token: u64,
91
- pub capability_snapshot: BTreeSet<String>,
92
- pub claimed_at: i64,
93
- pub expires_at: i64,
94
- pub last_heartbeat: i64,
95
- }
96
- #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
97
- #[serde(rename_all = "SCREAMING_SNAKE_CASE")]
98
- pub enum AttemptStatus {
99
- Claimed,
100
- Running,
101
- Checkpointed,
102
- Succeeded,
103
- Failed,
104
- TimedOut,
105
- Cancelled,
106
- WorkerLost,
107
- }
108
- #[derive(Debug, Clone, Serialize, Deserialize)]
109
- pub struct WorkloadAttempt {
110
- pub attempt_id: String,
111
- pub workload_id: String,
112
- pub worker_id: String,
113
- pub claim_id: String,
114
- pub fencing_token: u64,
115
- pub started_at: Option<i64>,
116
- pub ended_at: Option<i64>,
117
- pub input_hash: String,
118
- pub result_hash: Option<String>,
119
- pub error: Option<String>,
120
- #[serde(default)]
121
- pub resource_usage: BTreeMap<String, u64>,
122
- pub status: AttemptStatus,
123
- }
124
- #[derive(Debug, Clone, Serialize, Deserialize)]
125
- pub struct WorkloadCheckpoint {
126
- pub checkpoint_id: String,
127
- pub workload_id: String,
128
- pub attempt_id: String,
129
- pub sequence: u64,
130
- pub state: Value,
131
- pub state_hash: String,
132
- pub created_at: i64,
133
- pub produced_by: String,
134
- }
135
- #[derive(Debug, Clone, Serialize, Deserialize)]
136
- pub struct WorkloadResult {
137
- pub result_id: String,
138
- pub workload_id: String,
139
- pub attempt_id: String,
140
- pub status: String,
141
- pub output: Value,
142
- pub output_hash: String,
143
- #[serde(default)]
144
- pub artifacts: Vec<String>,
145
- #[serde(default)]
146
- pub provenance: BTreeMap<String, String>,
147
- pub produced_by: String,
148
- pub created_at: i64,
149
- pub schema_version: u64,
150
- }
151
- #[derive(Debug, Clone, Serialize, Deserialize)]
152
- pub struct WorkloadExecutionContext {
153
- pub workload_id: String,
154
- pub attempt_id: String,
155
- pub worker_id: String,
156
- pub capabilities: BTreeSet<String>,
157
- pub connection_scopes: BTreeSet<String>,
158
- pub secret_references: Vec<String>,
159
- pub expires_at: i64,
160
- pub fencing_token: u64,
161
- /// The work's input, so the claiming worker can execute it.
162
- #[serde(default)]
163
- pub input: Value,
164
- /// Every committed checkpoint, from this and earlier attempts, in order. A
165
- /// resumed attempt reads these to learn how far lost attempts got.
166
- #[serde(default)]
167
- pub checkpoints: Vec<WorkloadCheckpoint>,
168
- }
169
- #[derive(Debug, Clone, Serialize, Deserialize)]
170
- pub struct WorkloadEvent {
171
- pub sequence: u64,
172
- pub event_type: String,
173
- pub from: Option<WorkloadState>,
174
- pub to: WorkloadState,
175
- pub actor: String,
176
- pub at: i64,
177
- #[serde(default)]
178
- pub evidence: BTreeMap<String, String>,
179
- }
180
- #[derive(Debug, Clone, Serialize, Deserialize)]
181
- pub struct Workload {
182
- pub contract_version: u32,
183
- pub workload_id: String,
184
- pub tenant_id: String,
185
- pub application_id: String,
186
- pub environment: String,
187
- pub definition_id: String,
188
- pub definition_revision: String,
189
- pub trigger: String,
190
- pub input: Value,
191
- pub input_hash: String,
192
- pub requested_by: Subject,
193
- #[serde(default)]
194
- pub session_id: Option<String>,
195
- pub authorization_snapshot: SignedGrant,
196
- pub capability_snapshot: BTreeSet<String>,
197
- pub connection_scopes: BTreeSet<String>,
198
- pub priority: i32,
199
- pub idempotency_key: String,
200
- pub created_at: i64,
201
- pub updated_at: i64,
202
- pub state: WorkloadState,
203
- pub execution_history: Vec<WorkloadEvent>,
204
- pub attempts: Vec<WorkloadAttempt>,
205
- pub checkpoints: Vec<WorkloadCheckpoint>,
206
- pub result: Option<WorkloadResult>,
207
- pub retry_policy: WorkloadRetryPolicy,
208
- pub timeouts: WorkloadTimeouts,
209
- pub attempt_count: u32,
210
- pub retry_count: u32,
211
- /// The instant, in unix seconds, before which this workload must not be
212
- /// claimed. `None` means "eligible as soon as it is `Ready`".
213
- ///
214
- /// This is the single temporal eligibility concept in the workload model.
215
- /// It is what retry backoff sets after a failure, what a timeout sets before
216
- /// re-arming, and what [`CreateWorkload::not_before`] sets at creation, so a
217
- /// workload that is not yet executable is always not-yet-executable for the
218
- /// same reason and is refused by the same check in [`WorkloadStore::claim`].
219
- pub next_attempt_at: Option<i64>,
220
- pub last_failure: Option<String>,
221
- pub current_claim: Option<WorkloadClaim>,
222
- pub fencing_token: u64,
223
- #[serde(default)]
224
- pub dependencies: Vec<WorkloadDependency>,
225
- }
226
-
227
- impl Workload {
228
- /// Has this workload's eligibility instant arrived?
229
- ///
230
- /// Eligibility is only half of claimability — the state must also be
231
- /// `Ready`, and [`WorkloadStore::claim`] checks both. This answers the
232
- /// temporal half alone so a scheduler can skip a workload that is merely
233
- /// early without attempting a claim it knows will be refused.
234
- pub fn is_eligible_at(&self, now: i64) -> bool {
235
- self.next_attempt_at.is_none_or(|instant| instant <= now)
236
- }
237
- }
238
-
239
- #[derive(Debug, Clone, Serialize, Deserialize)]
240
- pub struct CreateWorkload {
241
- pub tenant_id: String,
242
- pub application_id: String,
243
- pub environment: String,
244
- pub definition_id: String,
245
- pub definition_revision: String,
246
- pub trigger: String,
247
- pub input: Value,
248
- pub requested_by: Subject,
249
- #[serde(default)]
250
- pub session_id: Option<String>,
251
- pub authorization_snapshot: SignedGrant,
252
- pub capability_snapshot: BTreeSet<String>,
253
- #[serde(default)]
254
- pub connection_scopes: BTreeSet<String>,
255
- #[serde(default)]
256
- pub priority: i32,
257
- pub idempotency_key: String,
258
- #[serde(default)]
259
- pub retry_policy: WorkloadRetryPolicy,
260
- #[serde(default)]
261
- pub timeouts: WorkloadTimeouts,
262
- #[serde(default)]
263
- pub dependencies: Vec<WorkloadDependency>,
264
- /// The instant, in unix seconds, before which this workload must not be
265
- /// claimed. Omitted or `None` means it is claimable as soon as it is
266
- /// `Ready`, which is what every workload did before this field existed.
267
- ///
268
- /// A workload created with `not_before` is durable immediately and
269
- /// executable later: it reaches `Ready`, survives restart, and is refused to
270
- /// every worker until the instant arrives. It is *not* a schedule — there is
271
- /// no recurrence, no expression, and no component that re-arms it. Creating
272
- /// the next occurrence is the caller's business.
273
- ///
274
- /// A `not_before` already in the past is not an error. It simply describes a
275
- /// workload that is eligible now, which is the same thing as omitting it.
276
- #[serde(default)]
277
- pub not_before: Option<i64>,
278
- pub created_at: i64,
279
- }
280
- #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
281
- pub struct WorkloadError {
282
- pub code: String,
283
- pub message: String,
284
- /// Structured detail for the caller, e.g. which request fields conflict.
285
- #[serde(default, skip_serializing_if = "Option::is_none")]
286
- pub details: Option<Value>,
287
- }
288
- impl WorkloadError {
289
- fn new(code: &str, message: impl Into<String>) -> Self {
290
- Self {
291
- code: code.into(),
292
- message: message.into(),
293
- details: None,
294
- }
295
- }
296
- }
297
- impl std::fmt::Display for WorkloadError {
298
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
299
- write!(f, "{}: {}", self.code, self.message)
300
- }
301
- }
302
- impl std::error::Error for WorkloadError {}
303
-
304
- #[derive(Clone)]
305
- pub struct WorkloadStore {
306
- db: FeltDb,
307
- pub workloads: BTreeMap<String, Workload>,
308
- idempotency: BTreeMap<String, String>,
309
- result_ids: BTreeMap<String, String>,
310
- /// Rebuildable cache of FeltDB-owned document versions. This is never
311
- /// serialized and never advances independently of a canonical commit.
312
- document_versions: BTreeMap<String, u64>,
313
- /// The store revision this projection last reconciled with. A store that
314
- /// has moved past it (a commit by another writer, or a replicated
315
- /// operation) is re-read before the projection answers.
316
- synced_revision: u64,
317
- /// Always false outside tests, which set it to act on the projection
318
- /// without re-reading the store: that reproduces another writer
319
- /// committing between a refresh and a commit — the window only the storage
320
- /// guard closes.
321
- refresh_disabled: bool,
322
- }
323
-
324
- #[derive(Serialize, Deserialize)]
325
- struct LegacyWorkloadStore {
326
- #[serde(default)]
327
- workloads: BTreeMap<String, Workload>,
328
- #[serde(default)]
329
- idempotency: BTreeMap<String, String>,
330
- #[serde(default)]
331
- result_ids: BTreeMap<String, String>,
332
- #[serde(default)]
333
- event_sequence: u64,
334
- }
335
-
336
- impl WorkloadStore {
337
- pub fn load(path: impl AsRef<Path>) -> Result<Self, WorkloadError> {
338
- Self::from_db(FeltDb::open(path).map_err(io)?)
339
- }
340
-
341
- pub fn from_db(db: FeltDb) -> Result<Self, WorkloadError> {
342
- // Read before listing: a commit that lands in between is re-read by
343
- // the next refresh rather than missed.
344
- let synced_revision = db.current_revision().map_err(io)?;
345
- let mut workloads = BTreeMap::new();
346
- let mut document_versions = BTreeMap::new();
347
- for row in db.list_collection(DURABLE_WORK_COLLECTION).map_err(io)? {
348
- let version = row
349
- .value
350
- .get("__version")
351
- .and_then(Value::as_u64)
352
- // Canonical FeltDB CAS defines a pre-versioned document as
353
- // version 1. The next guarded mutation writes version 2, which
354
- // upgrades rows created before Work adopted RecordPrecondition.
355
- .unwrap_or(1);
356
- let workload: Workload = serde_json::from_value(row.value).map_err(io)?;
357
- document_versions.insert(workload.workload_id.clone(), version);
358
- workloads.insert(workload.workload_id.clone(), workload);
359
- }
360
- let idempotency = idempotency_index(&workloads, idempotency_scope);
361
- let mut result_ids = BTreeMap::new();
362
- for workload in workloads.values() {
363
- if let Some(result) = &workload.result {
364
- result_ids.insert(result.result_id.clone(), workload.workload_id.clone());
365
- }
366
- }
367
- Ok(Self {
368
- db,
369
- workloads,
370
- idempotency,
371
- result_ids,
372
- document_versions,
373
- synced_revision,
374
- refresh_disabled: false,
375
- })
376
- }
377
-
378
- /// Reconcile the projection with the committed store.
379
- ///
380
- /// The store, not this projection, is authoritative. When its revision has
381
- /// moved since the last reconciliation, every workload row whose document
382
- /// version differs from the projection's is re-read; rows are borrowed and
383
- /// only changed ones are cloned. Commits by another writer of the store and
384
- /// operations replicated from a peer both become visible here, without a
385
- /// restart.
386
- pub fn refresh(&mut self) -> Result<(), WorkloadError> {
387
- if self.refresh_disabled {
388
- return Ok(());
389
- }
390
- let revision = self.db.current_revision().map_err(io)?;
391
- if revision == self.synced_revision {
392
- return Ok(());
393
- }
394
- let known = &self.document_versions;
395
- let changed = self
396
- .db
397
- .query_collection(DURABLE_WORK_COLLECTION, None, |row| {
398
- let version = row
399
- .value
400
- .get("__version")
401
- .and_then(Value::as_u64)
402
- .unwrap_or(1);
403
- let id = row.key.rsplit('/').next().unwrap_or_default();
404
- known.get(id) != Some(&version)
405
- })
406
- .map_err(io)?;
407
- for row in changed {
408
- self.adopt(row)?;
409
- }
410
- self.synced_revision = revision;
411
- Ok(())
412
- }
413
-
414
- /// Re-read one workload from the store when its committed version differs
415
- /// from the projection's: a keyed read, not a scan. An id the projection
416
- /// has never seen (admitted by another writer) falls back to [`Self::refresh`].
417
- fn refresh_id(&mut self, id: &str) -> Result<(), WorkloadError> {
418
- match self.workloads.get(id).map(work_key) {
419
- Some(key) => self.refresh_key(&key).map(|_| ()),
420
- None => self.refresh(),
421
- }
422
- }
423
-
424
- fn refresh_key(&mut self, key: &str) -> Result<Option<Workload>, WorkloadError> {
425
- if self.refresh_disabled {
426
- let id = key.rsplit('/').next().unwrap_or_default();
427
- return Ok(self.workloads.get(id).cloned());
428
- }
429
- let Some(row) = self.db.live_row(DURABLE_WORK_COLLECTION, key) else {
430
- return Ok(None);
431
- };
432
- let version = row
433
- .value
434
- .get("__version")
435
- .and_then(Value::as_u64)
436
- .unwrap_or(1);
437
- let id = key.rsplit('/').next().unwrap_or_default();
438
- if self.document_versions.get(id) == Some(&version) {
439
- return Ok(self.workloads.get(id).cloned());
440
- }
441
- self.adopt(row).map(Some)
442
- }
443
-
444
- /// Install one committed row in the projection and its derived indexes.
445
- fn adopt(&mut self, row: StoredRow) -> Result<Workload, WorkloadError> {
446
- let version = row
447
- .value
448
- .get("__version")
449
- .and_then(Value::as_u64)
450
- .unwrap_or(1);
451
- let workload: Workload = serde_json::from_value(row.value).map_err(io)?;
452
- if let Some(result) = &workload.result {
453
- self.result_ids
454
- .insert(result.result_id.clone(), workload.workload_id.clone());
455
- }
456
- self.idempotency.insert(
457
- idempotency_scope(
458
- &workload.tenant_id,
459
- &workload.application_id,
460
- &workload.requested_by,
461
- &workload.idempotency_key,
462
- ),
463
- workload.workload_id.clone(),
464
- );
465
- self.document_versions
466
- .insert(workload.workload_id.clone(), version);
467
- self.workloads
468
- .insert(workload.workload_id.clone(), workload.clone());
469
- Ok(workload)
470
- }
471
-
472
- /// A workload as committed in the store (see [`Self::refresh`]).
473
- pub fn fresh(&mut self, id: &str) -> Result<&Workload, WorkloadError> {
474
- self.refresh_id(id)?;
475
- self.get(id)
476
- }
477
-
478
- /// An application's workloads as committed in the store.
479
- pub fn fresh_list(
480
- &mut self,
481
- tenant: &str,
482
- application: &str,
483
- ) -> Result<Vec<Workload>, WorkloadError> {
484
- self.refresh()?;
485
- Ok(self.list(tenant, application))
486
- }
487
-
488
- pub fn migrate_legacy_file(db: FeltDb, path: &Path) -> Result<Self, WorkloadError> {
489
- let bytes = std::fs::read(path).map_err(io)?;
490
- let legacy: LegacyWorkloadStore = serde_json::from_slice(&bytes).map_err(io)?;
491
- let mut store = Self::from_db(db)?;
492
- for (id, workload) in legacy.workloads {
493
- if id != workload.workload_id {
494
- return Err(WorkloadError::new(
495
- "WORKLOAD_MIGRATION_CONFLICT",
496
- format!(
497
- "legacy key {id} does not match workload identity {}",
498
- workload.workload_id
499
- ),
500
- ));
501
- }
502
- if let Some(existing) = store.workloads.get(&id) {
503
- if serde_json::to_value(existing).map_err(io)?
504
- != serde_json::to_value(&workload).map_err(io)?
505
- {
506
- return Err(WorkloadError::new(
507
- "WORKLOAD_MIGRATION_CONFLICT",
508
- format!("canonical workload {id} differs from legacy source"),
509
- ));
510
- }
511
- continue;
512
- }
513
- store.workloads.insert(id.clone(), workload);
514
- store.persist_workload(&id, "workload_migration")?;
515
- }
516
- // The maps are verification evidence, not migrated authorities.
517
- let rebuilt = Self::from_db(store.db.clone())?;
518
- if legacy.idempotency != idempotency_index(&rebuilt.workloads, legacy_idempotency_scope)
519
- || legacy.result_ids != rebuilt.result_ids
520
- || legacy.event_sequence != derived_event_sequence(&rebuilt.workloads)
521
- {
522
- return Err(WorkloadError::new(
523
- "WORKLOAD_MIGRATION_CONFLICT",
524
- "legacy derived indexes do not match canonical work aggregates",
525
- ));
526
- }
527
- Ok(rebuilt)
528
- }
529
-
530
- fn persist_workload(&mut self, id: &str, action: &str) -> Result<(), WorkloadError> {
531
- let workload = self
532
- .workloads
533
- .get(id)
534
- .ok_or_else(|| WorkloadError::new("WORKLOAD_NOT_FOUND", "workload not found"))?;
535
- let key = work_key(workload);
536
- let expected_version = self.document_versions.get(id).copied();
537
- let transaction_id = runtime_id("work_tx");
538
- let aggregate_digest = hash(&serde_json::to_value(workload).map_err(io)?);
539
- let commit = self
540
- .db
541
- .apply_atomic_transaction_guarded(
542
- &transaction_id,
543
- None,
544
- None,
545
- &[],
546
- &[RecordPrecondition {
547
- capability: DURABLE_WORK_COLLECTION.into(),
548
- key: key.clone(),
549
- require_absent: expected_version.is_none(),
550
- expected_version,
551
- expected_epoch: None,
552
- expected_lease_id: None,
553
- }],
554
- &[AtomicMutation {
555
- capability: DURABLE_WORK_COLLECTION.into(),
556
- key,
557
- value: Some(serde_json::to_value(workload).map_err(io)?),
558
- }],
559
- Some(serde_json::json!({
560
- "kind": action,
561
- "subject": workload.requested_by.id(),
562
- "principal_id": workload.requested_by.id(),
563
- "agent_id": (workload.requested_by.kind() == "agent").then(|| workload.requested_by.id()),
564
- "session_id": workload.session_id,
565
- "delegator_id": workload.authorization_snapshot.grant.issuer.id(),
566
- "authorization_decision_id": workload.authorization_snapshot.grant.id,
567
- "tenant": workload.tenant_id,
568
- "application": workload.application_id,
569
- "work_id": workload.workload_id,
570
- "operation_id": transaction_id,
571
- "attempt_id": workload.attempts.last().map(|attempt| attempt.attempt_id.as_str()),
572
- "worker_id": workload.current_claim.as_ref().map(|claim| claim.worker_id.as_str()),
573
- "result_id": workload.result.as_ref().map(|result| result.result_id.as_str()),
574
- "source_kind": (action == "workload_migration").then_some("WorkloadStore"),
575
- "source_digest": aggregate_digest,
576
- "migration_outcome": (action == "workload_migration").then_some("imported"),
577
- })),
578
- )
579
- .map_err(workload_storage_error)?;
580
- let version = commit
581
- .rows
582
- .first()
583
- .and_then(|row| row.value.get("__version"))
584
- .and_then(Value::as_u64)
585
- .ok_or_else(|| io("canonical work commit has no document version"))?;
586
- self.document_versions.insert(id.into(), version);
587
- Ok(())
588
- }
589
- pub fn create(&mut self, input: CreateWorkload) -> Result<Workload, WorkloadError> {
590
- self.admit(input).map(|(workload, _)| workload)
591
- }
592
-
593
- /// Create a workload, or return the one this requester already created
594
- /// under the same key.
595
- ///
596
- /// The key is scoped to the tenant, the application **and the requester**,
597
- /// so one requester's key can never return, reveal or block another's
598
- /// workload. A repeated key is answered with the original workload only when
599
- /// the request is the same — definition, revision, environment, input and
600
- /// capability snapshot; otherwise it is refused with
601
- /// `IDEMPOTENCY_KEY_REUSED` and nothing is created. The requester's
602
- /// authorization snapshot is checked before any replay. The boolean is
603
- /// `true` when this call created the workload.
604
- pub fn admit(&mut self, input: CreateWorkload) -> Result<(Workload, bool), WorkloadError> {
605
- ensure_safe(&input.input)?;
606
- if input.idempotency_key.trim().is_empty() {
607
- return Err(WorkloadError::new(
608
- "WORKLOAD_INVALID",
609
- "idempotency key is required",
610
- ));
611
- }
612
- let t = &input.timeouts;
613
- if [
614
- t.queue_ms,
615
- t.claim_ms,
616
- t.execution_ms,
617
- t.heartbeat_ms,
618
- t.overall_ms,
619
- ]
620
- .contains(&Some(0))
621
- {
622
- return Err(WorkloadError::new(
623
- "WORKLOAD_INVALID",
624
- "a timeout must be at least 1 ms; omit it for no deadline",
625
- ));
626
- }
627
- if input.authorization_snapshot.grant.subject != input.requested_by
628
- || input.authorization_snapshot.grant.tenant_id != input.tenant_id
629
- || input.authorization_snapshot.grant.application_id != input.application_id
630
- {
631
- return Err(WorkloadError::new(
632
- "WORKLOAD_UNAUTHORIZED",
633
- "authorization snapshot does not bind the requester",
634
- ));
635
- }
636
- let idempotency = idempotency_scope(
637
- &input.tenant_id,
638
- &input.application_id,
639
- &input.requested_by,
640
- &input.idempotency_key,
641
- );
642
- // The scoped idempotency identity is the record identity. Competing
643
- // admissions therefore contend on one canonical require-absent CAS
644
- // even when their payloads differ; no separate durable uniqueness
645
- // index is needed.
646
- let id = format!("workload_{}", &hex(&Sha256::digest(&idempotency))[..24]);
647
- // Another writer, or a peer, may have admitted this identity: the
648
- // store is asked by key, not the projection.
649
- let committed_key = format!("{}/{}/{}", input.tenant_id, input.application_id, id);
650
- if let Some(existing) = self.refresh_key(&committed_key)? {
651
- return replay(existing, &input);
652
- }
653
- if let Some(existing_id) = self.idempotency.get(&idempotency) {
654
- let existing = self.workloads[existing_id].clone();
655
- return replay(existing, &input);
656
- }
657
- let request = input.clone();
658
- let input_hash = hash(&input.input);
659
- let mut workload = Workload {
660
- contract_version: DEADLINE_CONTRACT_VERSION,
661
- workload_id: id.clone(),
662
- tenant_id: input.tenant_id,
663
- application_id: input.application_id,
664
- environment: input.environment,
665
- definition_id: input.definition_id,
666
- definition_revision: input.definition_revision,
667
- trigger: input.trigger,
668
- input: input.input,
669
- input_hash,
670
- requested_by: input.requested_by,
671
- session_id: input.session_id,
672
- authorization_snapshot: input.authorization_snapshot,
673
- capability_snapshot: input.capability_snapshot,
674
- connection_scopes: input.connection_scopes,
675
- priority: input.priority,
676
- idempotency_key: input.idempotency_key,
677
- created_at: input.created_at,
678
- updated_at: input.created_at,
679
- state: WorkloadState::Created,
680
- execution_history: vec![],
681
- attempts: vec![],
682
- checkpoints: vec![],
683
- result: None,
684
- retry_policy: input.retry_policy,
685
- timeouts: input.timeouts,
686
- attempt_count: 0,
687
- retry_count: 0,
688
- // Creation-time eligibility and retry backoff are the same thing, so
689
- // they are the same field. A workload with `not_before` is created
690
- // exactly as any other and is simply not yet claimable.
691
- next_attempt_at: input.not_before,
692
- last_failure: None,
693
- current_claim: None,
694
- fencing_token: 0,
695
- dependencies: input.dependencies,
696
- };
697
- // The durable history records the eligibility the workload was created
698
- // with, so "why was this Ready and unclaimed for an hour" is answerable
699
- // from the workload itself rather than from the caller's intent.
700
- let evidence = match workload.next_attempt_at {
701
- Some(instant) => [("not_before".into(), instant.to_string())].into(),
702
- None => BTreeMap::new(),
703
- };
704
- self.transition(
705
- &mut workload,
706
- WorkloadState::Ready,
707
- "rust",
708
- input.created_at,
709
- evidence,
710
- )?;
711
- let key = work_key(&workload);
712
- self.idempotency.insert(idempotency.clone(), id.clone());
713
- self.workloads.insert(id.clone(), workload.clone());
714
- if let Err(error) = self.persist_workload(&id, "workload.created") {
715
- self.workloads.remove(&id);
716
- self.idempotency.remove(&idempotency);
717
- // Another writer of the same store (a second process, or a replica)
718
- // admitted this identity first. Converge on what it committed.
719
- if error.code == "WORKLOAD_CONFLICT" {
720
- if let Some(existing) = self.reload_committed(&key)? {
721
- self.idempotency.insert(idempotency, id);
722
- return replay(existing, &request);
723
- }
724
- }
725
- return Err(error);
726
- }
727
- Ok((workload, true))
728
- }
729
- pub fn get(&self, id: &str) -> Result<&Workload, WorkloadError> {
730
- self.workloads
731
- .get(id)
732
- .ok_or_else(|| WorkloadError::new("WORKLOAD_NOT_FOUND", "workload not found"))
733
- }
734
- pub fn list(&self, tenant: &str, application: &str) -> Vec<Workload> {
735
- let mut values = self
736
- .workloads
737
- .values()
738
- .filter(|v| v.tenant_id == tenant && v.application_id == application)
739
- .cloned()
740
- .collect::<Vec<_>>();
741
- values.sort_by_key(|v| (std::cmp::Reverse(v.priority), v.created_at));
742
- values
743
- }
744
- fn edit<R>(
745
- &mut self,
746
- id: &str,
747
- action: impl FnOnce(&mut Self, &mut Workload) -> Result<R, WorkloadError>,
748
- ) -> Result<R, WorkloadError> {
749
- // Act only on the committed version: a projection that missed another
750
- // writer's commit would otherwise lose its guarded commit at best.
751
- self.refresh_id(id)?;
752
- let workload = self
753
- .workloads
754
- .get(id)
755
- .cloned()
756
- .ok_or_else(|| WorkloadError::new("WORKLOAD_NOT_FOUND", "workload not found"))?;
757
- let mut edited = workload.clone();
758
- let result = action(self, &mut edited)?;
759
- self.workloads.insert(id.into(), edited);
760
- if let Err(error) = self.persist_workload(id, "workload.mutated") {
761
- self.workloads.insert(id.into(), workload.clone());
762
- // Another writer of the same store committed this workload first.
763
- // Nothing of this edit was committed; adopt what was, so this
764
- // projection does not keep acting on a version that no longer
765
- // exists (every later guarded commit from it would conflict).
766
- if error.code == "WORKLOAD_CONFLICT" {
767
- self.reload_committed(&work_key(&workload))?;
768
- }
769
- return Err(error);
770
- }
771
- Ok(result)
772
- }
773
- #[cfg(test)]
774
- pub(crate) fn db_for_tests(&self) -> FeltDb {
775
- self.db.clone()
776
- }
777
- /// See `refresh_disabled`.
778
- #[cfg(test)]
779
- pub(crate) fn without_refresh(mut self) -> Self {
780
- self.refresh_disabled = true;
781
- self
782
- }
783
- /// Replace this projection's copy of a workload with the committed row.
784
- fn reload_committed(&mut self, key: &str) -> Result<Option<Workload>, WorkloadError> {
785
- match self.db.live_row(DURABLE_WORK_COLLECTION, key) {
786
- Some(row) => self.adopt(row).map(Some),
787
- None => Ok(None),
788
- }
789
- }
790
- fn transition(
791
- &mut self,
792
- w: &mut Workload,
793
- to: WorkloadState,
794
- actor: &str,
795
- at: i64,
796
- evidence: BTreeMap<String, String>,
797
- ) -> Result<(), WorkloadError> {
798
- if !legal(w.state, to) {
799
- return Err(WorkloadError::new(
800
- "WORKLOAD_TRANSITION_DENIED",
801
- format!("{:?} cannot transition to {:?}", w.state, to),
802
- ));
803
- }
804
- let from = w.state;
805
- w.state = to;
806
- w.updated_at = at;
807
- let sequence = w
808
- .execution_history
809
- .last()
810
- .map_or(1, |event| event.sequence + 1);
811
- w.execution_history.push(WorkloadEvent {
812
- sequence,
813
- event_type: event_name(to).into(),
814
- from: Some(from),
815
- to,
816
- actor: actor.into(),
817
- at,
818
- evidence,
819
- });
820
- Ok(())
821
- }
822
- pub fn claim(
823
- &mut self,
824
- id: &str,
825
- worker: &str,
826
- capabilities: &BTreeSet<String>,
827
- now: i64,
828
- lease_ms: u64,
829
- ) -> Result<WorkloadClaim, WorkloadError> {
830
- self.enforce_deadline(id, now)?;
831
- // A Ready workload that has already used its budget (an operator
832
- // re-arm beyond it, or a record left by unbounded recovery before it
833
- // was bounded) is dead-lettered here rather than claimed again.
834
- let exhausted = self
835
- .get(id)
836
- .is_ok_and(|w| w.state == WorkloadState::Ready && w.attempt_count >= attempt_budget(w));
837
- if exhausted {
838
- self.edit(id, |store, w| {
839
- let evidence = [
840
- ("reason".into(), "attempt_budget_exhausted".into()),
841
- ("attempts_used".into(), w.attempt_count.to_string()),
842
- ("attempt_budget".into(), attempt_budget(w).to_string()),
843
- ]
844
- .into();
845
- store.transition(w, WorkloadState::DeadLetter, "rust.claim", now, evidence)
846
- })?;
847
- return Err(WorkloadError::new(
848
- "WORKLOAD_ATTEMPTS_EXHAUSTED",
849
- "workload has used its attempt budget",
850
- ));
851
- }
852
- self.edit(id, |store, w| {
853
- if w.state != WorkloadState::Ready {
854
- return Err(WorkloadError::new(
855
- "WORKLOAD_NOT_READY",
856
- "workload is not eligible",
857
- ));
858
- }
859
- // Separated from the state check because the two mean opposite
860
- // things to a worker: a workload in the wrong state may never become
861
- // claimable, while one that is merely early certainly will, at an
862
- // instant the error names. Collapsing them into one code left a
863
- // polling worker unable to tell "come back" from "give up".
864
- if !w.is_eligible_at(now) {
865
- return Err(WorkloadError::new(
866
- "WORKLOAD_NOT_YET_ELIGIBLE",
867
- format!(
868
- "workload is not claimable until {}",
869
- w.next_attempt_at.unwrap_or(now)
870
- ),
871
- ));
872
- }
873
- if !w.capability_snapshot.is_subset(capabilities) {
874
- return Err(WorkloadError::new(
875
- "WORKLOAD_CAPABILITY_DENIED",
876
- "worker lacks required capability snapshot",
877
- ));
878
- }
879
- w.fencing_token += 1;
880
- w.attempt_count += 1;
881
- let seed = format!("{}:{}:{}", w.workload_id, w.fencing_token, worker);
882
- let claim = WorkloadClaim {
883
- claim_id: format!("claim_{}", &hex(&Sha256::digest(seed.as_bytes()))[..20]),
884
- workload_id: w.workload_id.clone(),
885
- worker_id: worker.into(),
886
- lease_id: format!("lease_{}", w.fencing_token),
887
- fencing_token: w.fencing_token,
888
- capability_snapshot: w.capability_snapshot.clone(),
889
- claimed_at: now,
890
- expires_at: now + (lease_ms / 1000) as i64,
891
- last_heartbeat: now,
892
- };
893
- w.attempts.push(WorkloadAttempt {
894
- attempt_id: format!("attempt_{}_{}", w.workload_id, w.attempt_count),
895
- workload_id: w.workload_id.clone(),
896
- worker_id: worker.into(),
897
- claim_id: claim.claim_id.clone(),
898
- fencing_token: claim.fencing_token,
899
- started_at: None,
900
- ended_at: None,
901
- input_hash: w.input_hash.clone(),
902
- result_hash: None,
903
- error: None,
904
- resource_usage: BTreeMap::new(),
905
- status: AttemptStatus::Claimed,
906
- });
907
- w.current_claim = Some(claim.clone());
908
- store.transition(
909
- w,
910
- WorkloadState::Claimed,
911
- worker,
912
- now,
913
- [("fencing_token".into(), claim.fencing_token.to_string())].into(),
914
- )?;
915
- Ok(claim)
916
- })
917
- }
918
- fn fence(
919
- w: &Workload,
920
- claim: &str,
921
- worker: &str,
922
- token: u64,
923
- now: i64,
924
- ) -> Result<(), WorkloadError> {
925
- let current = w
926
- .current_claim
927
- .as_ref()
928
- .ok_or_else(|| WorkloadError::new("WORKLOAD_CLAIM_INVALID", "no active claim"))?;
929
- if current.claim_id != claim
930
- || current.worker_id != worker
931
- || current.fencing_token != token
932
- {
933
- return Err(WorkloadError::new(
934
- "WORKLOAD_STALE_FENCE",
935
- "claim or fencing token is stale",
936
- ));
937
- }
938
- if current.expires_at <= now {
939
- return Err(WorkloadError::new(
940
- "WORKLOAD_LEASE_EXPIRED",
941
- "claim lease expired",
942
- ));
943
- }
944
- Ok(())
945
- }
946
- pub fn start(
947
- &mut self,
948
- id: &str,
949
- claim: &str,
950
- worker: &str,
951
- token: u64,
952
- now: i64,
953
- ) -> Result<(), WorkloadError> {
954
- self.enforce_deadline(id, now)?;
955
- self.edit(id, |store, w| {
956
- Self::fence(w, claim, worker, token, now)?;
957
- let attempt = w.attempts.last_mut().unwrap();
958
- attempt.status = AttemptStatus::Running;
959
- attempt.started_at = Some(now);
960
- store.transition(w, WorkloadState::Running, worker, now, BTreeMap::new())
961
- })
962
- }
963
- pub fn heartbeat(
964
- &mut self,
965
- id: &str,
966
- claim: &str,
967
- worker: &str,
968
- token: u64,
969
- now: i64,
970
- extend_ms: u64,
971
- ) -> Result<WorkloadClaim, WorkloadError> {
972
- self.enforce_deadline(id, now)?;
973
- self.edit(id, |_, w| {
974
- Self::fence(w, claim, worker, token, now)?;
975
- let current = w.current_claim.as_mut().unwrap();
976
- current.last_heartbeat = now;
977
- current.expires_at = now + (extend_ms / 1000) as i64;
978
- Ok(current.clone())
979
- })
980
- }
981
- pub fn execution_context(
982
- &mut self,
983
- id: &str,
984
- claim: &str,
985
- worker: &str,
986
- token: u64,
987
- now: i64,
988
- secret_references: Vec<String>,
989
- ) -> Result<WorkloadExecutionContext, WorkloadError> {
990
- self.enforce_deadline(id, now)?;
991
- let workload = self.fresh(id)?;
992
- Self::fence(workload, claim, worker, token, now)?;
993
- if secret_references
994
- .iter()
995
- .any(|v| !v.starts_with("secret://"))
996
- {
997
- return Err(WorkloadError::new(
998
- "WORKLOAD_SECRET_REJECTED",
999
- "execution context accepts secret references only",
1000
- ));
1001
- }
1002
- let current = workload.current_claim.as_ref().unwrap();
1003
- Ok(WorkloadExecutionContext {
1004
- workload_id: id.into(),
1005
- attempt_id: workload.attempts.last().unwrap().attempt_id.clone(),
1006
- worker_id: worker.into(),
1007
- capabilities: current.capability_snapshot.clone(),
1008
- connection_scopes: workload.connection_scopes.clone(),
1009
- secret_references,
1010
- expires_at: current.expires_at,
1011
- fencing_token: token,
1012
- input: workload.input.clone(),
1013
- checkpoints: workload.checkpoints.clone(),
1014
- })
1015
- }
1016
- pub fn checkpoint(
1017
- &mut self,
1018
- id: &str,
1019
- claim: &str,
1020
- worker: &str,
1021
- token: u64,
1022
- state: Value,
1023
- now: i64,
1024
- ) -> Result<WorkloadCheckpoint, WorkloadError> {
1025
- ensure_safe(&state)?;
1026
- self.enforce_deadline(id, now)?;
1027
- self.edit(id, |store, w| {
1028
- Self::fence(w, claim, worker, token, now)?;
1029
- if !matches!(
1030
- w.state,
1031
- WorkloadState::Running | WorkloadState::Checkpointed
1032
- ) {
1033
- return Err(WorkloadError::new(
1034
- "WORKLOAD_TRANSITION_DENIED",
1035
- "checkpoint requires a running workload",
1036
- ));
1037
- }
1038
- let attempt = w.attempts.last_mut().unwrap();
1039
- attempt.status = AttemptStatus::Checkpointed;
1040
- let sequence = w.checkpoints.len() as u64 + 1;
1041
- let checkpoint = WorkloadCheckpoint {
1042
- checkpoint_id: format!("checkpoint_{}_{}", w.workload_id, sequence),
1043
- workload_id: w.workload_id.clone(),
1044
- attempt_id: attempt.attempt_id.clone(),
1045
- sequence,
1046
- state_hash: hash(&state),
1047
- state,
1048
- created_at: now,
1049
- produced_by: worker.into(),
1050
- };
1051
- w.checkpoints.push(checkpoint.clone());
1052
- if w.state == WorkloadState::Running {
1053
- store.transition(w, WorkloadState::Checkpointed, worker, now, BTreeMap::new())?;
1054
- }
1055
- Ok(checkpoint)
1056
- })
1057
- }
1058
- pub fn complete(
1059
- &mut self,
1060
- id: &str,
1061
- claim: &str,
1062
- worker: &str,
1063
- token: u64,
1064
- result_id: &str,
1065
- output: Value,
1066
- artifacts: Vec<String>,
1067
- schema_version: u64,
1068
- now: i64,
1069
- ) -> Result<WorkloadResult, WorkloadError> {
1070
- ensure_safe(&output)?;
1071
- if let Some(workload_id) = self.result_ids.get(result_id) {
1072
- if workload_id == id {
1073
- // A replay is the reporting attempt retrying the same report
1074
- // (e.g. after a lost response). Anything else naming this
1075
- // result id did not produce it and must not be told it landed.
1076
- let result = self.get(id)?.result.clone().unwrap();
1077
- let same_report = result.produced_by == worker
1078
- && result.provenance.get("claim_id").map(String::as_str) == Some(claim)
1079
- && result.provenance.get("fencing_token") == Some(&token.to_string())
1080
- && result.output_hash == hash(&output);
1081
- if same_report {
1082
- return Ok(result);
1083
- }
1084
- return Err(WorkloadError::new(
1085
- "WORKLOAD_RESULT_CONFLICT",
1086
- "the result id was already recorded by a different report",
1087
- ));
1088
- }
1089
- return Err(WorkloadError::new(
1090
- "WORKLOAD_RESULT_DUPLICATE",
1091
- "result id belongs to another workload",
1092
- ));
1093
- }
1094
- self.enforce_deadline(id, now)?;
1095
- let result = self.edit(id, |store, w| {
1096
- Self::fence(w, claim, worker, token, now)?;
1097
- if !matches!(
1098
- w.state,
1099
- WorkloadState::Running | WorkloadState::Checkpointed
1100
- ) {
1101
- return Err(WorkloadError::new(
1102
- "WORKLOAD_TRANSITION_DENIED",
1103
- "completion requires running state",
1104
- ));
1105
- }
1106
- let attempt = w.attempts.last_mut().unwrap();
1107
- let output_hash = hash(&output);
1108
- attempt.status = AttemptStatus::Succeeded;
1109
- attempt.ended_at = Some(now);
1110
- attempt.result_hash = Some(output_hash.clone());
1111
- let result = WorkloadResult {
1112
- result_id: result_id.into(),
1113
- workload_id: w.workload_id.clone(),
1114
- attempt_id: attempt.attempt_id.clone(),
1115
- status: "succeeded".into(),
1116
- output,
1117
- output_hash,
1118
- artifacts,
1119
- provenance: [
1120
- ("claim_id".into(), claim.into()),
1121
- ("fencing_token".into(), token.to_string()),
1122
- ]
1123
- .into(),
1124
- produced_by: worker.into(),
1125
- created_at: now,
1126
- schema_version,
1127
- };
1128
- w.result = Some(result.clone());
1129
- w.current_claim = None;
1130
- store.transition(w, WorkloadState::Succeeded, worker, now, BTreeMap::new())?;
1131
- Ok(result)
1132
- })?;
1133
- self.result_ids.insert(result_id.into(), id.into());
1134
- Ok(result)
1135
- }
1136
- pub fn fail(
1137
- &mut self,
1138
- id: &str,
1139
- claim: &str,
1140
- worker: &str,
1141
- token: u64,
1142
- kind: &str,
1143
- error: &str,
1144
- now: i64,
1145
- ) -> Result<WorkloadState, WorkloadError> {
1146
- self.fail_with_effect(id, claim, worker, token, kind, error, None, now)
1147
- }
1148
- /// A worker's failure report for its current attempt.
1149
- ///
1150
- /// Accepted from `CLAIMED` (the worker could not start) as well as from a
1151
- /// started attempt, fenced like every worker call. `effect` says what the
1152
- /// worker knows about external effects: `none` — it failed before causing
1153
- /// any (implied for an attempt that never started); `unknown` — an effect
1154
- /// may have happened. Without it, the policy decides by `kind`, as before.
1155
- /// An `unknown` effect is decided like a lost worker: retried only when the
1156
- /// policy retries `worker_failure`, whatever `kind` says. Repeating the same
1157
- /// report replays its outcome; any other report from an attempt that
1158
- /// already reported gets the fence's answer (`CLAIM_INVALID`/`STALE_FENCE`).
1159
- #[allow(clippy::too_many_arguments)]
1160
- pub fn fail_with_effect(
1161
- &mut self,
1162
- id: &str,
1163
- claim: &str,
1164
- worker: &str,
1165
- token: u64,
1166
- kind: &str,
1167
- error: &str,
1168
- effect: Option<&str>,
1169
- now: i64,
1170
- ) -> Result<WorkloadState, WorkloadError> {
1171
- if sensitive_text(error) {
1172
- return Err(WorkloadError::new(
1173
- "WORKLOAD_SECRET_REJECTED",
1174
- "errors may not contain secret material",
1175
- ));
1176
- }
1177
- if effect.is_some_and(|e| !matches!(e, "none" | "unknown")) {
1178
- return Err(WorkloadError::new(
1179
- "WORKLOAD_INVALID",
1180
- "effect must be \"none\" or \"unknown\"",
1181
- ));
1182
- }
1183
- // The same report repeated (a lost response) replays its outcome. Any
1184
- // other report from an attempt that already reported is answered by
1185
- // the fence, as before: its claim is no longer current.
1186
- let current = self.fresh(id)?.clone();
1187
- if let Some(attempt) = reported_attempt(&current, claim, worker, token) {
1188
- let same = attempt.status == AttemptStatus::Failed
1189
- && attempt.error.as_deref() == Some(error)
1190
- && failure_event(&current, &attempt.attempt_id).is_some_and(|event| {
1191
- let recorded_effect = event.evidence.get("effect").map(String::as_str);
1192
- event.evidence.get("failure_kind").map(String::as_str) == Some(kind)
1193
- && (effect.is_none() || effect == recorded_effect)
1194
- });
1195
- if same {
1196
- return Ok(current.state);
1197
- }
1198
- }
1199
- self.enforce_deadline(id, now)?;
1200
- self.edit(id, |store, w| {
1201
- Self::fence(w, claim, worker, token, now)?;
1202
- let started = w.state != WorkloadState::Claimed;
1203
- let effect = effect.or((!started).then_some("none"));
1204
- let attempt = w.attempts.last_mut().unwrap();
1205
- attempt.status = AttemptStatus::Failed;
1206
- attempt.ended_at = Some(now);
1207
- attempt.error = Some(error.into());
1208
- let attempt_id = attempt.attempt_id.clone();
1209
- w.last_failure = Some(format!("{kind}: {error}"));
1210
- w.current_claim = None;
1211
- let mut evidence: BTreeMap<String, String> = [
1212
- ("failure_kind".into(), kind.into()),
1213
- ("attempt_id".into(), attempt_id),
1214
- ]
1215
- .into();
1216
- if let Some(effect) = effect {
1217
- evidence.insert("effect".into(), effect.into());
1218
- }
1219
- store.transition(w, WorkloadState::Failed, worker, now, evidence)?;
1220
- let unknown = effect == Some("unknown");
1221
- let allowed =
1222
- w.retry_policy
1223
- .retry_on
1224
- .contains(if unknown { "worker_failure" } else { kind });
1225
- let decision = if !allowed && unknown {
1226
- "effect_unknown_not_retryable"
1227
- } else if !allowed {
1228
- "failure_kind_not_retryable"
1229
- } else if w.attempt_count >= attempt_budget(w) {
1230
- "attempt_budget_exhausted"
1231
- } else {
1232
- "retry"
1233
- };
1234
- let retryable = decision == "retry";
1235
- let evidence: BTreeMap<String, String> = [("decision".into(), decision.into())].into();
1236
- if retryable {
1237
- w.retry_count += 1;
1238
- let multiplier = if w.retry_policy.backoff == BackoffPolicy::Exponential {
1239
- 2u64.saturating_pow(w.retry_count.saturating_sub(1))
1240
- } else {
1241
- 1
1242
- };
1243
- let delay = w
1244
- .retry_policy
1245
- .initial_delay_ms
1246
- .saturating_mul(multiplier)
1247
- .min(w.retry_policy.max_delay_ms);
1248
- w.next_attempt_at = Some(now + (delay / 1000) as i64);
1249
- store.transition(w, WorkloadState::Ready, "rust.retry", now, evidence)?;
1250
- } else {
1251
- store.transition(w, WorkloadState::DeadLetter, "rust.retry", now, evidence)?;
1252
- }
1253
- Ok(w.state)
1254
- })
1255
- }
1256
- pub fn cancel(
1257
- &mut self,
1258
- id: &str,
1259
- actor: &str,
1260
- now: i64,
1261
- ) -> Result<WorkloadState, WorkloadError> {
1262
- self.edit(id, |store, w| {
1263
- match w.state {
1264
- WorkloadState::Ready
1265
- | WorkloadState::Created
1266
- | WorkloadState::Failed
1267
- | WorkloadState::DeadLetter => {
1268
- w.current_claim = None;
1269
- store.transition(w, WorkloadState::Cancelled, actor, now, BTreeMap::new())?
1270
- }
1271
- WorkloadState::Claimed | WorkloadState::Running | WorkloadState::Checkpointed => {
1272
- store.transition(
1273
- w,
1274
- WorkloadState::CancellationRequested,
1275
- actor,
1276
- now,
1277
- BTreeMap::new(),
1278
- )?
1279
- }
1280
- WorkloadState::CancellationRequested => {}
1281
- _ => {
1282
- return Err(WorkloadError::new(
1283
- "WORKLOAD_TRANSITION_DENIED",
1284
- "terminal workload cannot be cancelled",
1285
- ))
1286
- }
1287
- }
1288
- Ok(w.state)
1289
- })
1290
- }
1291
- pub fn confirm_cancel(
1292
- &mut self,
1293
- id: &str,
1294
- claim: &str,
1295
- worker: &str,
1296
- token: u64,
1297
- now: i64,
1298
- ) -> Result<(), WorkloadError> {
1299
- self.edit(id, |store, w| {
1300
- Self::fence(w, claim, worker, token, now)?;
1301
- if w.state != WorkloadState::CancellationRequested {
1302
- return Err(WorkloadError::new(
1303
- "WORKLOAD_TRANSITION_DENIED",
1304
- "cancellation was not requested",
1305
- ));
1306
- }
1307
- if let Some(attempt) = w.attempts.last_mut() {
1308
- attempt.status = AttemptStatus::Cancelled;
1309
- attempt.ended_at = Some(now);
1310
- }
1311
- w.current_claim = None;
1312
- store.transition(w, WorkloadState::Cancelled, worker, now, BTreeMap::new())
1313
- })
1314
- }
1315
- pub fn timeout(
1316
- &mut self,
1317
- id: &str,
1318
- actor: &str,
1319
- now: i64,
1320
- ) -> Result<WorkloadState, WorkloadError> {
1321
- self.edit(id, |store, w| {
1322
- if !matches!(
1323
- w.state,
1324
- WorkloadState::Ready
1325
- | WorkloadState::Claimed
1326
- | WorkloadState::Running
1327
- | WorkloadState::Checkpointed
1328
- ) {
1329
- return Err(WorkloadError::new(
1330
- "WORKLOAD_TRANSITION_DENIED",
1331
- "workload cannot time out in its current state",
1332
- ));
1333
- }
1334
- // A Ready workload has no attempt in progress: the last attempt's
1335
- // recorded outcome is not this timeout's to overwrite.
1336
- if w.state != WorkloadState::Ready {
1337
- if let Some(attempt) = w.attempts.last_mut() {
1338
- attempt.status = AttemptStatus::TimedOut;
1339
- attempt.ended_at = Some(now);
1340
- }
1341
- }
1342
- w.current_claim = None;
1343
- store.transition(w, WorkloadState::TimedOut, actor, now, BTreeMap::new())?;
1344
- if w.attempt_count < attempt_budget(w) && w.retry_policy.retry_on.contains("timeout")
1345
- {
1346
- w.retry_count += 1;
1347
- w.next_attempt_at = Some(now + (w.retry_policy.initial_delay_ms / 1000) as i64);
1348
- store.transition(w, WorkloadState::Ready, "rust.retry", now, BTreeMap::new())?;
1349
- } else {
1350
- store.transition(
1351
- w,
1352
- WorkloadState::DeadLetter,
1353
- "rust.timeout",
1354
- now,
1355
- BTreeMap::new(),
1356
- )?;
1357
- }
1358
- Ok(w.state)
1359
- })
1360
- }
1361
- pub fn retry(&mut self, id: &str, actor: &str, now: i64) -> Result<(), WorkloadError> {
1362
- self.retry_with_reason(id, actor, now, None)
1363
- }
1364
- /// An operator's re-arm of failed, dead-lettered or timed-out work.
1365
- ///
1366
- /// Each re-arm grants one more attempt, as it always has, and is recorded
1367
- /// in the history with its actor and reason. A workload may be re-armed at
1368
- /// most `retry_policy.max_attempts` times, so operator retries cannot
1369
- /// become an unbounded loop.
1370
- pub fn retry_with_reason(
1371
- &mut self,
1372
- id: &str,
1373
- actor: &str,
1374
- now: i64,
1375
- reason: Option<&str>,
1376
- ) -> Result<(), WorkloadError> {
1377
- self.edit(id, |store, w| {
1378
- if !matches!(
1379
- w.state,
1380
- WorkloadState::Failed | WorkloadState::DeadLetter | WorkloadState::TimedOut
1381
- ) {
1382
- return Err(WorkloadError::new(
1383
- "WORKLOAD_TRANSITION_DENIED",
1384
- "workload is not retryable",
1385
- ));
1386
- }
1387
- let previous = operator_retries(w);
1388
- if previous >= w.retry_policy.max_attempts {
1389
- return Err(WorkloadError::new(
1390
- "WORKLOAD_RETRY_EXHAUSTED",
1391
- "workload has used every operator retry its policy allows",
1392
- ));
1393
- }
1394
- w.next_attempt_at = Some(now);
1395
- w.current_claim = None;
1396
- let mut evidence: BTreeMap<String, String> = [
1397
- ("retry".into(), "operator".into()),
1398
- ("operator_retry".into(), (previous + 1).to_string()),
1399
- (
1400
- "attempt_budget".into(),
1401
- w.retry_policy
1402
- .max_attempts
1403
- .saturating_add(previous + 1)
1404
- .to_string(),
1405
- ),
1406
- ]
1407
- .into();
1408
- if let Some(reason) = reason.filter(|reason| !reason.trim().is_empty()) {
1409
- evidence.insert("reason".into(), reason.chars().take(256).collect());
1410
- }
1411
- store.transition(w, WorkloadState::Ready, actor, now, evidence)
1412
- })
1413
- }
1414
- /// Time out every workload past one of its deadlines (see
1415
- /// [`deadline_breach`]), as the recovery pass does each second. Deadlines
1416
- /// are computed from the workload's committed timestamps, so the decision
1417
- /// is the same after a restart and on any writer of the store.
1418
- pub fn enforce_deadlines(&mut self, now: i64) -> Result<Vec<String>, WorkloadError> {
1419
- self.refresh()?;
1420
- let due = self
1421
- .workloads
1422
- .values()
1423
- .filter_map(|w| deadline_breach(w, now).map(|breach| (w.workload_id.clone(), breach)))
1424
- .collect::<Vec<_>>();
1425
- let mut expired = Vec::with_capacity(due.len());
1426
- for (id, (deadline, at)) in due {
1427
- match self.expire(&id, deadline, at, now) {
1428
- Ok(()) => expired.push(id),
1429
- Err(error) if error.code == "WORKLOAD_CONFLICT" => {}
1430
- Err(error) => return Err(error),
1431
- }
1432
- }
1433
- Ok(expired)
1434
- }
1435
- /// Refuse a worker call on a workload past a deadline, timing it out
1436
- /// first, in the same way the recovery pass would have.
1437
- fn enforce_deadline(&mut self, id: &str, now: i64) -> Result<(), WorkloadError> {
1438
- let Some((deadline, at)) = deadline_breach(self.fresh(id)?, now) else {
1439
- return Ok(());
1440
- };
1441
- self.expire(id, deadline, at, now)?;
1442
- let mut error = WorkloadError::new(
1443
- "WORKLOAD_TIMED_OUT",
1444
- format!("the workload's {deadline} deadline passed"),
1445
- );
1446
- error.details = Some(serde_json::json!({ "deadline": deadline, "deadline_at": at }));
1447
- Err(error)
1448
- }
1449
- fn expire(&mut self, id: &str, deadline: &str, at: i64, now: i64) -> Result<(), WorkloadError> {
1450
- self.edit(id, |store, w| {
1451
- // Re-check on the committed version: a heartbeat or completion that
1452
- // committed first decides, and this changes nothing.
1453
- if deadline_breach(w, now).map(|(kind, _)| kind) != Some(deadline) {
1454
- return Err(WorkloadError::new(
1455
- "WORKLOAD_CONFLICT",
1456
- "the workload changed before its deadline was enforced",
1457
- ));
1458
- }
1459
- let lost = w.current_claim.take();
1460
- if w.state != WorkloadState::Ready {
1461
- if let Some(attempt) = w.attempts.last_mut() {
1462
- attempt.status = AttemptStatus::TimedOut;
1463
- attempt.ended_at = Some(now);
1464
- attempt.error = Some(format!("timeout: {deadline} deadline passed"));
1465
- }
1466
- }
1467
- let mut evidence: BTreeMap<String, String> = [
1468
- ("deadline".into(), deadline.into()),
1469
- ("deadline_at".into(), at.to_string()),
1470
- ]
1471
- .into();
1472
- if let Some(claim) = &lost {
1473
- evidence.insert("claim_id".into(), claim.claim_id.clone());
1474
- evidence.insert("worker_id".into(), claim.worker_id.clone());
1475
- }
1476
- store.transition(w, WorkloadState::TimedOut, "rust.deadline", now, evidence)?;
1477
- // The overall and queue deadlines bound the work, not an attempt:
1478
- // another attempt cannot meet them.
1479
- let decision = if matches!(deadline, "overall" | "queue") {
1480
- "deadline_not_retryable"
1481
- } else if !w.retry_policy.retry_on.contains("timeout") {
1482
- "timeout_not_retryable"
1483
- } else if w.attempt_count >= attempt_budget(w) {
1484
- "attempt_budget_exhausted"
1485
- } else {
1486
- "retry"
1487
- };
1488
- let evidence: BTreeMap<String, String> = [("decision".into(), decision.into())].into();
1489
- if decision == "retry" {
1490
- w.retry_count += 1;
1491
- w.next_attempt_at = Some(now + (w.retry_policy.initial_delay_ms / 1000) as i64);
1492
- store.transition(w, WorkloadState::Ready, "rust.deadline", now, evidence)?;
1493
- } else {
1494
- store.transition(w, WorkloadState::DeadLetter, "rust.deadline", now, evidence)?;
1495
- }
1496
- Ok(())
1497
- })
1498
- }
1499
- pub fn recover_expired(&mut self, now: i64) -> Result<Vec<String>, WorkloadError> {
1500
- self.refresh()?;
1501
- let ids = self
1502
- .workloads
1503
- .values()
1504
- .filter(|w| {
1505
- w.current_claim
1506
- .as_ref()
1507
- .is_some_and(|c| c.expires_at <= now)
1508
- && matches!(
1509
- w.state,
1510
- WorkloadState::Claimed
1511
- | WorkloadState::Running
1512
- | WorkloadState::Checkpointed
1513
- | WorkloadState::CancellationRequested
1514
- )
1515
- })
1516
- .map(|w| w.workload_id.clone())
1517
- .collect::<Vec<_>>();
1518
- let mut recovered = Vec::with_capacity(ids.len());
1519
- for id in ids {
1520
- let outcome = self.edit(&id, |store, w| {
1521
- // Decide on the committed version: another writer may have
1522
- // recovered (or the worker completed) it since it was listed.
1523
- let still_expired = w
1524
- .current_claim
1525
- .as_ref()
1526
- .is_some_and(|c| c.expires_at <= now)
1527
- && matches!(
1528
- w.state,
1529
- WorkloadState::Claimed
1530
- | WorkloadState::Running
1531
- | WorkloadState::Checkpointed
1532
- | WorkloadState::CancellationRequested
1533
- );
1534
- if !still_expired {
1535
- return Err(WorkloadError::new(
1536
- "WORKLOAD_CONFLICT",
1537
- "the workload changed before it was recovered",
1538
- ));
1539
- }
1540
- let lost = w.current_claim.take();
1541
- if let Some(attempt) = w.attempts.last_mut() {
1542
- attempt.status = AttemptStatus::WorkerLost;
1543
- attempt.ended_at = Some(now);
1544
- attempt.error = Some("worker_lost: claim lease expired".into());
1545
- }
1546
- let mut evidence: BTreeMap<String, String> = [
1547
- ("recovery_reason".into(), "lease_expired".into()),
1548
- ("attempts_used".into(), w.attempt_count.to_string()),
1549
- ("attempt_budget".into(), attempt_budget(w).to_string()),
1550
- ]
1551
- .into();
1552
- if let Some(claim) = &lost {
1553
- evidence.insert("claim_id".into(), claim.claim_id.clone());
1554
- evidence.insert("worker_id".into(), claim.worker_id.clone());
1555
- evidence.insert("lease_expired_at".into(), claim.expires_at.to_string());
1556
- }
1557
- if let Some(attempt) = w.attempts.last() {
1558
- evidence.insert("attempt_id".into(), attempt.attempt_id.clone());
1559
- }
1560
- if w.state == WorkloadState::CancellationRequested {
1561
- store.transition(w, WorkloadState::Cancelled, "rust.recovery", now, evidence)?;
1562
- } else {
1563
- store.transition(
1564
- w,
1565
- WorkloadState::Recovering,
1566
- "rust.recovery",
1567
- now,
1568
- evidence,
1569
- )?;
1570
- // A lost worker is the policy's `worker_failure`. Whether
1571
- // the lost attempt had effects is unknown to FeltDB, so a
1572
- // retry re-executes it: at-least-once, never exactly-once.
1573
- let (to, decision) = if !w.retry_policy.retry_on.contains("worker_failure") {
1574
- (WorkloadState::DeadLetter, "worker_failure_not_retryable")
1575
- } else if w.attempt_count >= attempt_budget(w) {
1576
- (WorkloadState::DeadLetter, "attempt_budget_exhausted")
1577
- } else {
1578
- (WorkloadState::Ready, "retry")
1579
- };
1580
- store.transition(
1581
- w,
1582
- to,
1583
- "rust.recovery",
1584
- now,
1585
- [("decision".into(), decision.into())].into(),
1586
- )?;
1587
- }
1588
- Ok(())
1589
- });
1590
- match outcome {
1591
- Ok(()) => recovered.push(id),
1592
- // Another writer of this store committed the workload first —
1593
- // in a race, its own recovery. Nothing of this attempt was
1594
- // committed and `edit` has adopted the committed workload, so
1595
- // the attempt is recovered once, by that writer. If it still
1596
- // needs recovery, the next pass sees it.
1597
- Err(error) if error.code == "WORKLOAD_CONFLICT" => {}
1598
- Err(error) => return Err(error),
1599
- }
1600
- }
1601
- Ok(recovered)
1602
- }
1603
- }
1604
-
1605
- pub trait WorkScheduler {
1606
- fn enqueue(&mut self, workload_id: &str);
1607
- fn claim(
1608
- &mut self,
1609
- store: &mut WorkloadStore,
1610
- tenant: &str,
1611
- application: &str,
1612
- worker: &str,
1613
- capabilities: &BTreeSet<String>,
1614
- now: i64,
1615
- lease_ms: u64,
1616
- ) -> Result<Option<WorkloadClaim>, WorkloadError>;
1617
- fn release(&mut self, claim_id: &str);
1618
- }
1619
- #[derive(Default)]
1620
- pub struct LocalDurableScheduler {
1621
- queue: BTreeSet<String>,
1622
- }
1623
- impl WorkScheduler for LocalDurableScheduler {
1624
- fn enqueue(&mut self, id: &str) {
1625
- self.queue.insert(id.into());
1626
- }
1627
- fn claim(
1628
- &mut self,
1629
- store: &mut WorkloadStore,
1630
- tenant: &str,
1631
- application: &str,
1632
- worker: &str,
1633
- capabilities: &BTreeSet<String>,
1634
- now: i64,
1635
- lease_ms: u64,
1636
- ) -> Result<Option<WorkloadClaim>, WorkloadError> {
1637
- let selected = store
1638
- .list(tenant, application)
1639
- .into_iter()
1640
- .filter(|w| {
1641
- // Eligibility is part of selection, not a reason to fail. A
1642
- // workload that is merely early is skipped so the scheduler can
1643
- // hand out work that is ready now; without this it would pick an
1644
- // ineligible workload, have `store.claim` refuse it, and return
1645
- // an error while claimable work sat behind it.
1646
- w.state == WorkloadState::Ready
1647
- && w.is_eligible_at(now)
1648
- && w.capability_snapshot.is_subset(capabilities)
1649
- })
1650
- .find(|w| {
1651
- w.dependencies.iter().all(|d| {
1652
- store
1653
- .workloads
1654
- .get(&d.workload_id)
1655
- .is_some_and(|dependency| dependency.state == d.required_state)
1656
- })
1657
- })
1658
- .map(|w| w.workload_id);
1659
- match selected {
1660
- Some(id) => store
1661
- .claim(&id, worker, capabilities, now, lease_ms)
1662
- .map(Some),
1663
- None => Ok(None),
1664
- }
1665
- }
1666
- fn release(&mut self, _: &str) {}
1667
- }
1668
- fn legal(from: WorkloadState, to: WorkloadState) -> bool {
1669
- use WorkloadState::*;
1670
- matches!(
1671
- (from, to),
1672
- (Created, Ready)
1673
- | (Ready, Claimed)
1674
- | (Claimed, Running)
1675
- | (Running, Checkpointed)
1676
- | (Checkpointed, Checkpointed)
1677
- | (Running, Succeeded)
1678
- | (Checkpointed, Succeeded)
1679
- | (Claimed, CancellationRequested)
1680
- | (Running, CancellationRequested)
1681
- | (Checkpointed, CancellationRequested)
1682
- | (CancellationRequested, Cancelled)
1683
- | (Ready, Cancelled)
1684
- | (Created, Cancelled)
1685
- | (Failed, Cancelled)
1686
- | (DeadLetter, Cancelled)
1687
- | (Claimed, Failed)
1688
- | (Running, Failed)
1689
- | (Checkpointed, Failed)
1690
- | (Claimed, Recovering)
1691
- | (Running, Recovering)
1692
- | (Checkpointed, Recovering)
1693
- | (Recovering, Ready)
1694
- | (Failed, Ready)
1695
- | (TimedOut, Ready)
1696
- | (DeadLetter, Ready)
1697
- | (Failed, DeadLetter)
1698
- | (Running, TimedOut)
1699
- | (Claimed, TimedOut)
1700
- | (Checkpointed, TimedOut)
1701
- | (Ready, TimedOut)
1702
- | (TimedOut, DeadLetter)
1703
- | (Recovering, DeadLetter)
1704
- | (Ready, DeadLetter)
1705
- )
1706
- }
1707
- pub fn can_transition(from: WorkloadState, to: WorkloadState) -> bool {
1708
- legal(from, to)
1709
- }
1710
- fn event_name(state: WorkloadState) -> &'static str {
1711
- use WorkloadState::*;
1712
- match state {
1713
- Created => "workload.created",
1714
- Ready => "workload.ready",
1715
- Claimed => "workload.claimed",
1716
- Running => "workload.started",
1717
- Checkpointed => "workload.checkpointed",
1718
- Recovering => "workload.recovered",
1719
- CancellationRequested => "workload.cancellation_requested",
1720
- Succeeded => "workload.completed",
1721
- Failed => "workload.failed",
1722
- Cancelled => "workload.cancelled",
1723
- TimedOut => "workload.timed_out",
1724
- DeadLetter => "workload.dead_lettered",
1725
- }
1726
- }
1727
- fn ensure_safe(value: &Value) -> Result<(), WorkloadError> {
1728
- fn walk(value: &Value) -> bool {
1729
- match value {
1730
- Value::Object(map) => map.iter().any(|(k, v)| {
1731
- matches!(
1732
- k.to_ascii_lowercase().as_str(),
1733
- "password" | "secret" | "token" | "api_key" | "private_key"
1734
- ) || walk(v)
1735
- }),
1736
- Value::Array(values) => values.iter().any(walk),
1737
- Value::String(v) => sensitive_text(v),
1738
- _ => false,
1739
- }
1740
- }
1741
- if walk(value) {
1742
- Err(WorkloadError::new(
1743
- "WORKLOAD_SECRET_REJECTED",
1744
- "plaintext secrets are forbidden in workload state",
1745
- ))
1746
- } else {
1747
- Ok(())
1748
- }
1749
- }
1750
- fn sensitive_text(v: &str) -> bool {
1751
- v.contains("BEGIN PRIVATE KEY") || v.starts_with("sk-") || v.starts_with("fdb_live_")
1752
- }
1753
- /// The identity a caller's idempotency key is unique within: tenant,
1754
- /// application and requester. Encoded as a JSON array, so no choice of key or
1755
- /// identifier can make two different scopes coincide.
1756
- fn idempotency_scope(tenant: &str, application: &str, requester: &Subject, key: &str) -> String {
1757
- serde_json::json!([tenant, application, requester.kind(), requester.id(), key]).to_string()
1758
- }
1759
- /// The scope before keys were bound to their requester, kept only to verify
1760
- /// legacy store files during migration.
1761
- fn legacy_idempotency_scope(
1762
- tenant: &str,
1763
- application: &str,
1764
- _requester: &Subject,
1765
- key: &str,
1766
- ) -> String {
1767
- format!("{tenant}:{application}:{key}")
1768
- }
1769
- fn idempotency_index(
1770
- workloads: &BTreeMap<String, Workload>,
1771
- scope: fn(&str, &str, &Subject, &str) -> String,
1772
- ) -> BTreeMap<String, String> {
1773
- workloads
1774
- .values()
1775
- .map(|w| {
1776
- (
1777
- scope(&w.tenant_id, &w.application_id, &w.requested_by, &w.idempotency_key),
1778
- w.workload_id.clone(),
1779
- )
1780
- })
1781
- .collect()
1782
- }
1783
- /// Answer a repeated key: the original workload for the same request, a
1784
- /// refusal for a different one. Never the original for a different request.
1785
- /// The execution-affecting part of an admission, normalized, field by field.
1786
- ///
1787
- /// A replay under the same key is answered with the original workload only when
1788
- /// every one of these is equal. Optional fields are compared after their
1789
- /// defaults apply (serde fills an omitted `retry_policy`, `timeouts`,
1790
- /// `priority`, `connection_scopes` or `dependencies`); sets compare as sets and
1791
- /// dependencies regardless of order; the input compares by its canonical hash,
1792
- /// so member order is not part of it. Provenance that does not change what runs
1793
- /// — `trigger`, `session_id`, `created_at`, and the authorization snapshot,
1794
- /// which is verified before any replay — is not compared.
1795
- fn execution_fields(
1796
- definition_id: &str,
1797
- definition_revision: &str,
1798
- environment: &str,
1799
- input_hash: &str,
1800
- capability_snapshot: &BTreeSet<String>,
1801
- connection_scopes: &BTreeSet<String>,
1802
- priority: i32,
1803
- retry_policy: &WorkloadRetryPolicy,
1804
- timeouts: &WorkloadTimeouts,
1805
- dependencies: &[WorkloadDependency],
1806
- not_before: Option<i64>,
1807
- ) -> Vec<(&'static str, Value)> {
1808
- let mut dependencies = dependencies
1809
- .iter()
1810
- .map(|d| serde_json::json!([d.workload_id, d.required_state]))
1811
- .collect::<Vec<_>>();
1812
- dependencies.sort_by_key(|d| d.to_string());
1813
- vec![
1814
- ("definition_id", definition_id.into()),
1815
- ("definition_revision", definition_revision.into()),
1816
- ("environment", environment.into()),
1817
- ("input", input_hash.into()),
1818
- (
1819
- "capability_snapshot",
1820
- serde_json::json!(capability_snapshot),
1821
- ),
1822
- ("connection_scopes", serde_json::json!(connection_scopes)),
1823
- ("priority", priority.into()),
1824
- ("retry_policy", serde_json::json!(retry_policy)),
1825
- ("timeouts", serde_json::json!(timeouts)),
1826
- ("dependencies", Value::Array(dependencies)),
1827
- ("not_before", serde_json::json!(not_before)),
1828
- ]
1829
- }
1830
- /// The `not_before` a workload was admitted with, from its creation event
1831
- /// (`next_attempt_at` moves with retries; the admission evidence does not).
1832
- fn admitted_not_before(w: &Workload) -> Option<i64> {
1833
- w.execution_history
1834
- .iter()
1835
- .find(|event| event.to == WorkloadState::Ready)
1836
- .and_then(|event| event.evidence.get("not_before"))
1837
- .and_then(|value| value.parse().ok())
1838
- }
1839
- fn replay(existing: Workload, input: &CreateWorkload) -> Result<(Workload, bool), WorkloadError> {
1840
- let recorded = execution_fields(
1841
- &existing.definition_id,
1842
- &existing.definition_revision,
1843
- &existing.environment,
1844
- &existing.input_hash,
1845
- &existing.capability_snapshot,
1846
- &existing.connection_scopes,
1847
- existing.priority,
1848
- &existing.retry_policy,
1849
- &existing.timeouts,
1850
- &existing.dependencies,
1851
- admitted_not_before(&existing),
1852
- );
1853
- let requested = execution_fields(
1854
- &input.definition_id,
1855
- &input.definition_revision,
1856
- &input.environment,
1857
- &hash(&input.input),
1858
- &input.capability_snapshot,
1859
- &input.connection_scopes,
1860
- input.priority,
1861
- &input.retry_policy,
1862
- &input.timeouts,
1863
- &input.dependencies,
1864
- input.not_before,
1865
- );
1866
- let mut conflicting = recorded
1867
- .iter()
1868
- .zip(&requested)
1869
- .filter(|((_, left), (_, right))| left != right)
1870
- .map(|((name, _), _)| *name)
1871
- .collect::<Vec<_>>();
1872
- if existing.requested_by != input.requested_by {
1873
- conflicting.insert(0, "requested_by");
1874
- }
1875
- if conflicting.is_empty() {
1876
- return Ok((existing, false));
1877
- }
1878
- let mut error = WorkloadError::new(
1879
- "IDEMPOTENCY_KEY_REUSED",
1880
- "idempotency key was already used for a different request",
1881
- );
1882
- error.details = Some(serde_json::json!({ "fields": conflicting }));
1883
- Err(error)
1884
- }
1885
- /// Seconds a millisecond timeout lasts, rounded up so a deadline is never early.
1886
- fn deadline_seconds(ms: u64) -> i64 {
1887
- ms.div_ceil(1000) as i64
1888
- }
1889
- /// The first deadline a workload has passed at `now`, and when it fell.
1890
- ///
1891
- /// | Deadline | Measured from | While |
1892
- /// | --- | --- | --- |
1893
- /// | `overall` | admission (`created_at`) | not terminal |
1894
- /// | `queue` | becoming `READY` (or its `next_attempt_at`, if later) | `READY` |
1895
- /// | `claim` | the claim | `CLAIMED` |
1896
- /// | `execution` | the attempt's start | `RUNNING`, `CHECKPOINTED` |
1897
- /// | `heartbeat` | the last heartbeat (or the claim) | claimed |
1898
- ///
1899
- /// Only workloads admitted at [`DEADLINE_CONTRACT_VERSION`] or later.
1900
- pub fn deadline_breach(w: &Workload, now: i64) -> Option<(&'static str, i64)> {
1901
- use WorkloadState::*;
1902
- if w.contract_version < DEADLINE_CONTRACT_VERSION {
1903
- return None;
1904
- }
1905
- let active = matches!(w.state, Ready | Claimed | Running | Checkpointed);
1906
- if !active {
1907
- return None;
1908
- }
1909
- let t = &w.timeouts;
1910
- let claim = w.current_claim.as_ref();
1911
- let candidates = [
1912
- (
1913
- "overall",
1914
- t.overall_ms.map(|ms| w.created_at + deadline_seconds(ms)),
1915
- ),
1916
- (
1917
- "queue",
1918
- t.queue_ms.filter(|_| w.state == Ready).map(|ms| {
1919
- let ready_since = w
1920
- .execution_history
1921
- .iter()
1922
- .rev()
1923
- .find(|event| event.to == Ready)
1924
- .map_or(w.created_at, |event| event.at);
1925
- ready_since.max(w.next_attempt_at.unwrap_or(i64::MIN)) + deadline_seconds(ms)
1926
- }),
1927
- ),
1928
- (
1929
- "claim",
1930
- t.claim_ms
1931
- .filter(|_| w.state == Claimed)
1932
- .and_then(|ms| claim.map(|c| c.claimed_at + deadline_seconds(ms))),
1933
- ),
1934
- (
1935
- "execution",
1936
- t.execution_ms
1937
- .filter(|_| matches!(w.state, Running | Checkpointed))
1938
- .and_then(|ms| {
1939
- w.attempts
1940
- .last()
1941
- .and_then(|a| a.started_at)
1942
- .map(|started| started + deadline_seconds(ms))
1943
- }),
1944
- ),
1945
- (
1946
- "heartbeat",
1947
- t.heartbeat_ms
1948
- .and_then(|ms| claim.map(|c| c.last_heartbeat + deadline_seconds(ms))),
1949
- ),
1950
- ];
1951
- candidates
1952
- .into_iter()
1953
- .find_map(|(kind, at)| at.filter(|at| now >= *at).map(|at| (kind, at)))
1954
- }
1955
- /// The attempt this fence named, when that attempt has already reported its
1956
- /// outcome (failed or succeeded). Lost and timed-out attempts are not reports.
1957
- fn reported_attempt<'w>(
1958
- w: &'w Workload,
1959
- claim: &str,
1960
- worker: &str,
1961
- token: u64,
1962
- ) -> Option<&'w WorkloadAttempt> {
1963
- w.attempts.iter().find(|attempt| {
1964
- attempt.claim_id == claim
1965
- && attempt.worker_id == worker
1966
- && attempt.fencing_token == token
1967
- && matches!(
1968
- attempt.status,
1969
- AttemptStatus::Failed | AttemptStatus::Succeeded
1970
- )
1971
- })
1972
- }
1973
- /// The `FAILED` event a failure report of `attempt_id` recorded.
1974
- fn failure_event<'w>(w: &'w Workload, attempt_id: &str) -> Option<&'w WorkloadEvent> {
1975
- w.execution_history.iter().rev().find(|event| {
1976
- event.to == WorkloadState::Failed
1977
- && event.evidence.get("attempt_id").map(String::as_str) == Some(attempt_id)
1978
- })
1979
- }
1980
- /// Operator re-arms recorded in the workload's own history.
1981
- fn operator_retries(w: &Workload) -> u32 {
1982
- w.execution_history
1983
- .iter()
1984
- .filter(|event| event.evidence.get("retry").map(String::as_str) == Some("operator"))
1985
- .count() as u32
1986
- }
1987
- /// How many claims the workload may have in total: its policy's
1988
- /// `max_attempts`, plus one for each operator re-arm (which is what an operator
1989
- /// retry has always granted).
1990
- fn attempt_budget(w: &Workload) -> u32 {
1991
- w.retry_policy
1992
- .max_attempts
1993
- .saturating_add(operator_retries(w))
1994
- }
1995
- fn work_key(workload: &Workload) -> String {
1996
- format!(
1997
- "{}/{}/{}",
1998
- workload.tenant_id, workload.application_id, workload.workload_id
1999
- )
2000
- }
2001
- fn derived_event_sequence(workloads: &BTreeMap<String, Workload>) -> u64 {
2002
- workloads
2003
- .values()
2004
- .flat_map(|workload| workload.execution_history.iter())
2005
- .map(|event| event.sequence)
2006
- .max()
2007
- .unwrap_or_default()
2008
- }
2009
- fn runtime_id(prefix: &str) -> String {
2010
- let value: String = rand::thread_rng()
2011
- .sample_iter(&Alphanumeric)
2012
- .take(24)
2013
- .map(char::from)
2014
- .collect();
2015
- format!("{prefix}_{value}")
2016
- }
2017
- fn io(e: impl std::fmt::Display) -> WorkloadError {
2018
- WorkloadError::new("WORKLOAD_STORAGE_FAILURE", e.to_string())
2019
- }
2020
- fn workload_storage_error(error: FlowError) -> WorkloadError {
2021
- match error {
2022
- FlowError::PreconditionFailed(_) | FlowError::RevisionConflict { .. } => {
2023
- WorkloadError::new("WORKLOAD_CONFLICT", error.to_string())
2024
- }
2025
- FlowError::CapabilityError(ref message) if message.starts_with("PRECONDITION_FAILED:") => {
2026
- WorkloadError::new("WORKLOAD_CONFLICT", error.to_string())
2027
- }
2028
- _ => io(error),
2029
- }
2030
- }
2031
- fn hash(value: &Value) -> String {
2032
- hex(&Sha256::digest(
2033
- serde_json::to_vec(value).unwrap_or_default(),
2034
- ))
2035
- }
2036
- fn hex(bytes: &[u8]) -> String {
2037
- bytes.iter().map(|v| format!("{v:02x}")).collect()
2038
- }
2039
-
2040
- #[cfg(test)]
2041
- mod tests {
2042
- use super::*;
2043
- use crate::authorization::{
2044
- Grant, GrantConstraints, GrantSigner, GrantStatus, ResourceScope, ResourceScopeKind,
2045
- };
2046
- use std::{fs, path::PathBuf};
2047
- fn create() -> CreateWorkload {
2048
- create_as("a1")
2049
- }
2050
- /// The fixture request, made by agent `agent`.
2051
- fn create_as(agent: &str) -> CreateWorkload {
2052
- let signer = GrantSigner::new("k", vec![1; 32]).unwrap();
2053
- let subject = Subject::Agent(agent.into());
2054
- let grant = Grant {
2055
- contract_version: 1,
2056
- id: "g".into(),
2057
- issuer: Subject::Human("h".into()),
2058
- subject: subject.clone(),
2059
- tenant_id: "t".into(),
2060
- application_id: "app".into(),
2061
- capabilities: ["workload:execute".into()].into(),
2062
- resources: vec![ResourceScope {
2063
- kind: ResourceScopeKind::Workload,
2064
- uri: "flow://app/workloads/*".into(),
2065
- }],
2066
- constraints: GrantConstraints::default(),
2067
- issued_at: 1,
2068
- expires_at: 9999,
2069
- parent_grant: None,
2070
- revision: 1,
2071
- status: GrantStatus::Active,
2072
- };
2073
- CreateWorkload {
2074
- tenant_id: "t".into(),
2075
- application_id: "app".into(),
2076
- environment: "production".into(),
2077
- definition_id: "send".into(),
2078
- definition_revision: "r1".into(),
2079
- trigger: "manual".into(),
2080
- input: serde_json::json!({"id":1}),
2081
- requested_by: subject,
2082
- session_id: Some("session-1".into()),
2083
- authorization_snapshot: signer.sign(grant).unwrap(),
2084
- capability_snapshot: ["email:send".into()].into(),
2085
- connection_scopes: BTreeSet::new(),
2086
- priority: 0,
2087
- idempotency_key: "one".into(),
2088
- retry_policy: Default::default(),
2089
- timeouts: Default::default(),
2090
- dependencies: vec![],
2091
- not_before: None,
2092
- created_at: 10,
2093
- }
2094
- }
2095
- fn capabilities() -> BTreeSet<String> {
2096
- ["email:send".into()].into()
2097
- }
2098
-
2099
- #[test]
2100
- fn idempotency_keys_are_bound_to_their_requester_and_request() {
2101
- let (path, mut store) = store("requester-scoped-keys");
2102
- let (original, created) = store.admit(create_as("a1")).unwrap();
2103
- assert!(created);
2104
-
2105
- // The same requester and request: the original, not a second workload.
2106
- let (replayed, created) = store.admit(create_as("a1")).unwrap();
2107
- assert!(!created);
2108
- assert_eq!(replayed.workload_id, original.workload_id);
2109
-
2110
- // The same key for a different request is refused and creates nothing.
2111
- let mut different = create_as("a1");
2112
- different.input = serde_json::json!({"id": 2});
2113
- assert_eq!(store.admit(different).unwrap_err().code, "IDEMPOTENCY_KEY_REUSED");
2114
- let mut other_definition = create_as("a1");
2115
- other_definition.definition_id = "other".into();
2116
- assert_eq!(
2117
- store.admit(other_definition).unwrap_err().code,
2118
- "IDEMPOTENCY_KEY_REUSED"
2119
- );
2120
-
2121
- // Another requester's identical key is its own workload: it neither
2122
- // returns, reveals nor blocks the first requester's.
2123
- let (theirs, created) = store.admit(create_as("a2")).unwrap();
2124
- assert!(created);
2125
- assert_ne!(theirs.workload_id, original.workload_id);
2126
- assert_eq!(store.workloads.len(), 2);
2127
-
2128
- // A snapshot that does not bind the requester is refused before any
2129
- // replay, so a key cannot be used to fetch a workload.
2130
- let mut unbound = create_as("a1");
2131
- unbound.requested_by = Subject::Agent("a2".into());
2132
- assert_eq!(store.admit(unbound).unwrap_err().code, "WORKLOAD_UNAUTHORIZED");
2133
- let _ = fs::remove_file(path);
2134
- }
2135
-
2136
- #[test]
2137
- fn lease_recovery_records_the_lost_worker_and_is_bounded_by_the_attempt_budget() {
2138
- let (path, mut store) = store("bounded-recovery");
2139
- let workload = store.create(create()).unwrap();
2140
- let id = workload.workload_id.clone();
2141
- for attempt in 1..=3u32 {
2142
- let now = 100 * attempt as i64;
2143
- let claim = store.claim(&id, "worker-a", &capabilities(), now, 10_000).unwrap();
2144
- assert_eq!(store.recover_expired(claim.expires_at).unwrap(), vec![id.clone()]);
2145
- let recovered = store.get(&id).unwrap();
2146
- let lost = recovered.attempts.last().unwrap();
2147
- assert_eq!(lost.status, AttemptStatus::WorkerLost);
2148
- assert_eq!(lost.claim_id, claim.claim_id);
2149
- let recovering = recovered
2150
- .execution_history
2151
- .iter()
2152
- .rev()
2153
- .find(|event| event.to == WorkloadState::Recovering)
2154
- .unwrap();
2155
- assert_eq!(recovering.evidence["recovery_reason"], "lease_expired");
2156
- assert_eq!(recovering.evidence["worker_id"], "worker-a");
2157
- assert_eq!(recovering.evidence["claim_id"], claim.claim_id);
2158
- assert_eq!(recovering.evidence["attempts_used"], attempt.to_string());
2159
- let decision = &recovered.execution_history.last().unwrap();
2160
- if attempt < 3 {
2161
- assert_eq!(recovered.state, WorkloadState::Ready);
2162
- assert_eq!(decision.evidence["decision"], "retry");
2163
- } else {
2164
- assert_eq!(recovered.state, WorkloadState::DeadLetter);
2165
- assert_eq!(decision.evidence["decision"], "attempt_budget_exhausted");
2166
- }
2167
- }
2168
- assert_eq!(store.get(&id).unwrap().attempt_count, 3);
2169
- assert_eq!(
2170
- store.claim(&id, "worker-b", &capabilities(), 1_000, 10_000).unwrap_err().code,
2171
- "WORKLOAD_NOT_READY"
2172
- );
2173
- let _ = fs::remove_file(path);
2174
- }
2175
-
2176
- /// The recovery history of `id` as committed, read by a fresh projection.
2177
- fn committed_workload(db: &FeltDb, id: &str) -> Workload {
2178
- WorkloadStore::from_db(db.clone()).unwrap().get(id).unwrap().clone()
2179
- }
2180
-
2181
- fn recoveries(workload: &Workload) -> usize {
2182
- workload
2183
- .execution_history
2184
- .iter()
2185
- .filter(|event| event.to == WorkloadState::Recovering)
2186
- .count()
2187
- }
2188
-
2189
- /// Two writers of one store recover the same expired workload at once.
2190
- fn race_recovery(db: &FeltDb, now: i64) -> (WorkloadStore, WorkloadStore, Vec<Vec<String>>) {
2191
- let stores = [
2192
- WorkloadStore::from_db(db.clone()).unwrap(),
2193
- WorkloadStore::from_db(db.clone()).unwrap(),
2194
- ];
2195
- let barrier = std::sync::Barrier::new(2);
2196
- let results = std::thread::scope(|scope| {
2197
- let handles = stores
2198
- .into_iter()
2199
- .map(|mut store| {
2200
- let barrier = &barrier;
2201
- scope.spawn(move || {
2202
- barrier.wait();
2203
- let recovered = store.recover_expired(now).unwrap();
2204
- (store, recovered)
2205
- })
2206
- })
2207
- .collect::<Vec<_>>();
2208
- handles
2209
- .into_iter()
2210
- .map(|handle| handle.join().unwrap())
2211
- .collect::<Vec<_>>()
2212
- });
2213
- let mut results = results.into_iter();
2214
- let (first, first_recovered) = results.next().unwrap();
2215
- let (second, second_recovered) = results.next().unwrap();
2216
- (first, second, vec![first_recovered, second_recovered])
2217
- }
2218
-
2219
- #[test]
2220
- fn concurrent_recovery_commits_one_transition_and_the_loser_converges() {
2221
- for round in 0..20 {
2222
- let (path, mut store) = store(&format!("concurrent-recovery-{round}"));
2223
- let db = store.db.clone();
2224
- let id = store.create(create()).unwrap().workload_id;
2225
- let claim = store.claim(&id, "worker-a", &capabilities(), 100, 10_000).unwrap();
2226
- // A writer that loaded the workload before either recovery.
2227
- // Commits land in this projection's window between refresh and
2228
- // commit: only the storage guard can refuse them.
2229
- let mut stale = WorkloadStore::from_db(db.clone())
2230
- .unwrap()
2231
- .without_refresh();
2232
-
2233
- let (mut first, mut second, recovered) = race_recovery(&db, claim.expires_at);
2234
- // Exactly one writer committed the recovery; the other reported none.
2235
- let mut outcomes = recovered.clone();
2236
- outcomes.sort();
2237
- assert_eq!(outcomes, vec![vec![], vec![id.clone()]], "round {round}");
2238
-
2239
- let committed = committed_workload(&db, &id);
2240
- assert_eq!(recoveries(&committed), 1, "one recovery transition");
2241
- assert_eq!(committed.state, WorkloadState::Ready);
2242
- assert_eq!(committed.attempt_count, 1);
2243
- assert_eq!(committed.attempts.len(), 1);
2244
- assert_eq!(committed.attempts[0].status, AttemptStatus::WorkerLost);
2245
- assert!(committed.current_claim.is_none());
2246
- let recovering = committed
2247
- .execution_history
2248
- .iter()
2249
- .find(|event| event.to == WorkloadState::Recovering)
2250
- .unwrap();
2251
- assert_eq!(recovering.evidence["claim_id"], claim.claim_id);
2252
- assert_eq!(recovering.evidence["attempts_used"], "1");
2253
-
2254
- // The losing writer adopted the committed workload instead of
2255
- // keeping its own, uncommitted view.
2256
- let expected = serde_json::to_value(&committed).unwrap();
2257
- assert_eq!(serde_json::to_value(first.get(&id).unwrap()).unwrap(), expected);
2258
- assert_eq!(serde_json::to_value(second.get(&id).unwrap()).unwrap(), expected);
2259
- // And neither recovers it again, or commits anything more.
2260
- assert!(first.recover_expired(claim.expires_at + 1).unwrap().is_empty());
2261
- assert!(second.recover_expired(claim.expires_at + 1).unwrap().is_empty());
2262
- assert_eq!(
2263
- serde_json::to_value(committed_workload(&db, &id)).unwrap(),
2264
- expected,
2265
- "no further commit"
2266
- );
2267
-
2268
- // The lost claimant cannot complete, even through a projection
2269
- // that still shows its claim as current and unexpired: the guarded
2270
- // commit refuses the stale version.
2271
- let refused = stale
2272
- .start(&id, &claim.claim_id, "worker-a", claim.fencing_token, claim.expires_at - 1)
2273
- .unwrap_err();
2274
- assert_eq!(refused.code, "WORKLOAD_CONFLICT");
2275
- assert_eq!(stale.get(&id).unwrap().state, WorkloadState::Ready);
2276
- let refused = stale
2277
- .complete(
2278
- &id,
2279
- &claim.claim_id,
2280
- "worker-a",
2281
- claim.fencing_token,
2282
- "late-result",
2283
- serde_json::json!({"late": true}),
2284
- vec![],
2285
- 1,
2286
- claim.expires_at - 1,
2287
- )
2288
- .unwrap_err();
2289
- assert_eq!(refused.code, "WORKLOAD_CLAIM_INVALID");
2290
- // Through a current projection its fence no longer matches.
2291
- let refused = first
2292
- .heartbeat(&id, &claim.claim_id, "worker-a", claim.fencing_token, claim.expires_at - 1, 10_000)
2293
- .unwrap_err();
2294
- assert_eq!(refused.code, "WORKLOAD_CLAIM_INVALID");
2295
- assert_eq!(serde_json::to_value(committed_workload(&db, &id)).unwrap(), expected);
2296
- drop((store, first, second, stale, db));
2297
- let _ = fs::remove_file(path);
2298
- }
2299
- }
2300
-
2301
- #[test]
2302
- fn concurrent_recovery_never_exceeds_the_attempt_budget() {
2303
- let (path, mut store) = store("concurrent-recovery-budget");
2304
- let db = store.db.clone();
2305
- let id = store.create(create()).unwrap().workload_id;
2306
- for attempt in 1..=3i64 {
2307
- // Each attempt is claimed by an up-to-date writer and lost.
2308
- let mut claimer = WorkloadStore::from_db(db.clone()).unwrap();
2309
- let claim = claimer
2310
- .claim(&id, "worker-a", &capabilities(), 100 * attempt, 10_000)
2311
- .unwrap();
2312
- let (_, _, recovered) = race_recovery(&db, claim.expires_at);
2313
- assert_eq!(recovered.concat(), vec![id.clone()], "attempt {attempt}");
2314
- let committed = committed_workload(&db, &id);
2315
- assert_eq!(recoveries(&committed), attempt as usize);
2316
- assert_eq!(committed.attempt_count, attempt as u32);
2317
- }
2318
- let committed = committed_workload(&db, &id);
2319
- assert_eq!(committed.state, WorkloadState::DeadLetter);
2320
- assert_eq!(
2321
- committed.execution_history.last().unwrap().evidence["decision"],
2322
- "attempt_budget_exhausted"
2323
- );
2324
- assert!(committed
2325
- .attempts
2326
- .iter()
2327
- .all(|attempt| attempt.status == AttemptStatus::WorkerLost));
2328
- let refused = WorkloadStore::from_db(db.clone())
2329
- .unwrap()
2330
- .claim(&id, "worker-b", &capabilities(), 10_000, 10_000)
2331
- .unwrap_err();
2332
- assert_eq!(refused.code, "WORKLOAD_NOT_READY");
2333
- assert_eq!(committed_workload(&db, &id).attempt_count, 3);
2334
- drop((store, db));
2335
- let _ = fs::remove_file(path);
2336
- }
2337
-
2338
- #[test]
2339
- fn a_policy_that_does_not_retry_worker_failure_dead_letters_a_lost_attempt() {
2340
- let (path, mut store) = store("no-worker-failure-retry");
2341
- let mut request = create();
2342
- request.retry_policy.retry_on = ["timeout".into()].into();
2343
- let id = store.create(request).unwrap().workload_id;
2344
- let claim = store.claim(&id, "worker-a", &capabilities(), 100, 10_000).unwrap();
2345
- store.recover_expired(claim.expires_at).unwrap();
2346
- let workload = store.get(&id).unwrap();
2347
- assert_eq!(workload.state, WorkloadState::DeadLetter);
2348
- assert_eq!(
2349
- workload.execution_history.last().unwrap().evidence["decision"],
2350
- "worker_failure_not_retryable"
2351
- );
2352
- let _ = fs::remove_file(path);
2353
- }
2354
-
2355
- #[test]
2356
- fn claiming_exhausted_ready_work_dead_letters_it_instead() {
2357
- let (path, mut store) = store("exhausted-claim");
2358
- let id = store.create(create()).unwrap().workload_id;
2359
- // What unbounded recovery used to leave behind: Ready, budget spent.
2360
- let mut legacy = store.get(&id).unwrap().clone();
2361
- legacy.attempt_count = 7;
2362
- store.workloads.insert(id.clone(), legacy);
2363
- store.persist_workload(&id, "fixture.legacy").unwrap();
2364
- let refused = store.claim(&id, "worker-a", &capabilities(), 100, 10_000).unwrap_err();
2365
- assert_eq!(refused.code, "WORKLOAD_ATTEMPTS_EXHAUSTED");
2366
- let workload = store.get(&id).unwrap();
2367
- assert_eq!(workload.state, WorkloadState::DeadLetter);
2368
- assert_eq!(
2369
- workload.execution_history.last().unwrap().evidence["reason"],
2370
- "attempt_budget_exhausted"
2371
- );
2372
- // Durable: a restart sees the same outcome.
2373
- let reopened = WorkloadStore::from_db(store.db.clone()).unwrap();
2374
- assert_eq!(reopened.get(&id).unwrap().state, WorkloadState::DeadLetter);
2375
- let _ = fs::remove_file(path);
2376
- }
2377
-
2378
- #[test]
2379
- fn operator_retry_grants_one_attempt_is_recorded_and_is_bounded() {
2380
- let (path, mut store) = store("operator-retry");
2381
- let id = store.create(create()).unwrap().workload_id;
2382
- // Claim and lose leases until the workload stops being claimable.
2383
- fn exhaust(store: &mut WorkloadStore, id: &str, now: &mut i64) {
2384
- loop {
2385
- *now += 100;
2386
- match store.claim(id, "worker-a", &capabilities(), *now, 10_000) {
2387
- Ok(claim) => {
2388
- store.recover_expired(claim.expires_at).unwrap();
2389
- }
2390
- Err(error) => {
2391
- assert!(
2392
- matches!(
2393
- error.code.as_str(),
2394
- "WORKLOAD_NOT_READY" | "WORKLOAD_ATTEMPTS_EXHAUSTED"
2395
- ),
2396
- "{error:?}"
2397
- );
2398
- return;
2399
- }
2400
- }
2401
- }
2402
- }
2403
- let mut now = 100;
2404
- exhaust(&mut store, &id, &mut now);
2405
- assert_eq!(store.get(&id).unwrap().attempt_count, 3);
2406
- for retry in 1..=3u32 {
2407
- now += 100;
2408
- store
2409
- .retry_with_reason(&id, "operator-1", now, Some("dependency restored"))
2410
- .unwrap();
2411
- let event = store.get(&id).unwrap().execution_history.last().unwrap().clone();
2412
- assert_eq!(event.actor, "operator-1");
2413
- assert_eq!(event.evidence["retry"], "operator");
2414
- assert_eq!(event.evidence["reason"], "dependency restored");
2415
- assert_eq!(event.evidence["operator_retry"], retry.to_string());
2416
- exhaust(&mut store, &id, &mut now);
2417
- assert_eq!(
2418
- store.get(&id).unwrap().attempt_count,
2419
- 3 + retry,
2420
- "each operator retry grants exactly one attempt"
2421
- );
2422
- assert_eq!(store.get(&id).unwrap().state, WorkloadState::DeadLetter);
2423
- }
2424
- assert_eq!(
2425
- store.retry(&id, "operator-1", now + 100).unwrap_err().code,
2426
- "WORKLOAD_RETRY_EXHAUSTED"
2427
- );
2428
- assert_eq!(store.get(&id).unwrap().state, WorkloadState::DeadLetter);
2429
- let _ = fs::remove_file(path);
2430
- }
2431
-
2432
- fn store(name: &str) -> (PathBuf, WorkloadStore) {
2433
- let path =
2434
- std::env::temp_dir().join(format!("workload-{name}-{}.json", std::process::id()));
2435
- let store = WorkloadStore::load(&path).unwrap();
2436
- (path, store)
2437
- }
2438
- #[test]
2439
- fn legacy_migration_is_idempotent_conflict_detecting_and_source_independent() {
2440
- let (source_db, mut source) = store("legacy-source-builder");
2441
- let workload = source.create(create()).unwrap();
2442
- let legacy = LegacyWorkloadStore {
2443
- workloads: source.workloads.clone(),
2444
- // What a pre-requester-scope release wrote.
2445
- idempotency: idempotency_index(&source.workloads, legacy_idempotency_scope),
2446
- result_ids: source.result_ids.clone(),
2447
- event_sequence: derived_event_sequence(&source.workloads),
2448
- };
2449
- let legacy_path = source_db.with_extension("legacy-workloads.json");
2450
- fs::write(&legacy_path, serde_json::to_vec(&legacy).unwrap()).unwrap();
2451
- let canonical_path = source_db.with_extension("canonical.log");
2452
- let db = FeltDb::open(&canonical_path).unwrap();
2453
- let migrated = WorkloadStore::migrate_legacy_file(db.clone(), &legacy_path).unwrap();
2454
- assert_eq!(
2455
- migrated.get(&workload.workload_id).unwrap().input_hash,
2456
- workload.input_hash
2457
- );
2458
- WorkloadStore::migrate_legacy_file(db.clone(), &legacy_path).unwrap();
2459
- fs::remove_file(&legacy_path).unwrap();
2460
- let reopened = WorkloadStore::from_db(FeltDb::open(&canonical_path).unwrap()).unwrap();
2461
- assert_eq!(
2462
- reopened.get(&workload.workload_id).unwrap().workload_id,
2463
- workload.workload_id
2464
- );
2465
-
2466
- let mut conflicting = legacy;
2467
- conflicting
2468
- .workloads
2469
- .get_mut(&workload.workload_id)
2470
- .unwrap()
2471
- .priority += 1;
2472
- fs::write(&legacy_path, serde_json::to_vec(&conflicting).unwrap()).unwrap();
2473
- assert_eq!(
2474
- WorkloadStore::migrate_legacy_file(db, &legacy_path)
2475
- .err()
2476
- .unwrap()
2477
- .code,
2478
- "WORKLOAD_MIGRATION_CONFLICT"
2479
- );
2480
- let _ = fs::remove_file(source_db);
2481
- let _ = fs::remove_file(canonical_path);
2482
- let _ = fs::remove_file(legacy_path);
2483
- }
2484
-
2485
- #[test]
2486
- fn canonical_history_reconstructs_work_identity_provenance() {
2487
- let (path, mut store) = store("provenance");
2488
- let workload = store.create(create()).unwrap();
2489
- let transaction = store
2490
- .db
2491
- .committed_transactions_after(0, usize::MAX)
2492
- .unwrap()
2493
- .into_iter()
2494
- .last()
2495
- .unwrap();
2496
- let audit = transaction.audit.unwrap();
2497
- assert_eq!(audit["work_id"], workload.workload_id);
2498
- assert_eq!(audit["session_id"], workload.session_id.as_deref().unwrap());
2499
- assert_eq!(audit["agent_id"], workload.requested_by.id());
2500
- assert_eq!(
2501
- audit["delegator_id"],
2502
- workload.authorization_snapshot.grant.issuer.id()
2503
- );
2504
- assert!(transaction.commit_revision > transaction.base_revision);
2505
- assert_eq!(transaction.work_id.as_deref(), Some(workload.workload_id.as_str()));
2506
- assert_eq!(transaction.agent_id.as_deref(), Some(workload.requested_by.id()));
2507
- assert_eq!(transaction.session_id, workload.session_id);
2508
- assert_eq!(
2509
- transaction.delegator_id.as_deref(),
2510
- Some(workload.authorization_snapshot.grant.issuer.id())
2511
- );
2512
- assert_eq!(transaction.operation_ids.len(), 1);
2513
- assert_ne!(transaction.operation_ids[0], transaction.transaction_id);
2514
- drop(store);
2515
- let reopened = WorkloadStore::load(&path).unwrap();
2516
- assert_eq!(
2517
- reopened.get(&workload.workload_id).unwrap().workload_id,
2518
- workload.workload_id
2519
- );
2520
- let _ = fs::remove_file(path);
2521
- }
2522
-
2523
- #[test]
2524
- fn competing_writers_use_canonical_document_cas_before_and_after_restart() {
2525
- let (path, mut seed) = store("canonical-cas");
2526
- let workload = seed.create(create()).unwrap();
2527
- let id = workload.workload_id.clone();
2528
- let db = seed.db.clone();
2529
- let caps = ["email:send".into()].into();
2530
-
2531
- let mut winner = WorkloadStore::from_db(db.clone()).unwrap();
2532
- // The loser's claim races the winner's inside the window between its
2533
- // refresh and its commit, which only the storage guard closes.
2534
- let mut loser = WorkloadStore::from_db(db.clone())
2535
- .unwrap()
2536
- .without_refresh();
2537
- winner.claim(&id, "worker-a", &caps, 11, 1_000).unwrap();
2538
- let conflict = loser.claim(&id, "worker-b", &caps, 11, 1_000).unwrap_err();
2539
- assert_eq!(conflict.code, "WORKLOAD_CONFLICT");
2540
- // The loser committed nothing and now projects the winner's claim.
2541
- let projected = loser.get(&id).unwrap();
2542
- assert_eq!(projected.state, WorkloadState::Claimed);
2543
- assert_eq!(projected.current_claim.as_ref().unwrap().worker_id, "worker-a");
2544
-
2545
- let row = db
2546
- .list_collection(DURABLE_WORK_COLLECTION)
2547
- .unwrap()
2548
- .into_iter()
2549
- .find(|row| row.key.ends_with(&id))
2550
- .unwrap();
2551
- assert_eq!(row.value["__version"], 2);
2552
-
2553
- drop((seed, winner, loser, db));
2554
- let restarted_db = FeltDb::open(&path).unwrap();
2555
- let mut recovery = WorkloadStore::from_db(restarted_db.clone()).unwrap();
2556
- recovery.recover_expired(13).unwrap();
2557
- let mut restarted_winner = WorkloadStore::from_db(restarted_db.clone()).unwrap();
2558
- let mut restarted_loser = WorkloadStore::from_db(restarted_db.clone())
2559
- .unwrap()
2560
- .without_refresh();
2561
- restarted_winner
2562
- .claim(&id, "worker-c", &caps, 13, 10_000)
2563
- .unwrap();
2564
- let conflict = restarted_loser
2565
- .claim(&id, "worker-d", &caps, 13, 10_000)
2566
- .unwrap_err();
2567
- assert_eq!(conflict.code, "WORKLOAD_CONFLICT");
2568
- let projected = restarted_loser.get(&id).unwrap();
2569
- assert_eq!(projected.state, WorkloadState::Claimed);
2570
- assert_eq!(projected.current_claim.as_ref().unwrap().worker_id, "worker-c");
2571
- drop((recovery, restarted_winner, restarted_loser, restarted_db));
2572
- let _ = fs::remove_file(path);
2573
- }
2574
-
2575
- #[test]
2576
- fn competing_admissions_share_one_canonical_idempotency_identity() {
2577
- let path = std::env::temp_dir().join(format!(
2578
- "workload-idempotency-cas-{}.log",
2579
- std::process::id()
2580
- ));
2581
- let db = FeltDb::open(&path).unwrap();
2582
- let mut first = WorkloadStore::from_db(db.clone()).unwrap();
2583
- let mut second = WorkloadStore::from_db(db.clone()).unwrap();
2584
- let accepted = first.create(create()).unwrap();
2585
- // A second writer of the same store loses the require-absent CAS and
2586
- // converges on what was committed: a different request is refused as a
2587
- // reused key, never answered with the original.
2588
- let mut competing = create();
2589
- competing.input = serde_json::json!({"id": 2});
2590
- let conflict = second.create(competing).unwrap_err();
2591
- assert_eq!(conflict.code, "IDEMPOTENCY_KEY_REUSED");
2592
- assert_eq!(
2593
- second.workloads.keys().collect::<Vec<_>>(),
2594
- vec![&accepted.workload_id],
2595
- "only the committed workload is projected"
2596
- );
2597
- // The same request from a third, equally stale writer converges on it.
2598
- let mut third = WorkloadStore::from_db(db.clone()).unwrap();
2599
- third.workloads.clear();
2600
- third.idempotency.clear();
2601
- third.document_versions.clear();
2602
- let (converged, created) = third.admit(create()).unwrap();
2603
- assert!(!created);
2604
- assert_eq!(converged.workload_id, accepted.workload_id);
2605
-
2606
- drop((first, second, third, db));
2607
- let restarted = WorkloadStore::from_db(FeltDb::open(&path).unwrap()).unwrap();
2608
- assert_eq!(restarted.workloads.len(), 1);
2609
- assert_eq!(
2610
- restarted.idempotency[&idempotency_scope("t", "app", &accepted.requested_by, "one")],
2611
- accepted.workload_id
2612
- );
2613
- let _ = fs::remove_file(path);
2614
- }
2615
-
2616
- #[test]
2617
- fn idempotency_scopes_cannot_be_made_to_coincide() {
2618
- let (path, mut store) = store("scope-separators");
2619
- let mut first = create_as("x:k1");
2620
- first.idempotency_key = "k2".into();
2621
- let mut second = create_as("x");
2622
- second.idempotency_key = "k1:k2".into();
2623
- let first = store.create(first).unwrap();
2624
- let second = store.create(second).unwrap();
2625
- assert_ne!(first.workload_id, second.workload_id);
2626
- let _ = fs::remove_file(path);
2627
- }
2628
-
2629
- #[test]
2630
- fn concurrent_admissions_by_competing_writers_converge_or_conflict() {
2631
- for round in 0..20 {
2632
- let (path, seed) = store(&format!("concurrent-admission-{round}"));
2633
- let db = seed.db.clone();
2634
- let admit_concurrently = |requests: [CreateWorkload; 2]| {
2635
- let barrier = std::sync::Barrier::new(2);
2636
- std::thread::scope(|scope| {
2637
- requests
2638
- .into_iter()
2639
- .map(|request| {
2640
- let mut writer = WorkloadStore::from_db(db.clone()).unwrap();
2641
- let barrier = &barrier;
2642
- scope.spawn(move || {
2643
- barrier.wait();
2644
- writer.admit(request)
2645
- })
2646
- })
2647
- .collect::<Vec<_>>()
2648
- .into_iter()
2649
- .map(|handle| handle.join().unwrap())
2650
- .collect::<Vec<_>>()
2651
- })
2652
- };
2653
- // The same request: both writers answer with one workload, and
2654
- // exactly one of them created it.
2655
- let same = admit_concurrently([create(), create()]);
2656
- let (first, first_created) = same[0].as_ref().unwrap().clone();
2657
- let (second, second_created) = same[1].as_ref().unwrap().clone();
2658
- assert_eq!(first.workload_id, second.workload_id);
2659
- assert!(first_created ^ second_created, "round {round}");
2660
-
2661
- // Different requests under one key: one is admitted, the other is
2662
- // refused as a reused key and never answered with the winner.
2663
- let mut left = create();
2664
- left.idempotency_key = "contested".into();
2665
- let mut right = left.clone();
2666
- right.input = serde_json::json!({"id": 2});
2667
- let contested = admit_concurrently([left, right]);
2668
- let admitted = contested.iter().filter(|outcome| outcome.is_ok()).count();
2669
- assert_eq!(admitted, 1, "round {round}");
2670
- assert!(contested.iter().any(|outcome| outcome
2671
- .as_ref()
2672
- .is_err_and(|error| error.code == "IDEMPOTENCY_KEY_REUSED")));
2673
- let (winner, _) = contested.iter().find_map(|o| o.as_ref().ok()).unwrap();
2674
- let stored = WorkloadStore::from_db(db.clone()).unwrap();
2675
- assert_eq!(stored.workloads.len(), 2);
2676
- assert_eq!(stored.get(&winner.workload_id).unwrap().input, winner.input);
2677
- drop((seed, stored, db));
2678
- let _ = fs::remove_file(path);
2679
- }
2680
- }
2681
-
2682
- #[test]
2683
- fn pre_versioned_canonical_rows_upgrade_through_the_standard_cas_contract() {
2684
- let (path, mut store) = store("pre-versioned-upgrade");
2685
- let workload = store.create(create()).unwrap();
2686
- let db = store.db.clone();
2687
- db.apply_atomic_transaction(
2688
- "fixture:pre-versioned-work",
2689
- None,
2690
- &[],
2691
- &[AtomicMutation {
2692
- capability: DURABLE_WORK_COLLECTION.into(),
2693
- key: work_key(&workload),
2694
- value: Some(serde_json::to_value(&workload).unwrap()),
2695
- }],
2696
- None,
2697
- )
2698
- .unwrap();
2699
-
2700
- let mut upgraded = WorkloadStore::from_db(db.clone()).unwrap();
2701
- upgraded
2702
- .claim(
2703
- &workload.workload_id,
2704
- "worker",
2705
- &["email:send".into()].into(),
2706
- 11,
2707
- 10_000,
2708
- )
2709
- .unwrap();
2710
- let row = db
2711
- .list_collection(DURABLE_WORK_COLLECTION)
2712
- .unwrap()
2713
- .into_iter()
2714
- .find(|row| row.key.ends_with(&workload.workload_id))
2715
- .unwrap();
2716
- assert_eq!(row.value["__version"], 2);
2717
- let _ = fs::remove_file(path);
2718
- }
2719
-
2720
- #[test]
2721
- fn derived_indexes_rebuild_exclusively_from_canonical_work() {
2722
- let (path, mut store) = store("derived-index-rebuild");
2723
- let workload = store.create(create()).unwrap();
2724
- let caps = ["email:send".into()].into();
2725
- let claim = store
2726
- .claim(&workload.workload_id, "worker", &caps, 11, 10_000)
2727
- .unwrap();
2728
- store
2729
- .start(
2730
- &workload.workload_id,
2731
- &claim.claim_id,
2732
- "worker",
2733
- claim.fencing_token,
2734
- 12,
2735
- )
2736
- .unwrap();
2737
- store
2738
- .complete(
2739
- &workload.workload_id,
2740
- &claim.claim_id,
2741
- "worker",
2742
- claim.fencing_token,
2743
- "result-rebuilt",
2744
- serde_json::json!({"ok": true}),
2745
- vec![],
2746
- 1,
2747
- 13,
2748
- )
2749
- .unwrap();
2750
-
2751
- store.idempotency.clear();
2752
- store.result_ids.clear();
2753
- store.document_versions.clear();
2754
- let rebuilt = WorkloadStore::from_db(store.db.clone()).unwrap();
2755
- assert_eq!(
2756
- rebuilt
2757
- .idempotency
2758
- .get(&idempotency_scope("t", "app", &workload.requested_by, "one")),
2759
- Some(&workload.workload_id)
2760
- );
2761
- assert_eq!(
2762
- rebuilt.result_ids.get("result-rebuilt"),
2763
- Some(&workload.workload_id)
2764
- );
2765
- assert!(rebuilt.document_versions[&workload.workload_id] > 1);
2766
- assert_eq!(
2767
- rebuilt
2768
- .get(&workload.workload_id)
2769
- .unwrap()
2770
- .execution_history
2771
- .iter()
2772
- .map(|event| event.sequence)
2773
- .collect::<Vec<_>>(),
2774
- vec![1, 2, 3, 4]
2775
- );
2776
- let duplicate = WorkloadStore::from_db(store.db.clone())
2777
- .unwrap()
2778
- .create(create())
2779
- .unwrap();
2780
- assert_eq!(duplicate.workload_id, workload.workload_id);
2781
- let _ = fs::remove_file(path);
2782
- }
2783
- #[test]
2784
- fn idempotent_create_and_legal_lifecycle() {
2785
- let (path, mut store) = store("lifecycle");
2786
- let one = store.create(create()).unwrap();
2787
- let two = store.create(create()).unwrap();
2788
- assert_eq!(one.workload_id, two.workload_id);
2789
- let caps = ["email:send".into()].into();
2790
- let claim = store
2791
- .claim(&one.workload_id, "worker", &caps, 11, 10_000)
2792
- .unwrap();
2793
- store
2794
- .start(
2795
- &one.workload_id,
2796
- &claim.claim_id,
2797
- "worker",
2798
- claim.fencing_token,
2799
- 12,
2800
- )
2801
- .unwrap();
2802
- let result = store
2803
- .complete(
2804
- &one.workload_id,
2805
- &claim.claim_id,
2806
- "worker",
2807
- claim.fencing_token,
2808
- "result-1",
2809
- serde_json::json!({"ok":true}),
2810
- vec![],
2811
- 1,
2812
- 13,
2813
- )
2814
- .unwrap();
2815
- assert_eq!(result.result_id, "result-1");
2816
- assert!(store
2817
- .start(
2818
- &one.workload_id,
2819
- &claim.claim_id,
2820
- "worker",
2821
- claim.fencing_token,
2822
- 14
2823
- )
2824
- .is_err());
2825
- let recovered = WorkloadStore::load(&path).unwrap();
2826
- assert_eq!(
2827
- recovered.get(&one.workload_id).unwrap().state,
2828
- WorkloadState::Succeeded
2829
- );
2830
- let _ = fs::remove_file(path);
2831
- }
2832
- #[test]
2833
- fn stale_fence_and_lease_recovery() {
2834
- let (path, mut store) = store("fence");
2835
- let w = store.create(create()).unwrap();
2836
- let caps = ["email:send".into()].into();
2837
- let old = store.claim(&w.workload_id, "a", &caps, 11, 1000).unwrap();
2838
- store
2839
- .start(&w.workload_id, &old.claim_id, "a", old.fencing_token, 11)
2840
- .unwrap();
2841
- store.recover_expired(13).unwrap();
2842
- let new = store.claim(&w.workload_id, "b", &caps, 13, 10_000).unwrap();
2843
- assert!(new.fencing_token > old.fencing_token);
2844
- assert_eq!(
2845
- store
2846
- .complete(
2847
- &w.workload_id,
2848
- &old.claim_id,
2849
- "a",
2850
- old.fencing_token,
2851
- "stale",
2852
- Value::Null,
2853
- vec![],
2854
- 1,
2855
- 14
2856
- )
2857
- .unwrap_err()
2858
- .code,
2859
- "WORKLOAD_STALE_FENCE"
2860
- );
2861
- let _ = fs::remove_file(path);
2862
- }
2863
- #[test]
2864
- fn retry_dead_letter_checkpoint_and_secret_safety() {
2865
- let (path, mut store) = store("retry");
2866
- let mut input = create();
2867
- input.retry_policy.max_attempts = 1;
2868
- let w = store.create(input).unwrap();
2869
- let caps = ["email:send".into()].into();
2870
- let claim = store.claim(&w.workload_id, "a", &caps, 11, 10_000).unwrap();
2871
- store
2872
- .start(
2873
- &w.workload_id,
2874
- &claim.claim_id,
2875
- "a",
2876
- claim.fencing_token,
2877
- 12,
2878
- )
2879
- .unwrap();
2880
- assert!(store
2881
- .checkpoint(
2882
- &w.workload_id,
2883
- &claim.claim_id,
2884
- "a",
2885
- claim.fencing_token,
2886
- serde_json::json!({"password":"bad"}),
2887
- 13
2888
- )
2889
- .is_err());
2890
- assert_eq!(
2891
- store
2892
- .fail(
2893
- &w.workload_id,
2894
- &claim.claim_id,
2895
- "a",
2896
- claim.fencing_token,
2897
- "worker_failure",
2898
- "crashed",
2899
- 13
2900
- )
2901
- .unwrap(),
2902
- WorkloadState::DeadLetter
2903
- );
2904
- let _ = fs::remove_file(path);
2905
- }
2906
- #[test]
2907
- fn a_repeated_completion_replays_only_for_the_claim_and_output_that_produced_it() {
2908
- let (path, mut store) = store("complete-replay");
2909
- let w = store.create(create()).unwrap();
2910
- let id = w.workload_id.clone();
2911
- let caps = capabilities();
2912
- // Worker a loses its lease; worker b completes the work.
2913
- let lost = store.claim(&id, "a", &caps, 11, 1000).unwrap();
2914
- store
2915
- .start(&id, &lost.claim_id, "a", lost.fencing_token, 11)
2916
- .unwrap();
2917
- store.recover_expired(13).unwrap();
2918
- let won = store.claim(&id, "b", &caps, 13, 10_000).unwrap();
2919
- store
2920
- .start(&id, &won.claim_id, "b", won.fencing_token, 13)
2921
- .unwrap();
2922
- let output = serde_json::json!({"sent": 1});
2923
- let first = store
2924
- .complete(
2925
- &id,
2926
- &won.claim_id,
2927
- "b",
2928
- won.fencing_token,
2929
- "r1",
2930
- output.clone(),
2931
- vec![],
2932
- 1,
2933
- 14,
2934
- )
2935
- .unwrap();
2936
- // The reporting attempt retrying after a lost response gets the same result.
2937
- let replay = store
2938
- .complete(
2939
- &id,
2940
- &won.claim_id,
2941
- "b",
2942
- won.fencing_token,
2943
- "r1",
2944
- output.clone(),
2945
- vec![],
2946
- 1,
2947
- 15,
2948
- )
2949
- .unwrap();
2950
- assert_eq!(replay.output_hash, first.output_hash);
2951
- assert_eq!(replay.attempt_id, first.attempt_id);
2952
- // The same result id with a different output is a conflict, not a replay.
2953
- assert_eq!(
2954
- store
2955
- .complete(
2956
- &id,
2957
- &won.claim_id,
2958
- "b",
2959
- won.fencing_token,
2960
- "r1",
2961
- serde_json::json!({"sent": 2}),
2962
- vec![],
2963
- 1,
2964
- 15
2965
- )
2966
- .unwrap_err()
2967
- .code,
2968
- "WORKLOAD_RESULT_CONFLICT"
2969
- );
2970
- // A stale worker naming the winner's result id learns nothing landed for it.
2971
- assert_eq!(
2972
- store
2973
- .complete(
2974
- &id,
2975
- &lost.claim_id,
2976
- "a",
2977
- lost.fencing_token,
2978
- "r1",
2979
- output.clone(),
2980
- vec![],
2981
- 1,
2982
- 15
2983
- )
2984
- .unwrap_err()
2985
- .code,
2986
- "WORKLOAD_RESULT_CONFLICT"
2987
- );
2988
- let committed = store.get(&id).unwrap();
2989
- assert_eq!(committed.state, WorkloadState::Succeeded);
2990
- assert_eq!(committed.result.as_ref().unwrap().output, output);
2991
- assert_eq!(committed.result.as_ref().unwrap().produced_by, "b");
2992
- let _ = fs::remove_file(path);
2993
- }
2994
- #[test]
2995
- fn a_resumed_attempt_reads_the_input_and_the_lost_attempts_checkpoints_through_its_fence() {
2996
- let (path, mut store) = store("resume-context");
2997
- let w = store.create(create()).unwrap();
2998
- let id = w.workload_id.clone();
2999
- let caps = capabilities();
3000
- let lost = store.claim(&id, "a", &caps, 11, 1000).unwrap();
3001
- store
3002
- .start(&id, &lost.claim_id, "a", lost.fencing_token, 11)
3003
- .unwrap();
3004
- store
3005
- .checkpoint(
3006
- &id,
3007
- &lost.claim_id,
3008
- "a",
3009
- lost.fencing_token,
3010
- serde_json::json!({"step": "reserve", "status": "done"}),
3011
- 11,
3012
- )
3013
- .unwrap();
3014
- store.recover_expired(13).unwrap();
3015
- let next = store.claim(&id, "b", &caps, 13, 10_000).unwrap();
3016
- let context = store
3017
- .execution_context(&id, &next.claim_id, "b", next.fencing_token, 13, vec![])
3018
- .unwrap();
3019
- assert_eq!(context.input, serde_json::json!({"id": 1}));
3020
- assert_eq!(context.checkpoints.len(), 1);
3021
- assert_eq!(context.checkpoints[0].state["step"], "reserve");
3022
- assert_ne!(
3023
- context.checkpoints[0].attempt_id, context.attempt_id,
3024
- "written by the lost attempt"
3025
- );
3026
- // The lost worker's fence reads nothing.
3027
- assert_eq!(
3028
- store
3029
- .execution_context(&id, &lost.claim_id, "a", lost.fencing_token, 13, vec![])
3030
- .unwrap_err()
3031
- .code,
3032
- "WORKLOAD_STALE_FENCE"
3033
- );
3034
- let _ = fs::remove_file(path);
3035
- }
3036
- #[test]
3037
- fn cancellation_race_and_duplicate_completion_are_deterministic() {
3038
- let (path, mut store) = store("cancel");
3039
- let w = store.create(create()).unwrap();
3040
- let caps = ["email:send".into()].into();
3041
- let claim = store.claim(&w.workload_id, "a", &caps, 11, 10_000).unwrap();
3042
- store
3043
- .start(
3044
- &w.workload_id,
3045
- &claim.claim_id,
3046
- "a",
3047
- claim.fencing_token,
3048
- 12,
3049
- )
3050
- .unwrap();
3051
- assert_eq!(
3052
- store.cancel(&w.workload_id, "user", 13).unwrap(),
3053
- WorkloadState::CancellationRequested
3054
- );
3055
- assert_eq!(
3056
- store
3057
- .complete(
3058
- &w.workload_id,
3059
- &claim.claim_id,
3060
- "a",
3061
- claim.fencing_token,
3062
- "r",
3063
- Value::Null,
3064
- vec![],
3065
- 1,
3066
- 14
3067
- )
3068
- .unwrap_err()
3069
- .code,
3070
- "WORKLOAD_TRANSITION_DENIED"
3071
- );
3072
- store
3073
- .confirm_cancel(
3074
- &w.workload_id,
3075
- &claim.claim_id,
3076
- "a",
3077
- claim.fencing_token,
3078
- 14,
3079
- )
3080
- .unwrap();
3081
- assert_eq!(
3082
- store.get(&w.workload_id).unwrap().state,
3083
- WorkloadState::Cancelled
3084
- );
3085
- let _ = fs::remove_file(path);
3086
- }
3087
- #[test]
3088
- fn scheduler_respects_dependencies_and_tenant_application_isolation() {
3089
- let (path, mut store) = store("dependency");
3090
- let parent = store.create(create()).unwrap();
3091
- let mut child = create();
3092
- child.idempotency_key = "child".into();
3093
- child.dependencies = vec![WorkloadDependency {
3094
- workload_id: parent.workload_id.clone(),
3095
- required_state: WorkloadState::Succeeded,
3096
- }];
3097
- let child = store.create(child).unwrap();
3098
- assert_eq!(store.list("other", "app").len(), 0);
3099
- let caps = ["email:send".into()].into();
3100
- let mut scheduler = LocalDurableScheduler::default();
3101
- let first = scheduler
3102
- .claim(&mut store, "t", "app", "worker", &caps, 11, 10_000)
3103
- .unwrap()
3104
- .unwrap();
3105
- assert_eq!(first.workload_id, parent.workload_id);
3106
- assert_ne!(first.workload_id, child.workload_id);
3107
- let _ = fs::remove_file(path);
3108
- }
3109
-
3110
- // ------------------------------------------------------------------
3111
- // Temporal work eligibility.
3112
- //
3113
- // The invariant under test: a workload may exist durably before it is
3114
- // executable. Every test below drives the real store — creating, claiming,
3115
- // reloading from disk — rather than asserting that a field or a function
3116
- // name exists.
3117
- // ------------------------------------------------------------------
3118
-
3119
- /// A store on a path unique to this test, cleared first so a recycled
3120
- /// process id cannot hand a test a previous run's file.
3121
- fn temporal_store(name: &str) -> (PathBuf, WorkloadStore) {
3122
- let path = std::env::temp_dir().join(format!(
3123
- "workload-temporal-{name}-{}-{}.json",
3124
- std::process::id(),
3125
- rand::random::<u64>()
3126
- ));
3127
- let _ = fs::remove_file(&path);
3128
- (path.clone(), WorkloadStore::load(&path).unwrap())
3129
- }
3130
-
3131
- fn scheduled(not_before: i64) -> CreateWorkload {
3132
- let mut input = create();
3133
- input.not_before = Some(not_before);
3134
- input
3135
- }
3136
-
3137
- fn caps() -> BTreeSet<String> {
3138
- ["email:send".into()].into()
3139
- }
3140
-
3141
- #[test]
3142
- fn scheduled_work_is_not_claimable_before_not_before() {
3143
- let (path, mut store) = temporal_store("before");
3144
- let workload = store.create(scheduled(2_000)).unwrap();
3145
-
3146
- // Durable and Ready immediately: existing before it is executable is
3147
- // the whole point.
3148
- assert_eq!(workload.state, WorkloadState::Ready);
3149
- assert_eq!(workload.next_attempt_at, Some(2_000));
3150
-
3151
- let refused = store
3152
- .claim(&workload.workload_id, "worker", &caps(), 1_999, 10_000)
3153
- .unwrap_err();
3154
- assert_eq!(refused.code, "WORKLOAD_NOT_YET_ELIGIBLE");
3155
- assert!(
3156
- refused.message.contains("2000"),
3157
- "the refusal names the instant the worker should return at: {}",
3158
- refused.message
3159
- );
3160
-
3161
- // Refusing a claim is not a state change. Nothing executed.
3162
- let after = store.get(&workload.workload_id).unwrap();
3163
- assert_eq!(after.state, WorkloadState::Ready);
3164
- assert_eq!(after.attempt_count, 0);
3165
- assert!(after.attempts.is_empty());
3166
- assert!(after.current_claim.is_none());
3167
- let _ = fs::remove_file(path);
3168
- }
3169
-
3170
- #[test]
3171
- fn scheduled_work_becomes_claimable_after_not_before() {
3172
- let (path, mut store) = temporal_store("after");
3173
- let workload = store.create(scheduled(2_000)).unwrap();
3174
-
3175
- // The boundary belongs to the eligible side: at exactly `not_before`
3176
- // the workload is claimable, so a worker polling on the second does not
3177
- // have to wait a further tick.
3178
- let claim = store
3179
- .claim(&workload.workload_id, "worker", &caps(), 2_000, 10_000)
3180
- .expect("eligible at the instant itself");
3181
- assert_eq!(claim.worker_id, "worker");
3182
- assert_eq!(
3183
- store.get(&workload.workload_id).unwrap().state,
3184
- WorkloadState::Claimed
3185
- );
3186
- let _ = fs::remove_file(path);
3187
- }
3188
-
3189
- #[test]
3190
- fn scheduled_work_survives_restart_before_eligibility() {
3191
- let (path, mut store) = temporal_store("restart-before");
3192
- let id = store.create(scheduled(2_000)).unwrap().workload_id;
3193
- drop(store);
3194
-
3195
- // Restart the authority. The workload is still here, still Ready, still
3196
- // carrying its eligibility, and still refused.
3197
- let mut reopened = WorkloadStore::load(&path).unwrap();
3198
- let recovered = reopened.get(&id).unwrap();
3199
- assert_eq!(recovered.state, WorkloadState::Ready);
3200
- assert_eq!(recovered.next_attempt_at, Some(2_000));
3201
-
3202
- assert_eq!(
3203
- reopened
3204
- .claim(&id, "worker", &caps(), 1_999, 10_000)
3205
- .unwrap_err()
3206
- .code,
3207
- "WORKLOAD_NOT_YET_ELIGIBLE"
3208
- );
3209
- let _ = fs::remove_file(path);
3210
- }
3211
-
3212
- #[test]
3213
- fn scheduled_work_executes_after_restart() {
3214
- let (path, mut store) = temporal_store("restart-after");
3215
- let id = store.create(scheduled(2_000)).unwrap().workload_id;
3216
- drop(store);
3217
-
3218
- let mut reopened = WorkloadStore::load(&path).unwrap();
3219
- let claim = reopened
3220
- .claim(&id, "worker", &caps(), 2_001, 10_000)
3221
- .expect("claimable after restart once eligible");
3222
- reopened
3223
- .start(&id, &claim.claim_id, "worker", claim.fencing_token, 2_002)
3224
- .unwrap();
3225
- reopened
3226
- .complete(
3227
- &id,
3228
- &claim.claim_id,
3229
- "worker",
3230
- claim.fencing_token,
3231
- "result-scheduled",
3232
- serde_json::json!({"ok": true}),
3233
- vec![],
3234
- 1,
3235
- 2_003,
3236
- )
3237
- .unwrap();
3238
-
3239
- // And the outcome is durable across a second restart.
3240
- let again = WorkloadStore::load(&path).unwrap();
3241
- let finished = again.get(&id).unwrap();
3242
- assert_eq!(finished.state, WorkloadState::Succeeded);
3243
- assert_eq!(
3244
- finished.result.as_ref().map(|r| r.result_id.as_str()),
3245
- Some("result-scheduled")
3246
- );
3247
- let _ = fs::remove_file(path);
3248
- }
3249
-
3250
- #[test]
3251
- fn immediate_work_behavior_is_unchanged() {
3252
- let (path, mut store) = temporal_store("immediate");
3253
- // `create()` sets no `not_before`, which is what every caller did
3254
- // before this field existed.
3255
- let workload = store.create(create()).unwrap();
3256
- assert_eq!(workload.next_attempt_at, None);
3257
- assert!(workload.is_eligible_at(workload.created_at));
3258
-
3259
- // Claimable at its own creation instant, with no waiting.
3260
- store
3261
- .claim(
3262
- &workload.workload_id,
3263
- "worker",
3264
- &caps(),
3265
- workload.created_at,
3266
- 10_000,
3267
- )
3268
- .expect("work with no eligibility instant is claimable immediately");
3269
- let _ = fs::remove_file(path);
3270
- }
3271
-
3272
- #[test]
3273
- fn multiple_workers_claim_temporally_eligible_work_once() {
3274
- let (path, mut store) = temporal_store("contention");
3275
- let id = store.create(scheduled(2_000)).unwrap().workload_id;
3276
-
3277
- // Both workers are polling and both see eligibility open at the same
3278
- // instant. Exactly one wins.
3279
- let winner = store
3280
- .claim(&id, "worker-a", &caps(), 2_000, 10_000)
3281
- .unwrap();
3282
- let loser = store
3283
- .claim(&id, "worker-b", &caps(), 2_000, 10_000)
3284
- .unwrap_err();
3285
- assert_eq!(
3286
- loser.code, "WORKLOAD_NOT_READY",
3287
- "the second worker is refused on state, not on time: the workload is claimed"
3288
- );
3289
-
3290
- // And the loser cannot complete work it never owned.
3291
- assert_eq!(
3292
- store
3293
- .complete(
3294
- &id,
3295
- &winner.claim_id,
3296
- "worker-b",
3297
- winner.fencing_token,
3298
- "duplicate",
3299
- Value::Null,
3300
- vec![],
3301
- 1,
3302
- 2_001,
3303
- )
3304
- .unwrap_err()
3305
- .code,
3306
- "WORKLOAD_STALE_FENCE"
3307
- );
3308
- assert_eq!(store.get(&id).unwrap().attempt_count, 1);
3309
- let _ = fs::remove_file(path);
3310
- }
3311
-
3312
- #[test]
3313
- fn retry_sets_next_eligibility_without_duplicate_execution() {
3314
- let (path, mut store) = temporal_store("retry");
3315
- let id = store.create(scheduled(2_000)).unwrap().workload_id;
3316
-
3317
- let first = store
3318
- .claim(&id, "worker-a", &caps(), 2_000, 10_000)
3319
- .unwrap();
3320
- store
3321
- .start(&id, &first.claim_id, "worker-a", first.fencing_token, 2_001)
3322
- .unwrap();
3323
- let state = store
3324
- .fail(
3325
- &id,
3326
- &first.claim_id,
3327
- "worker-a",
3328
- first.fencing_token,
3329
- "timeout",
3330
- "upstream did not answer",
3331
- 2_002,
3332
- )
3333
- .unwrap();
3334
-
3335
- // Retry re-arms the same eligibility field creation used, so there is
3336
- // one temporal concept rather than two competing ones.
3337
- assert_eq!(state, WorkloadState::Ready);
3338
- let backing_off = store.get(&id).unwrap();
3339
- let next = backing_off.next_attempt_at.expect("retry set eligibility");
3340
- assert!(
3341
- next > 2_002,
3342
- "backoff pushed eligibility into the future: {next}"
3343
- );
3344
-
3345
- assert_eq!(
3346
- store
3347
- .claim(&id, "worker-b", &caps(), next - 1, 10_000)
3348
- .unwrap_err()
3349
- .code,
3350
- "WORKLOAD_NOT_YET_ELIGIBLE"
3351
- );
3352
-
3353
- let second = store.claim(&id, "worker-b", &caps(), next, 10_000).unwrap();
3354
- assert!(second.fencing_token > first.fencing_token);
3355
-
3356
- // The first attempt cannot land a result after the retry was handed on.
3357
- assert_eq!(
3358
- store
3359
- .complete(
3360
- &id,
3361
- &first.claim_id,
3362
- "worker-a",
3363
- first.fencing_token,
3364
- "stale",
3365
- Value::Null,
3366
- vec![],
3367
- 1,
3368
- next,
3369
- )
3370
- .unwrap_err()
3371
- .code,
3372
- "WORKLOAD_STALE_FENCE"
3373
- );
3374
- assert_eq!(store.get(&id).unwrap().attempt_count, 2);
3375
- let _ = fs::remove_file(path);
3376
- }
3377
-
3378
- #[test]
3379
- fn cancellation_before_not_before_prevents_execution() {
3380
- let (path, mut store) = temporal_store("cancel");
3381
- let id = store.create(scheduled(2_000)).unwrap().workload_id;
3382
-
3383
- // Cancelling work that is durable but not yet executable is the whole
3384
- // reason to be able to see it before it runs.
3385
- assert_eq!(
3386
- store.cancel(&id, "operator", 1_500).unwrap(),
3387
- WorkloadState::Cancelled
3388
- );
3389
-
3390
- // Eligibility arriving does not resurrect it.
3391
- assert_eq!(
3392
- store
3393
- .claim(&id, "worker", &caps(), 2_001, 10_000)
3394
- .unwrap_err()
3395
- .code,
3396
- "WORKLOAD_NOT_READY"
3397
- );
3398
-
3399
- let reopened = WorkloadStore::load(&path).unwrap();
3400
- let cancelled = reopened.get(&id).unwrap();
3401
- assert_eq!(cancelled.state, WorkloadState::Cancelled);
3402
- assert!(cancelled.attempts.is_empty(), "nothing ever executed");
3403
- let _ = fs::remove_file(path);
3404
- }
3405
-
3406
- #[test]
3407
- fn a_worker_dying_after_a_claim_returns_the_work_and_it_executes_once() {
3408
- let (path, mut store) = temporal_store("worker-death");
3409
- let id = store.create(create()).unwrap().workload_id;
3410
-
3411
- // Worker A claims a short lease, starts, and dies without completing.
3412
- let dead = store.claim(&id, "worker-a", &caps(), 10, 1_000).unwrap();
3413
- store
3414
- .start(&id, &dead.claim_id, "worker-a", dead.fencing_token, 10)
3415
- .unwrap();
3416
- assert_eq!(
3417
- store.get(&id).unwrap().state,
3418
- WorkloadState::Running,
3419
- "the work was in flight when its worker died"
3420
- );
3421
-
3422
- // The lease expiring is the only thing that can notice.
3423
- let recovered = store.recover_expired(12).unwrap();
3424
- assert_eq!(recovered, vec![id.clone()]);
3425
- let back = store.get(&id).unwrap();
3426
- assert_eq!(back.state, WorkloadState::Ready, "returned to the pool");
3427
- assert_eq!(
3428
- back.attempts.last().map(|attempt| attempt.status),
3429
- Some(AttemptStatus::WorkerLost),
3430
- "and the dead attempt is recorded as lost rather than erased"
3431
- );
3432
-
3433
- // Worker B picks it up and finishes it.
3434
- let live = store.claim(&id, "worker-b", &caps(), 12, 10_000).unwrap();
3435
- assert!(live.fencing_token > dead.fencing_token);
3436
- store
3437
- .start(&id, &live.claim_id, "worker-b", live.fencing_token, 13)
3438
- .unwrap();
3439
- store
3440
- .complete(
3441
- &id,
3442
- &live.claim_id,
3443
- "worker-b",
3444
- live.fencing_token,
3445
- "result-recovered",
3446
- serde_json::json!({"ok": true}),
3447
- vec![],
3448
- 1,
3449
- 14,
3450
- )
3451
- .unwrap();
3452
-
3453
- // Exactly one execution succeeded, and the dead worker cannot add another.
3454
- let finished = store.get(&id).unwrap();
3455
- assert_eq!(finished.state, WorkloadState::Succeeded);
3456
- assert_eq!(
3457
- finished
3458
- .attempts
3459
- .iter()
3460
- .filter(|attempt| attempt.status == AttemptStatus::Succeeded)
3461
- .count(),
3462
- 1
3463
- );
3464
- // The dead worker cannot add a second result. Its claim was released by
3465
- // the recovery, and completion released the successor's, so it is
3466
- // refused for having no claim at all rather than a stale one.
3467
- assert_eq!(
3468
- store
3469
- .complete(
3470
- &id,
3471
- &dead.claim_id,
3472
- "worker-a",
3473
- dead.fencing_token,
3474
- "resurrected",
3475
- Value::Null,
3476
- vec![],
3477
- 1,
3478
- 15,
3479
- )
3480
- .unwrap_err()
3481
- .code,
3482
- "WORKLOAD_CLAIM_INVALID"
3483
- );
3484
- let _ = fs::remove_file(path);
3485
- }
3486
-
3487
- #[test]
3488
- fn a_recovered_workload_refuses_its_dead_worker_while_the_successor_still_holds_it() {
3489
- let (path, mut store) = temporal_store("worker-death-inflight");
3490
- let id = store.create(create()).unwrap().workload_id;
3491
-
3492
- let dead = store.claim(&id, "worker-a", &caps(), 10, 1_000).unwrap();
3493
- store
3494
- .start(&id, &dead.claim_id, "worker-a", dead.fencing_token, 10)
3495
- .unwrap();
3496
- store.recover_expired(12).unwrap();
3497
- let live = store.claim(&id, "worker-b", &caps(), 12, 10_000).unwrap();
3498
-
3499
- // While the successor is mid-flight, the resurrected worker is refused
3500
- // on the fence itself — the case a lease exists to decide.
3501
- assert_eq!(
3502
- store
3503
- .complete(
3504
- &id,
3505
- &dead.claim_id,
3506
- "worker-a",
3507
- dead.fencing_token,
3508
- "resurrected",
3509
- Value::Null,
3510
- vec![],
3511
- 1,
3512
- 13,
3513
- )
3514
- .unwrap_err()
3515
- .code,
3516
- "WORKLOAD_STALE_FENCE"
3517
- );
3518
- assert_eq!(store.get(&id).unwrap().state, WorkloadState::Claimed);
3519
- assert!(live.fencing_token > dead.fencing_token);
3520
- let _ = fs::remove_file(path);
3521
- }
3522
-
3523
- #[test]
3524
- fn the_scheduler_skips_ineligible_work_rather_than_failing_on_it() {
3525
- let (path, mut store) = temporal_store("scheduler");
3526
- let mut early = create();
3527
- early.idempotency_key = "early".into();
3528
- early.not_before = Some(9_000);
3529
- let early = store.create(early).unwrap();
3530
-
3531
- let mut ready = create();
3532
- ready.idempotency_key = "ready".into();
3533
- let ready = store.create(ready).unwrap();
3534
-
3535
- // The ineligible workload sorts first by id in some orderings and not
3536
- // others, so what matters is that the scheduler never returns it and
3537
- // never fails because of it.
3538
- let mut scheduler = LocalDurableScheduler::default();
3539
- let claim = scheduler
3540
- .claim(&mut store, "t", "app", "worker", &caps(), 1_000, 10_000)
3541
- .expect("an ineligible workload must not fail the whole pass")
3542
- .expect("the eligible workload is still handed out");
3543
- assert_eq!(claim.workload_id, ready.workload_id);
3544
- assert_eq!(
3545
- store.get(&early.workload_id).unwrap().state,
3546
- WorkloadState::Ready
3547
- );
3548
- let _ = fs::remove_file(path);
3549
- }
3550
-
3551
- // ---- PR 2: authoritative reads, canonical admission, failure reports, deadlines ----
3552
-
3553
- fn version_of(store: &WorkloadStore, id: &str) -> u64 {
3554
- let db = store.db_for_tests();
3555
- let w = store.get(id).unwrap();
3556
- db.live_row(DURABLE_WORK_COLLECTION, &work_key(w))
3557
- .and_then(|row| row.value.get("__version").and_then(Value::as_u64))
3558
- .unwrap()
3559
- }
3560
-
3561
- #[test]
3562
- fn another_writers_commits_are_visible_and_acted_on_without_restart() {
3563
- let (path, mut a) = store("two-writers");
3564
- let mut b = WorkloadStore::from_db(a.db_for_tests()).unwrap();
3565
- let id = a.create(create()).unwrap().workload_id;
3566
- // B never saw the admission, yet reads it from the store.
3567
- assert!(b.get(&id).is_err(), "the projection alone is stale");
3568
- assert_eq!(b.fresh(&id).unwrap().state, WorkloadState::Ready);
3569
- assert_eq!(b.fresh_list("t", "app").unwrap().len(), 1);
3570
- // A claims; B's claim decides on the committed claim, not its own view.
3571
- let claim = a
3572
- .claim(&id, "worker-a", &capabilities(), 11, 10_000)
3573
- .unwrap();
3574
- assert_eq!(
3575
- b.claim(&id, "worker-b", &capabilities(), 11, 10_000)
3576
- .unwrap_err()
3577
- .code,
3578
- "WORKLOAD_NOT_READY"
3579
- );
3580
- // A transition committed through A is what B fences against.
3581
- b.start(&id, &claim.claim_id, "worker-a", claim.fencing_token, 12)
3582
- .unwrap();
3583
- assert_eq!(a.fresh(&id).unwrap().state, WorkloadState::Running);
3584
- // B replays an admission A committed, by key, from the store.
3585
- let mut c = WorkloadStore::from_db(a.db_for_tests()).unwrap();
3586
- c.workloads.clear();
3587
- c.idempotency.clear();
3588
- c.document_versions.clear();
3589
- let (replayed, created) = c.admit(create()).unwrap();
3590
- assert!(!created);
3591
- assert_eq!(replayed.workload_id, id);
3592
- let _ = fs::remove_file(path);
3593
- }
3594
-
3595
- #[test]
3596
- fn a_replay_matches_only_an_equivalent_request_and_names_every_conflicting_field() {
3597
- let (path, mut store) = store("canonical-admission");
3598
- let mut original = create();
3599
- original.input = serde_json::json!({"a": 1, "b": 2});
3600
- original.dependencies = vec![
3601
- WorkloadDependency {
3602
- workload_id: "w1".into(),
3603
- required_state: WorkloadState::Succeeded,
3604
- },
3605
- WorkloadDependency {
3606
- workload_id: "w2".into(),
3607
- required_state: WorkloadState::Succeeded,
3608
- },
3609
- ];
3610
- let first = store.create(original.clone()).unwrap();
3611
-
3612
- // Equivalent after normalization: member order, dependency order, the
3613
- // defaults written out, and provenance that does not change what runs.
3614
- let mut equivalent = original.clone();
3615
- equivalent.input = serde_json::json!({"b": 2, "a": 1});
3616
- equivalent.dependencies.reverse();
3617
- equivalent.retry_policy = WorkloadRetryPolicy::default();
3618
- equivalent.timeouts = WorkloadTimeouts::default();
3619
- equivalent.priority = 0;
3620
- equivalent.trigger = "retried-by-client".into();
3621
- equivalent.session_id = Some("another-session".into());
3622
- equivalent.created_at = 99;
3623
- let (replayed, created) = store.admit(equivalent).unwrap();
3624
- assert!(!created);
3625
- assert_eq!(replayed.workload_id, first.workload_id);
3626
-
3627
- let mutations: Vec<(&str, Box<dyn Fn(&mut CreateWorkload)>)> = vec![
3628
- (
3629
- "definition_id",
3630
- Box::new(|r| r.definition_id = "other".into()),
3631
- ),
3632
- (
3633
- "definition_revision",
3634
- Box::new(|r| r.definition_revision = "r2".into()),
3635
- ),
3636
- (
3637
- "environment",
3638
- Box::new(|r| r.environment = "staging".into()),
3639
- ),
3640
- (
3641
- "input",
3642
- Box::new(|r| r.input = serde_json::json!({"a": 1, "b": 3})),
3643
- ),
3644
- (
3645
- "capability_snapshot",
3646
- Box::new(|r| {
3647
- r.capability_snapshot.insert("sms:send".into());
3648
- }),
3649
- ),
3650
- (
3651
- "connection_scopes",
3652
- Box::new(|r| {
3653
- r.connection_scopes.insert("crm".into());
3654
- }),
3655
- ),
3656
- ("priority", Box::new(|r| r.priority = 5)),
3657
- (
3658
- "retry_policy",
3659
- Box::new(|r| r.retry_policy.max_attempts = 9),
3660
- ),
3661
- (
3662
- "retry_policy",
3663
- Box::new(|r| {
3664
- r.retry_policy.retry_on.remove("worker_failure");
3665
- }),
3666
- ),
3667
- (
3668
- "timeouts",
3669
- Box::new(|r| r.timeouts.execution_ms = Some(60_000)),
3670
- ),
3671
- (
3672
- "dependencies",
3673
- Box::new(|r| r.dependencies.pop().map(|_| ()).unwrap()),
3674
- ),
3675
- ("not_before", Box::new(|r| r.not_before = Some(5_000))),
3676
- ];
3677
- for (field, mutate) in mutations {
3678
- let mut request = original.clone();
3679
- mutate(&mut request);
3680
- let error = store.admit(request).unwrap_err();
3681
- assert_eq!(error.code, "IDEMPOTENCY_KEY_REUSED", "{field}");
3682
- assert_eq!(
3683
- error.details.unwrap()["fields"],
3684
- serde_json::json!([field]),
3685
- "{field}"
3686
- );
3687
- }
3688
- // Several at once are all named.
3689
- let mut request = original.clone();
3690
- request.priority = 1;
3691
- request.environment = "staging".into();
3692
- let fields = store.admit(request).unwrap_err().details.unwrap()["fields"].clone();
3693
- assert_eq!(fields, serde_json::json!(["environment", "priority"]));
3694
-
3695
- // After a restart: the same answers from the committed record.
3696
- drop(store);
3697
- let mut reopened = WorkloadStore::load(&path).unwrap();
3698
- assert!(!reopened.admit(original.clone()).unwrap().1);
3699
- let mut changed = original.clone();
3700
- changed.retry_policy.initial_delay_ms = 5;
3701
- assert_eq!(
3702
- reopened.admit(changed).unwrap_err().code,
3703
- "IDEMPOTENCY_KEY_REUSED"
3704
- );
3705
- assert_eq!(reopened.workloads.len(), 1, "nothing was created");
3706
- let _ = fs::remove_file(path);
3707
- }
3708
-
3709
- #[test]
3710
- fn concurrent_conflicting_admissions_admit_one_and_refuse_the_other() {
3711
- for round in 0..10 {
3712
- let path = std::env::temp_dir().join(format!(
3713
- "workload-admission-race-{round}-{}.log",
3714
- std::process::id()
3715
- ));
3716
- let _ = fs::remove_file(&path);
3717
- let db = FeltDb::open(&path).unwrap();
3718
- let stores = [
3719
- WorkloadStore::from_db(db.clone()).unwrap(),
3720
- WorkloadStore::from_db(db.clone()).unwrap(),
3721
- ];
3722
- let barrier = std::sync::Barrier::new(2);
3723
- let outcomes = std::thread::scope(|scope| {
3724
- stores
3725
- .into_iter()
3726
- .enumerate()
3727
- .map(|(i, mut store)| {
3728
- let barrier = &barrier;
3729
- scope.spawn(move || {
3730
- let mut request = create();
3731
- request.priority = i as i32;
3732
- barrier.wait();
3733
- store
3734
- .admit(request)
3735
- .map(|(w, created)| (w.priority, created))
3736
- })
3737
- })
3738
- .collect::<Vec<_>>()
3739
- .into_iter()
3740
- .map(|handle| handle.join().unwrap())
3741
- .collect::<Vec<_>>()
3742
- });
3743
- let admitted = outcomes
3744
- .iter()
3745
- .filter(|o| matches!(o, Ok((_, true))))
3746
- .count();
3747
- let refused = outcomes
3748
- .iter()
3749
- .filter(|o| {
3750
- o.as_ref()
3751
- .is_err_and(|e| e.code == "IDEMPOTENCY_KEY_REUSED")
3752
- })
3753
- .count();
3754
- assert_eq!((admitted, refused), (1, 1), "round {round}: {outcomes:?}");
3755
- let _ = fs::remove_file(path);
3756
- }
3757
- }
3758
-
3759
- fn claimed(
3760
- name: &str,
3761
- policy: impl FnOnce(&mut CreateWorkload),
3762
- ) -> (PathBuf, WorkloadStore, String, WorkloadClaim) {
3763
- let (path, mut store) = store(name);
3764
- let mut request = create();
3765
- policy(&mut request);
3766
- let id = store.create(request).unwrap().workload_id;
3767
- let claim = store.claim(&id, "a", &capabilities(), 11, 10_000).unwrap();
3768
- (path, store, id, claim)
3769
- }
3770
-
3771
- #[test]
3772
- fn a_worker_reports_failure_from_claimed_fenced_and_by_policy() {
3773
- // Could not start: a known failure before any effect, retried by kind.
3774
- let (path, mut store, id, claim) = claimed("fail-claimed", |_| {});
3775
- let state = store
3776
- .fail_with_effect(
3777
- &id,
3778
- &claim.claim_id,
3779
- "a",
3780
- claim.fencing_token,
3781
- "transient_connection",
3782
- "no route",
3783
- None,
3784
- 12,
3785
- )
3786
- .unwrap();
3787
- assert_eq!(state, WorkloadState::Ready);
3788
- let w = store.get(&id).unwrap();
3789
- let failed = w
3790
- .execution_history
3791
- .iter()
3792
- .find(|e| e.to == WorkloadState::Failed)
3793
- .unwrap();
3794
- assert_eq!(
3795
- failed.evidence["effect"], "none",
3796
- "never started, so no effect"
3797
- );
3798
- assert_eq!(w.attempts[0].status, AttemptStatus::Failed);
3799
- // The identical report again (lost response) replays and commits nothing.
3800
- let version = version_of(&store, &id);
3801
- assert_eq!(
3802
- store
3803
- .fail_with_effect(
3804
- &id,
3805
- &claim.claim_id,
3806
- "a",
3807
- claim.fencing_token,
3808
- "transient_connection",
3809
- "no route",
3810
- None,
3811
- 13
3812
- )
3813
- .unwrap(),
3814
- WorkloadState::Ready
3815
- );
3816
- assert_eq!(version_of(&store, &id), version);
3817
- // A different report from the same attempt is answered by its fence.
3818
- assert_eq!(
3819
- store
3820
- .fail_with_effect(
3821
- &id,
3822
- &claim.claim_id,
3823
- "a",
3824
- claim.fencing_token,
3825
- "timeout",
3826
- "other",
3827
- None,
3828
- 13
3829
- )
3830
- .unwrap_err()
3831
- .code,
3832
- "WORKLOAD_CLAIM_INVALID"
3833
- );
3834
- // So is a completion: the failure stands.
3835
- assert_eq!(
3836
- store
3837
- .complete(
3838
- &id,
3839
- &claim.claim_id,
3840
- "a",
3841
- claim.fencing_token,
3842
- "r",
3843
- Value::Null,
3844
- vec![],
3845
- 1,
3846
- 13
3847
- )
3848
- .unwrap_err()
3849
- .code,
3850
- "WORKLOAD_CLAIM_INVALID"
3851
- );
3852
- // Another worker's claim fences the old attempt out.
3853
- let next = store.claim(&id, "b", &capabilities(), 14, 10_000).unwrap();
3854
- assert_eq!(
3855
- store
3856
- .fail_with_effect(
3857
- &id,
3858
- &claim.claim_id,
3859
- "a",
3860
- claim.fencing_token,
3861
- "timeout",
3862
- "late",
3863
- None,
3864
- 15
3865
- )
3866
- .unwrap_err()
3867
- .code,
3868
- "WORKLOAD_STALE_FENCE"
3869
- );
3870
- assert_eq!(
3871
- store
3872
- .get(&id)
3873
- .unwrap()
3874
- .current_claim
3875
- .as_ref()
3876
- .unwrap()
3877
- .claim_id,
3878
- next.claim_id
3879
- );
3880
- let _ = fs::remove_file(path);
3881
- }
3882
-
3883
- #[test]
3884
- fn failure_policy_decides_retry_and_an_unknown_effect_is_decided_like_a_lost_worker() {
3885
- // A kind the policy does not retry dead-letters.
3886
- let (path, mut store, id, claim) = claimed("fail-kind", |_| {});
3887
- store
3888
- .start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
3889
- .unwrap();
3890
- assert_eq!(
3891
- store
3892
- .fail_with_effect(
3893
- &id,
3894
- &claim.claim_id,
3895
- "a",
3896
- claim.fencing_token,
3897
- "validation",
3898
- "bad input",
3899
- Some("none"),
3900
- 13
3901
- )
3902
- .unwrap(),
3903
- WorkloadState::DeadLetter
3904
- );
3905
- assert_eq!(
3906
- store
3907
- .get(&id)
3908
- .unwrap()
3909
- .execution_history
3910
- .last()
3911
- .unwrap()
3912
- .evidence["decision"],
3913
- "failure_kind_not_retryable"
3914
- );
3915
- let _ = fs::remove_file(path);
3916
-
3917
- // A retryable kind with an unknown effect, under a policy that does not
3918
- // re-run lost work: not retried automatically.
3919
- let (path, mut store, id, claim) = claimed("fail-unknown", |r| {
3920
- r.retry_policy.retry_on = ["timeout".into(), "transient_connection".into()].into();
3921
- });
3922
- store
3923
- .start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
3924
- .unwrap();
3925
- assert_eq!(
3926
- store
3927
- .fail_with_effect(
3928
- &id,
3929
- &claim.claim_id,
3930
- "a",
3931
- claim.fencing_token,
3932
- "timeout",
3933
- "charge sent, no answer",
3934
- Some("unknown"),
3935
- 13
3936
- )
3937
- .unwrap(),
3938
- WorkloadState::DeadLetter
3939
- );
3940
- let w = store.get(&id).unwrap();
3941
- assert_eq!(
3942
- w.execution_history.last().unwrap().evidence["decision"],
3943
- "effect_unknown_not_retryable"
3944
- );
3945
- // An operator can still re-arm it, with a reason.
3946
- store
3947
- .retry_with_reason(&id, "operator", 14, Some("verified: not charged"))
3948
- .unwrap();
3949
- assert_eq!(store.get(&id).unwrap().state, WorkloadState::Ready);
3950
- let _ = fs::remove_file(path);
3951
-
3952
- // The same report under a policy that re-runs lost work is retried.
3953
- let (path, mut store, id, claim) = claimed("fail-unknown-retry", |_| {});
3954
- store
3955
- .start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
3956
- .unwrap();
3957
- assert_eq!(
3958
- store
3959
- .fail_with_effect(
3960
- &id,
3961
- &claim.claim_id,
3962
- "a",
3963
- claim.fencing_token,
3964
- "validation",
3965
- "x",
3966
- Some("unknown"),
3967
- 13
3968
- )
3969
- .unwrap(),
3970
- WorkloadState::Ready
3971
- );
3972
- // An invalid effect is refused before anything changes.
3973
- assert_eq!(
3974
- store
3975
- .fail_with_effect(&id, "c", "a", 9, "x", "y", Some("maybe"), 14)
3976
- .unwrap_err()
3977
- .code,
3978
- "WORKLOAD_INVALID"
3979
- );
3980
- let _ = fs::remove_file(path);
3981
-
3982
- // The attempt budget bounds retries.
3983
- let (path, mut store, id, mut claim) =
3984
- claimed("fail-budget", |r| r.retry_policy.max_attempts = 2);
3985
- assert_eq!(
3986
- store
3987
- .fail(
3988
- &id,
3989
- &claim.claim_id,
3990
- "a",
3991
- claim.fencing_token,
3992
- "timeout",
3993
- "x",
3994
- 12
3995
- )
3996
- .unwrap(),
3997
- WorkloadState::Ready
3998
- );
3999
- let next = store.get(&id).unwrap().next_attempt_at.unwrap();
4000
- claim = store
4001
- .claim(&id, "a", &capabilities(), next, 10_000)
4002
- .unwrap();
4003
- assert_eq!(
4004
- store
4005
- .fail(
4006
- &id,
4007
- &claim.claim_id,
4008
- "a",
4009
- claim.fencing_token,
4010
- "timeout",
4011
- "x",
4012
- next
4013
- )
4014
- .unwrap(),
4015
- WorkloadState::DeadLetter
4016
- );
4017
- assert_eq!(
4018
- store
4019
- .get(&id)
4020
- .unwrap()
4021
- .execution_history
4022
- .last()
4023
- .unwrap()
4024
- .evidence["decision"],
4025
- "attempt_budget_exhausted"
4026
- );
4027
- let _ = fs::remove_file(path);
4028
- }
4029
-
4030
- #[test]
4031
- fn failure_racing_completion_commits_exactly_one_outcome() {
4032
- for completion_first in [true, false] {
4033
- let (path, mut store, id, claim) = claimed(&format!("race-{completion_first}"), |_| {});
4034
- store
4035
- .start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
4036
- .unwrap();
4037
- let complete = |s: &mut WorkloadStore| {
4038
- s.complete(
4039
- &id,
4040
- &claim.claim_id,
4041
- "a",
4042
- claim.fencing_token,
4043
- "r",
4044
- Value::Null,
4045
- vec![],
4046
- 1,
4047
- 13,
4048
- )
4049
- .map(|_| ())
4050
- };
4051
- let fail = |s: &mut WorkloadStore| {
4052
- s.fail(
4053
- &id,
4054
- &claim.claim_id,
4055
- "a",
4056
- claim.fencing_token,
4057
- "timeout",
4058
- "x",
4059
- 13,
4060
- )
4061
- .map(|_| ())
4062
- };
4063
- let (first, second) = if completion_first {
4064
- (complete(&mut store), fail(&mut store))
4065
- } else {
4066
- (fail(&mut store), complete(&mut store))
4067
- };
4068
- assert!(first.is_ok());
4069
- assert_eq!(second.unwrap_err().code, "WORKLOAD_CLAIM_INVALID");
4070
- let w = store.get(&id).unwrap();
4071
- assert_eq!(
4072
- w.state,
4073
- if completion_first {
4074
- WorkloadState::Succeeded
4075
- } else {
4076
- WorkloadState::Ready
4077
- }
4078
- );
4079
- assert_eq!(w.attempts.len(), 1);
4080
- let _ = fs::remove_file(path);
4081
- }
4082
- }
4083
-
4084
- #[test]
4085
- fn deadlines_are_enforced_from_committed_timestamps_and_race_safely() {
4086
- // execution: a heartbeat inside the deadline keeps the attempt; after it,
4087
- // every worker call is refused and the attempt is timed out, once.
4088
- let (path, mut store, id, claim) =
4089
- claimed("deadline-exec", |r| r.timeouts.execution_ms = Some(5_000));
4090
- store
4091
- .start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
4092
- .unwrap();
4093
- store
4094
- .heartbeat(&id, &claim.claim_id, "a", claim.fencing_token, 16, 10_000)
4095
- .unwrap();
4096
- assert!(store.enforce_deadlines(16).unwrap().is_empty());
4097
- let refused = store
4098
- .complete(
4099
- &id,
4100
- &claim.claim_id,
4101
- "a",
4102
- claim.fencing_token,
4103
- "late",
4104
- Value::Null,
4105
- vec![],
4106
- 1,
4107
- 17,
4108
- )
4109
- .unwrap_err();
4110
- assert_eq!(refused.code, "WORKLOAD_TIMED_OUT");
4111
- assert_eq!(refused.details.unwrap()["deadline"], "execution");
4112
- let w = store.get(&id).unwrap().clone();
4113
- assert_eq!(
4114
- w.state,
4115
- WorkloadState::Ready,
4116
- "timeout is retryable by default"
4117
- );
4118
- assert_eq!(w.attempts[0].status, AttemptStatus::TimedOut);
4119
- assert!(w.result.is_none(), "a late completion never lands");
4120
- assert!(
4121
- store.enforce_deadlines(30).unwrap().is_empty(),
4122
- "timed out once"
4123
- );
4124
- let _ = fs::remove_file(path);
4125
-
4126
- // heartbeat: the gap since the last heartbeat; a heartbeat that commits
4127
- // first moves the deadline, the recovery pass after it changes nothing.
4128
- let (path, mut store, id, claim) = claimed("deadline-heartbeat", |r| {
4129
- r.timeouts.heartbeat_ms = Some(3_000)
4130
- });
4131
- store
4132
- .start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
4133
- .unwrap();
4134
- store
4135
- .heartbeat(&id, &claim.claim_id, "a", claim.fencing_token, 13, 10_000)
4136
- .unwrap();
4137
- assert!(store.enforce_deadlines(15).unwrap().is_empty());
4138
- assert_eq!(store.enforce_deadlines(16).unwrap(), vec![id.clone()]);
4139
- assert_eq!(
4140
- store
4141
- .heartbeat(&id, &claim.claim_id, "a", claim.fencing_token, 16, 10_000)
4142
- .unwrap_err()
4143
- .code,
4144
- "WORKLOAD_CLAIM_INVALID",
4145
- "the timed-out claim is gone"
4146
- );
4147
- let _ = fs::remove_file(path);
4148
-
4149
- // claim: claimed but never started; not retried when the policy says so.
4150
- let (path, mut store, id, claim) = claimed("deadline-claim", |r| {
4151
- r.timeouts.claim_ms = Some(2_000);
4152
- r.retry_policy.retry_on = ["worker_failure".into()].into();
4153
- });
4154
- let refused = store
4155
- .start(&id, &claim.claim_id, "a", claim.fencing_token, 13)
4156
- .unwrap_err();
4157
- assert_eq!(refused.code, "WORKLOAD_TIMED_OUT");
4158
- let w = store.get(&id).unwrap();
4159
- assert_eq!(w.state, WorkloadState::DeadLetter);
4160
- assert_eq!(
4161
- w.execution_history.last().unwrap().evidence["decision"],
4162
- "timeout_not_retryable"
4163
- );
4164
- let _ = fs::remove_file(path);
4165
-
4166
- // queue and overall bound the work: dead-lettered, never retried, and a
4167
- // queued workload's previous attempt is not rewritten.
4168
- let (path, mut store) = super::tests::store("deadline-queue");
4169
- let mut request = create();
4170
- request.timeouts.queue_ms = Some(4_000);
4171
- let id = store.create(request).unwrap().workload_id;
4172
- assert_eq!(
4173
- store
4174
- .claim(&id, "a", &capabilities(), 14, 10_000)
4175
- .unwrap_err()
4176
- .code,
4177
- "WORKLOAD_TIMED_OUT"
4178
- );
4179
- let w = store.get(&id).unwrap();
4180
- assert_eq!(w.state, WorkloadState::DeadLetter);
4181
- assert_eq!(
4182
- w.execution_history.last().unwrap().evidence["decision"],
4183
- "deadline_not_retryable"
4184
- );
4185
- assert!(w.attempts.is_empty());
4186
- let _ = fs::remove_file(path);
4187
-
4188
- let (path, mut store, id, claim) =
4189
- claimed("deadline-overall", |r| r.timeouts.overall_ms = Some(10_000));
4190
- store
4191
- .start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
4192
- .unwrap();
4193
- // Restart: the deadline comes from the committed record.
4194
- drop(store);
4195
- let mut store = WorkloadStore::load(&path).unwrap();
4196
- assert!(store.enforce_deadlines(19).unwrap().is_empty());
4197
- assert_eq!(store.enforce_deadlines(20).unwrap(), vec![id.clone()]);
4198
- assert_eq!(store.get(&id).unwrap().state, WorkloadState::DeadLetter);
4199
- let _ = fs::remove_file(path);
4200
- }
4201
-
4202
- #[test]
4203
- fn deadlines_are_versioned_and_zero_is_refused() {
4204
- let (path, mut store) = store("deadline-version");
4205
- let mut request = create();
4206
- request.timeouts.queue_ms = Some(0);
4207
- assert_eq!(store.create(request).unwrap_err().code, "WORKLOAD_INVALID");
4208
- // A record admitted before deadlines were enforced keeps its contract.
4209
- let mut request = create();
4210
- request.timeouts.queue_ms = Some(1_000);
4211
- let id = store.create(request).unwrap().workload_id;
4212
- assert_eq!(
4213
- store.get(&id).unwrap().contract_version,
4214
- DEADLINE_CONTRACT_VERSION
4215
- );
4216
- let mut legacy = store.get(&id).unwrap().clone();
4217
- legacy.contract_version = 1;
4218
- assert_eq!(deadline_breach(&legacy, 1_000_000), None);
4219
- assert!(deadline_breach(store.get(&id).unwrap(), 1_000_000).is_some());
4220
- let _ = fs::remove_file(path);
4221
- }
4222
- }