@feltdb/core 0.15.0 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-state.d.ts +20 -0
- package/dist/agent-state.d.ts.map +1 -1
- package/dist/agent-state.js +36 -0
- package/dist/build-identity.json +12 -4
- package/dist/cli/commands.js +187 -4
- package/dist/cli/publish-engine.js +116 -16
- package/dist/collection.d.ts.map +1 -1
- package/dist/collection.js +20 -2
- package/dist/create/build-identity.json +12 -4
- package/dist/create/managed-account.js +93 -35
- package/dist/create/package-versions.js +1 -1
- package/dist/create/server-source/Cargo.lock +6 -509
- package/dist/create/server-source/Cargo.toml +2 -13
- package/dist/create/server-source/crates/feltdb-client/Cargo.toml +1 -1
- package/dist/create/server-source/crates/feltdb-server/Cargo.toml +1 -1
- package/dist/create/server-source/crates/feltdb-server/src/main.rs +356 -24
- package/dist/create/server-source/crates/feltdb-server/tests/rust_client.rs +10 -1
- package/dist/create/server-source/crates/feltdb-wasm/Cargo.toml +1 -1
- package/dist/db.d.ts +11 -1
- package/dist/db.d.ts.map +1 -1
- package/dist/db.js +15 -2
- package/dist/error-codes.d.ts +5 -0
- package/dist/error-codes.d.ts.map +1 -1
- package/dist/error-codes.js +9 -1
- package/dist/http-db.d.ts +16 -0
- package/dist/http-db.d.ts.map +1 -1
- package/dist/http-db.js +36 -6
- package/dist/managed-recovery.d.ts +6 -0
- package/dist/managed-recovery.d.ts.map +1 -1
- package/dist/managed-recovery.js +9 -0
- package/dist/studio-app/assets/{feltdb_wasm-BjM1ylL3.js → feltdb_wasm-BUIxCuTR.js} +1 -1
- package/dist/studio-app/assets/feltdb_wasm_bg-C3D6y6DK.wasm +0 -0
- package/dist/studio-app/assets/index-BhXYAhgS.js +37 -0
- package/dist/studio-app/index.html +1 -1
- package/dist/wasm/feltdb_wasm_bg.wasm +0 -0
- package/dist/wasm/package.json +1 -1
- package/dist/workload.d.ts +55 -3
- package/dist/workload.d.ts.map +1 -1
- package/dist/workload.js +23 -0
- package/dist/workspace/workspace-connection.d.ts.map +1 -1
- package/dist/workspace/workspace-connection.js +14 -3
- package/package.json +1 -1
- package/dist/create/server-source/crates/feltdb/Cargo.lock +0 -175
- package/dist/create/server-source/crates/feltdb/Cargo.toml +0 -102
- package/dist/create/server-source/crates/feltdb/LICENSE +0 -21
- package/dist/create/server-source/crates/feltdb/README.md +0 -75
- package/dist/create/server-source/crates/feltdb/benches/baselines/gate-13-redux.json +0 -370
- package/dist/create/server-source/crates/feltdb/benches/gate13_baseline.rs +0 -589
- package/dist/create/server-source/crates/feltdb/benches/gate13_phase_7_1_release_economics.rs +0 -247
- package/dist/create/server-source/crates/feltdb/benches/gate_13_redux.rs +0 -440
- package/dist/create/server-source/crates/feltdb/benches/gate_13_regression_runner.rs +0 -275
- package/dist/create/server-source/crates/feltdb/benches/gate_14a_concurrent_writer_scaling.rs +0 -381
- package/dist/create/server-source/crates/feltdb/benches/gate_14a_production_admission_revalidation.rs +0 -467
- package/dist/create/server-source/crates/feltdb/benches/gate_14a_rc2_admission_contract.rs +0 -489
- package/dist/create/server-source/crates/feltdb/benches/gate_14a_rc_root_cause.rs +0 -281
- package/dist/create/server-source/crates/feltdb/benches/gate_14a_sync1_queued_prototype.rs +0 -606
- package/dist/create/server-source/crates/feltdb/benches/gate_14a_sync_economics.rs +0 -531
- package/dist/create/server-source/crates/feltdb/benches/gate_14b_causal_backlog_scaling.rs +0 -462
- package/dist/create/server-source/crates/feltdb/benches/gate_14c_replication_contract_test.rs +0 -505
- package/dist/create/server-source/crates/feltdb/benches/gate_14c_replication_scaling.rs +0 -484
- package/dist/create/server-source/crates/feltdb/benches/gate_14d_combined_dimension_scaling.rs +0 -758
- package/dist/create/server-source/crates/feltdb/benches/phase_7_1_2_optimization_benchmark.rs +0 -421
- package/dist/create/server-source/crates/feltdb/benches/phase_7_1_3_crossover_analysis.rs +0 -329
- package/dist/create/server-source/crates/feltdb/src/acceptance_tests.rs +0 -1705
- package/dist/create/server-source/crates/feltdb/src/acquisition.rs +0 -286
- package/dist/create/server-source/crates/feltdb/src/admission.rs +0 -185
- package/dist/create/server-source/crates/feltdb/src/admission_contract_tests.rs +0 -507
- package/dist/create/server-source/crates/feltdb/src/adversarial_transport.rs +0 -539
- package/dist/create/server-source/crates/feltdb/src/agent_state.rs +0 -320
- package/dist/create/server-source/crates/feltdb/src/agent_transition_contract.rs +0 -1545
- package/dist/create/server-source/crates/feltdb/src/analytics.rs +0 -521
- package/dist/create/server-source/crates/feltdb/src/application.rs +0 -2674
- package/dist/create/server-source/crates/feltdb/src/application_runtime.rs +0 -1070
- package/dist/create/server-source/crates/feltdb/src/authority.rs +0 -479
- package/dist/create/server-source/crates/feltdb/src/authority_failover.rs +0 -847
- package/dist/create/server-source/crates/feltdb/src/authorization.rs +0 -1030
- package/dist/create/server-source/crates/feltdb/src/authorization_security_tests.rs +0 -1122
- package/dist/create/server-source/crates/feltdb/src/bin/feltdb-authority-client.rs +0 -32
- package/dist/create/server-source/crates/feltdb/src/bin/feltdb-authority.rs +0 -19
- package/dist/create/server-source/crates/feltdb/src/bin/feltdb_node.rs +0 -1474
- package/dist/create/server-source/crates/feltdb/src/capabilities/mod.rs +0 -8
- package/dist/create/server-source/crates/feltdb/src/capabilities/search.rs +0 -418
- package/dist/create/server-source/crates/feltdb/src/capabilities/vector.rs +0 -482
- package/dist/create/server-source/crates/feltdb/src/capability.rs +0 -903
- package/dist/create/server-source/crates/feltdb/src/cardinality_diagnostics.rs +0 -263
- package/dist/create/server-source/crates/feltdb/src/cardinality_endpoint.rs +0 -77
- package/dist/create/server-source/crates/feltdb/src/causal_backlog_bound.rs +0 -496
- package/dist/create/server-source/crates/feltdb/src/causal_dependency_barrier.rs +0 -2574
- package/dist/create/server-source/crates/feltdb/src/causal_dependency_barrier_phase_7_1.rs +0 -212
- package/dist/create/server-source/crates/feltdb/src/concurrency_fuzzing.rs +0 -422
- package/dist/create/server-source/crates/feltdb/src/consistency_contract.rs +0 -445
- package/dist/create/server-source/crates/feltdb/src/content_distribution.rs +0 -465
- package/dist/create/server-source/crates/feltdb/src/convergence.rs +0 -634
- package/dist/create/server-source/crates/feltdb/src/crash_atomic_boundary.rs +0 -427
- package/dist/create/server-source/crates/feltdb/src/crash_injection.rs +0 -376
- package/dist/create/server-source/crates/feltdb/src/crash_recovery_tests.rs +0 -365
- package/dist/create/server-source/crates/feltdb/src/cron.rs +0 -294
- package/dist/create/server-source/crates/feltdb/src/dedup_bound_investigation.rs +0 -483
- package/dist/create/server-source/crates/feltdb/src/distributed_indexing.rs +0 -477
- package/dist/create/server-source/crates/feltdb/src/distributed_tests.rs +0 -1003
- package/dist/create/server-source/crates/feltdb/src/distributed_transactions.rs +0 -1334
- package/dist/create/server-source/crates/feltdb/src/durability_guarantees.rs +0 -368
- package/dist/create/server-source/crates/feltdb/src/durable_dedup_set.rs +0 -231
- package/dist/create/server-source/crates/feltdb/src/durable_operation_identity.rs +0 -482
- package/dist/create/server-source/crates/feltdb/src/durable_operation_log.rs +0 -203
- package/dist/create/server-source/crates/feltdb/src/durable_sync.rs +0 -317
- package/dist/create/server-source/crates/feltdb/src/equality_index.rs +0 -595
- package/dist/create/server-source/crates/feltdb/src/execution.rs +0 -426
- package/dist/create/server-source/crates/feltdb/src/in_process_transport.rs +0 -214
- package/dist/create/server-source/crates/feltdb/src/indexing.rs +0 -776
- package/dist/create/server-source/crates/feltdb/src/lib.rs +0 -7365
- package/dist/create/server-source/crates/feltdb/src/managed_cas_tests.rs +0 -234
- package/dist/create/server-source/crates/feltdb/src/materialization.rs +0 -184
- package/dist/create/server-source/crates/feltdb/src/metrics.rs +0 -266
- package/dist/create/server-source/crates/feltdb/src/multi_node_convergence.rs +0 -310
- package/dist/create/server-source/crates/feltdb/src/multi_operation_transaction.rs +0 -736
- package/dist/create/server-source/crates/feltdb/src/observability.rs +0 -379
- package/dist/create/server-source/crates/feltdb/src/operation.rs +0 -290
- package/dist/create/server-source/crates/feltdb/src/operation_algebra.rs +0 -439
- package/dist/create/server-source/crates/feltdb/src/operation_log.rs +0 -349
- package/dist/create/server-source/crates/feltdb/src/p1_application_atomicity.rs +0 -332
- package/dist/create/server-source/crates/feltdb/src/p1_atomicity_acceptance.rs +0 -718
- package/dist/create/server-source/crates/feltdb/src/partition_reconciliation.rs +0 -487
- package/dist/create/server-source/crates/feltdb/src/peer_registry.rs +0 -166
- package/dist/create/server-source/crates/feltdb/src/permutation_scheduler.rs +0 -289
- package/dist/create/server-source/crates/feltdb/src/persistence_reality.rs +0 -566
- package/dist/create/server-source/crates/feltdb/src/phase1b_acceptance.rs +0 -3391
- package/dist/create/server-source/crates/feltdb/src/phase1c1_acceptance.rs +0 -203
- package/dist/create/server-source/crates/feltdb/src/phase1c2_acceptance.rs +0 -261
- package/dist/create/server-source/crates/feltdb/src/phase1c3_acceptance.rs +0 -502
- package/dist/create/server-source/crates/feltdb/src/phase1c_atomicity_proof.rs +0 -216
- package/dist/create/server-source/crates/feltdb/src/phase5_integration.rs +0 -303
- package/dist/create/server-source/crates/feltdb/src/phase5_scenarios.rs +0 -323
- package/dist/create/server-source/crates/feltdb/src/phase6_adversarial_scenarios.rs +0 -531
- package/dist/create/server-source/crates/feltdb/src/phase6_convergence_validator.rs +0 -406
- package/dist/create/server-source/crates/feltdb/src/phase6_persistence.rs +0 -436
- package/dist/create/server-source/crates/feltdb/src/phase_1c_real_tcp.rs +0 -387
- package/dist/create/server-source/crates/feltdb/src/phase_2a_failures.rs +0 -378
- package/dist/create/server-source/crates/feltdb/src/phase_2b_network.rs +0 -359
- package/dist/create/server-source/crates/feltdb/src/phase_2c_cascading.rs +0 -372
- package/dist/create/server-source/crates/feltdb/src/phase_3_durability.rs +0 -404
- package/dist/create/server-source/crates/feltdb/src/phase_4_baseline.rs +0 -376
- package/dist/create/server-source/crates/feltdb/src/phase_5_soak.rs +0 -461
- package/dist/create/server-source/crates/feltdb/src/policy_evaluation.rs +0 -2125
- package/dist/create/server-source/crates/feltdb/src/production_api.rs +0 -418
- package/dist/create/server-source/crates/feltdb/src/provenance.rs +0 -207
- package/dist/create/server-source/crates/feltdb/src/query_execution_diagnostics.rs +0 -126
- package/dist/create/server-source/crates/feltdb/src/query_performance.rs +0 -218
- package/dist/create/server-source/crates/feltdb/src/read_basis.rs +0 -257
- package/dist/create/server-source/crates/feltdb/src/references.rs +0 -404
- package/dist/create/server-source/crates/feltdb/src/replay_fuzzing.rs +0 -401
- package/dist/create/server-source/crates/feltdb/src/replica_acknowledgements.rs +0 -501
- package/dist/create/server-source/crates/feltdb/src/replica_membership.rs +0 -680
- package/dist/create/server-source/crates/feltdb/src/replication_manager.rs +0 -163
- package/dist/create/server-source/crates/feltdb/src/replication_protocol.rs +0 -133
- package/dist/create/server-source/crates/feltdb/src/routing.rs +0 -409
- package/dist/create/server-source/crates/feltdb/src/semantic_decision.rs +0 -2029
- package/dist/create/server-source/crates/feltdb/src/semantic_query.rs +0 -599
- package/dist/create/server-source/crates/feltdb/src/sharding.rs +0 -549
- package/dist/create/server-source/crates/feltdb/src/state_conflict_contract.rs +0 -516
- package/dist/create/server-source/crates/feltdb/src/state_contract.rs +0 -6003
- package/dist/create/server-source/crates/feltdb/src/state_diff_contract.rs +0 -222
- package/dist/create/server-source/crates/feltdb/src/state_facade.rs +0 -320
- package/dist/create/server-source/crates/feltdb/src/state_hash.rs +0 -291
- package/dist/create/server-source/crates/feltdb/src/state_model.rs +0 -4158
- package/dist/create/server-source/crates/feltdb/src/state_transition_store.rs +0 -379
- package/dist/create/server-source/crates/feltdb/src/state_trigger.rs +0 -673
- package/dist/create/server-source/crates/feltdb/src/storage.rs +0 -273
- package/dist/create/server-source/crates/feltdb/src/submission.rs +0 -471
- package/dist/create/server-source/crates/feltdb/src/sync.rs +0 -495
- package/dist/create/server-source/crates/feltdb/src/sync_contract.rs +0 -822
- package/dist/create/server-source/crates/feltdb/src/tcp_transport.rs +0 -433
- package/dist/create/server-source/crates/feltdb/src/transaction_api.rs +0 -462
- package/dist/create/server-source/crates/feltdb/src/transaction_invariants.rs +0 -965
- package/dist/create/server-source/crates/feltdb/src/transaction_preconditions.rs +0 -1120
- package/dist/create/server-source/crates/feltdb/src/transactions.rs +0 -643
- package/dist/create/server-source/crates/feltdb/src/trigger.rs +0 -390
- package/dist/create/server-source/crates/feltdb/src/trigger_contract.rs +0 -749
- package/dist/create/server-source/crates/feltdb/src/versioned_object_substrate.rs +0 -708
- package/dist/create/server-source/crates/feltdb/src/worker_mesh.rs +0 -1269
- package/dist/create/server-source/crates/feltdb/src/workflow.rs +0 -713
- package/dist/create/server-source/crates/feltdb/src/workflow_acceptance_tests.rs +0 -510
- package/dist/create/server-source/crates/feltdb/src/workflow_integration.rs +0 -354
- package/dist/create/server-source/crates/feltdb/src/workflow_runtime.rs +0 -594
- package/dist/create/server-source/crates/feltdb/src/workload.rs +0 -4222
- package/dist/create/server-source/crates/feltdb/src/workload_diagnostics.rs +0 -456
- package/dist/create/server-source/crates/feltdb/tests/agent_state_native.rs +0 -1188
- package/dist/create/server-source/crates/feltdb/tests/agent_transition_contract_conformance.rs +0 -2884
- package/dist/create/server-source/crates/feltdb/tests/authority_process.rs +0 -297
- package/dist/create/server-source/crates/feltdb/tests/authority_transaction_identity.rs +0 -534
- package/dist/create/server-source/crates/feltdb/tests/bounded_read_contract.rs +0 -132
- package/dist/create/server-source/crates/feltdb/tests/branching_evidence.rs +0 -299
- package/dist/create/server-source/crates/feltdb/tests/common/mod.rs +0 -96
- package/dist/create/server-source/crates/feltdb/tests/compaction_stall_contract.rs +0 -272
- package/dist/create/server-source/crates/feltdb/tests/crash_durability_contract.rs +0 -467
- package/dist/create/server-source/crates/feltdb/tests/current_revision_authority_evidence.rs +0 -309
- package/dist/create/server-source/crates/feltdb/tests/durable_backup_contract.rs +0 -445
- package/dist/create/server-source/crates/feltdb/tests/durable_corruption_contract.rs +0 -518
- package/dist/create/server-source/crates/feltdb/tests/durable_format_compatibility.rs +0 -392
- package/dist/create/server-source/crates/feltdb/tests/feltdb_state_boundary_tests.rs +0 -850
- package/dist/create/server-source/crates/feltdb/tests/fixtures/agent_transition_corpus.json +0 -23340
- package/dist/create/server-source/crates/feltdb/tests/fixtures/state_conflict_contract_corpus.json +0 -1916
- package/dist/create/server-source/crates/feltdb/tests/fixtures/state_diff_contract_corpus.json +0 -1878
- package/dist/create/server-source/crates/feltdb/tests/fixtures/trigger_contract_corpus.json +0 -1862
- package/dist/create/server-source/crates/feltdb/tests/generalized_version_control_evidence.rs +0 -123
- package/dist/create/server-source/crates/feltdb/tests/generalized_version_control_substrate_completion.rs +0 -299
- package/dist/create/server-source/crates/feltdb/tests/local_process_ownership.rs +0 -69
- package/dist/create/server-source/crates/feltdb/tests/managed_incident_regression.rs +0 -191
- package/dist/create/server-source/crates/feltdb/tests/operational_health_contract.rs +0 -278
- package/dist/create/server-source/crates/feltdb/tests/pr34_query_collection.rs +0 -234
- package/dist/create/server-source/crates/feltdb/tests/pr35_equality_index.rs +0 -947
- package/dist/create/server-source/crates/feltdb/tests/pr7_self_authorization_proof.rs +0 -404
- package/dist/create/server-source/crates/feltdb/tests/pr8_vocabulary_assessment.rs +0 -916
- package/dist/create/server-source/crates/feltdb/tests/pr9_phase2_boundary_tests.rs +0 -1045
- package/dist/create/server-source/crates/feltdb/tests/pr9_phase3a_path_a_tests.rs +0 -347
- package/dist/create/server-source/crates/feltdb/tests/pr9_phase3c_authorized_mutations.rs +0 -361
- package/dist/create/server-source/crates/feltdb/tests/pr9_phase3c_role_based_authorization.rs +0 -330
- package/dist/create/server-source/crates/feltdb/tests/pr9_phase3c_simple_auth_delete.rs +0 -93
- package/dist/create/server-source/crates/feltdb/tests/pr9_phase3c_team_delete_role_authorization.rs +0 -622
- package/dist/create/server-source/crates/feltdb/tests/pr9_teams_role_based_access.rs +0 -517
- package/dist/create/server-source/crates/feltdb/tests/production_certification.rs +0 -1153
- package/dist/create/server-source/crates/feltdb/tests/production_contract.rs +0 -771
- package/dist/create/server-source/crates/feltdb/tests/production_readiness_contract.rs +0 -1383
- package/dist/create/server-source/crates/feltdb/tests/reconciliation_application.rs +0 -868
- package/dist/create/server-source/crates/feltdb/tests/reconciliation_wire_format_evidence.rs +0 -221
- package/dist/create/server-source/crates/feltdb/tests/replicated_history_contract.rs +0 -417
- package/dist/create/server-source/crates/feltdb/tests/resource_scoped_revisions.rs +0 -338
- package/dist/create/server-source/crates/feltdb/tests/revision_identity_contract.rs +0 -1039
- package/dist/create/server-source/crates/feltdb/tests/revision_model_decision.rs +0 -739
- package/dist/create/server-source/crates/feltdb/tests/revision_retention_boundary_evidence.rs +0 -452
- package/dist/create/server-source/crates/feltdb/tests/saas_authorization_integration.rs +0 -81
- package/dist/create/server-source/crates/feltdb/tests/saas_invitation_lifecycle.rs +0 -438
- package/dist/create/server-source/crates/feltdb/tests/semantic_decision.rs +0 -1165
- package/dist/create/server-source/crates/feltdb/tests/semantic_query.rs +0 -767
- package/dist/create/server-source/crates/feltdb/tests/semantic_query_contract.rs +0 -363
- package/dist/create/server-source/crates/feltdb/tests/state_conflict_contract_conformance.rs +0 -1799
- package/dist/create/server-source/crates/feltdb/tests/state_diff_contract_conformance.rs +0 -1316
- package/dist/create/server-source/crates/feltdb/tests/state_model_integration.rs +0 -358
- package/dist/create/server-source/crates/feltdb/tests/state_persistence_integration.rs +0 -365
- package/dist/create/server-source/crates/feltdb/tests/state_ref_authority.rs +0 -918
- package/dist/create/server-source/crates/feltdb/tests/state_store_boundary_evidence.rs +0 -299
- package/dist/create/server-source/crates/feltdb/tests/storage_independent_versioned_object_substrate.rs +0 -328
- package/dist/create/server-source/crates/feltdb/tests/sync_divergence_evidence.rs +0 -255
- package/dist/create/server-source/crates/feltdb/tests/three_way_input_boundary_evidence.rs +0 -249
- package/dist/create/server-source/crates/feltdb/tests/trigger_contract_conformance.rs +0 -994
- package/dist/create/server-source/crates/feltdb/tests/workload_envelope_contract.rs +0 -484
- package/dist/studio-app/assets/feltdb_wasm_bg-TseJn2Ax.wasm +0 -0
- package/dist/studio-app/assets/index-1wd0uKua.js +0 -30
|
@@ -1,4222 +0,0 @@
|
|
|
1
|
-
//! Durable workload authority. Workers execute; this state machine decides.
|
|
2
|
-
use crate::{
|
|
3
|
-
authorization::{SignedGrant, Subject},
|
|
4
|
-
AtomicMutation, FeltDb, FlowError, RecordPrecondition, StoredRow,
|
|
5
|
-
};
|
|
6
|
-
use rand::{distributions::Alphanumeric, Rng};
|
|
7
|
-
use serde::{Deserialize, Serialize};
|
|
8
|
-
use serde_json::Value;
|
|
9
|
-
use sha2::{Digest, Sha256};
|
|
10
|
-
use std::collections::{BTreeMap, BTreeSet};
|
|
11
|
-
use std::path::Path;
|
|
12
|
-
pub const WORKLOAD_CONTRACT_VERSION: u32 = 1;
|
|
13
|
-
pub const DURABLE_WORK_COLLECTION: &str = "system_durable_work";
|
|
14
|
-
/// Workloads admitted at this contract version or later have their `timeouts`
|
|
15
|
-
/// enforced. Earlier records were admitted when timeouts were stored and
|
|
16
|
-
/// ignored; they keep that behaviour, so an upgrade never times out work whose
|
|
17
|
-
/// requester was told nothing would.
|
|
18
|
-
pub const DEADLINE_CONTRACT_VERSION: u32 = 2;
|
|
19
|
-
|
|
20
|
-
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
|
|
21
|
-
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
|
|
22
|
-
pub enum WorkloadState {
|
|
23
|
-
Created,
|
|
24
|
-
Ready,
|
|
25
|
-
Claimed,
|
|
26
|
-
Running,
|
|
27
|
-
Checkpointed,
|
|
28
|
-
Recovering,
|
|
29
|
-
CancellationRequested,
|
|
30
|
-
Succeeded,
|
|
31
|
-
Failed,
|
|
32
|
-
Cancelled,
|
|
33
|
-
TimedOut,
|
|
34
|
-
DeadLetter,
|
|
35
|
-
}
|
|
36
|
-
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
|
37
|
-
#[serde(rename_all = "snake_case")]
|
|
38
|
-
pub enum BackoffPolicy {
|
|
39
|
-
Fixed,
|
|
40
|
-
Exponential,
|
|
41
|
-
}
|
|
42
|
-
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
43
|
-
pub struct WorkloadRetryPolicy {
|
|
44
|
-
pub max_attempts: u32,
|
|
45
|
-
pub initial_delay_ms: u64,
|
|
46
|
-
pub max_delay_ms: u64,
|
|
47
|
-
pub backoff: BackoffPolicy,
|
|
48
|
-
#[serde(default)]
|
|
49
|
-
pub retry_on: BTreeSet<String>,
|
|
50
|
-
}
|
|
51
|
-
impl Default for WorkloadRetryPolicy {
|
|
52
|
-
fn default() -> Self {
|
|
53
|
-
Self {
|
|
54
|
-
max_attempts: 3,
|
|
55
|
-
initial_delay_ms: 1000,
|
|
56
|
-
max_delay_ms: 300_000,
|
|
57
|
-
backoff: BackoffPolicy::Exponential,
|
|
58
|
-
retry_on: [
|
|
59
|
-
"timeout".into(),
|
|
60
|
-
"worker_failure".into(),
|
|
61
|
-
"transient_connection".into(),
|
|
62
|
-
]
|
|
63
|
-
.into(),
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
|
68
|
-
pub struct WorkloadTimeouts {
|
|
69
|
-
pub queue_ms: Option<u64>,
|
|
70
|
-
pub claim_ms: Option<u64>,
|
|
71
|
-
pub execution_ms: Option<u64>,
|
|
72
|
-
pub heartbeat_ms: Option<u64>,
|
|
73
|
-
pub overall_ms: Option<u64>,
|
|
74
|
-
}
|
|
75
|
-
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
76
|
-
pub struct WorkloadDependency {
|
|
77
|
-
pub workload_id: String,
|
|
78
|
-
#[serde(default = "succeeded_state")]
|
|
79
|
-
pub required_state: WorkloadState,
|
|
80
|
-
}
|
|
81
|
-
fn succeeded_state() -> WorkloadState {
|
|
82
|
-
WorkloadState::Succeeded
|
|
83
|
-
}
|
|
84
|
-
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
85
|
-
pub struct WorkloadClaim {
|
|
86
|
-
pub claim_id: String,
|
|
87
|
-
pub workload_id: String,
|
|
88
|
-
pub worker_id: String,
|
|
89
|
-
pub lease_id: String,
|
|
90
|
-
pub fencing_token: u64,
|
|
91
|
-
pub capability_snapshot: BTreeSet<String>,
|
|
92
|
-
pub claimed_at: i64,
|
|
93
|
-
pub expires_at: i64,
|
|
94
|
-
pub last_heartbeat: i64,
|
|
95
|
-
}
|
|
96
|
-
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
|
97
|
-
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
|
|
98
|
-
pub enum AttemptStatus {
|
|
99
|
-
Claimed,
|
|
100
|
-
Running,
|
|
101
|
-
Checkpointed,
|
|
102
|
-
Succeeded,
|
|
103
|
-
Failed,
|
|
104
|
-
TimedOut,
|
|
105
|
-
Cancelled,
|
|
106
|
-
WorkerLost,
|
|
107
|
-
}
|
|
108
|
-
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
109
|
-
pub struct WorkloadAttempt {
|
|
110
|
-
pub attempt_id: String,
|
|
111
|
-
pub workload_id: String,
|
|
112
|
-
pub worker_id: String,
|
|
113
|
-
pub claim_id: String,
|
|
114
|
-
pub fencing_token: u64,
|
|
115
|
-
pub started_at: Option<i64>,
|
|
116
|
-
pub ended_at: Option<i64>,
|
|
117
|
-
pub input_hash: String,
|
|
118
|
-
pub result_hash: Option<String>,
|
|
119
|
-
pub error: Option<String>,
|
|
120
|
-
#[serde(default)]
|
|
121
|
-
pub resource_usage: BTreeMap<String, u64>,
|
|
122
|
-
pub status: AttemptStatus,
|
|
123
|
-
}
|
|
124
|
-
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
125
|
-
pub struct WorkloadCheckpoint {
|
|
126
|
-
pub checkpoint_id: String,
|
|
127
|
-
pub workload_id: String,
|
|
128
|
-
pub attempt_id: String,
|
|
129
|
-
pub sequence: u64,
|
|
130
|
-
pub state: Value,
|
|
131
|
-
pub state_hash: String,
|
|
132
|
-
pub created_at: i64,
|
|
133
|
-
pub produced_by: String,
|
|
134
|
-
}
|
|
135
|
-
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
136
|
-
pub struct WorkloadResult {
|
|
137
|
-
pub result_id: String,
|
|
138
|
-
pub workload_id: String,
|
|
139
|
-
pub attempt_id: String,
|
|
140
|
-
pub status: String,
|
|
141
|
-
pub output: Value,
|
|
142
|
-
pub output_hash: String,
|
|
143
|
-
#[serde(default)]
|
|
144
|
-
pub artifacts: Vec<String>,
|
|
145
|
-
#[serde(default)]
|
|
146
|
-
pub provenance: BTreeMap<String, String>,
|
|
147
|
-
pub produced_by: String,
|
|
148
|
-
pub created_at: i64,
|
|
149
|
-
pub schema_version: u64,
|
|
150
|
-
}
|
|
151
|
-
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
152
|
-
pub struct WorkloadExecutionContext {
|
|
153
|
-
pub workload_id: String,
|
|
154
|
-
pub attempt_id: String,
|
|
155
|
-
pub worker_id: String,
|
|
156
|
-
pub capabilities: BTreeSet<String>,
|
|
157
|
-
pub connection_scopes: BTreeSet<String>,
|
|
158
|
-
pub secret_references: Vec<String>,
|
|
159
|
-
pub expires_at: i64,
|
|
160
|
-
pub fencing_token: u64,
|
|
161
|
-
/// The work's input, so the claiming worker can execute it.
|
|
162
|
-
#[serde(default)]
|
|
163
|
-
pub input: Value,
|
|
164
|
-
/// Every committed checkpoint, from this and earlier attempts, in order. A
|
|
165
|
-
/// resumed attempt reads these to learn how far lost attempts got.
|
|
166
|
-
#[serde(default)]
|
|
167
|
-
pub checkpoints: Vec<WorkloadCheckpoint>,
|
|
168
|
-
}
|
|
169
|
-
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
170
|
-
pub struct WorkloadEvent {
|
|
171
|
-
pub sequence: u64,
|
|
172
|
-
pub event_type: String,
|
|
173
|
-
pub from: Option<WorkloadState>,
|
|
174
|
-
pub to: WorkloadState,
|
|
175
|
-
pub actor: String,
|
|
176
|
-
pub at: i64,
|
|
177
|
-
#[serde(default)]
|
|
178
|
-
pub evidence: BTreeMap<String, String>,
|
|
179
|
-
}
|
|
180
|
-
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
181
|
-
pub struct Workload {
|
|
182
|
-
pub contract_version: u32,
|
|
183
|
-
pub workload_id: String,
|
|
184
|
-
pub tenant_id: String,
|
|
185
|
-
pub application_id: String,
|
|
186
|
-
pub environment: String,
|
|
187
|
-
pub definition_id: String,
|
|
188
|
-
pub definition_revision: String,
|
|
189
|
-
pub trigger: String,
|
|
190
|
-
pub input: Value,
|
|
191
|
-
pub input_hash: String,
|
|
192
|
-
pub requested_by: Subject,
|
|
193
|
-
#[serde(default)]
|
|
194
|
-
pub session_id: Option<String>,
|
|
195
|
-
pub authorization_snapshot: SignedGrant,
|
|
196
|
-
pub capability_snapshot: BTreeSet<String>,
|
|
197
|
-
pub connection_scopes: BTreeSet<String>,
|
|
198
|
-
pub priority: i32,
|
|
199
|
-
pub idempotency_key: String,
|
|
200
|
-
pub created_at: i64,
|
|
201
|
-
pub updated_at: i64,
|
|
202
|
-
pub state: WorkloadState,
|
|
203
|
-
pub execution_history: Vec<WorkloadEvent>,
|
|
204
|
-
pub attempts: Vec<WorkloadAttempt>,
|
|
205
|
-
pub checkpoints: Vec<WorkloadCheckpoint>,
|
|
206
|
-
pub result: Option<WorkloadResult>,
|
|
207
|
-
pub retry_policy: WorkloadRetryPolicy,
|
|
208
|
-
pub timeouts: WorkloadTimeouts,
|
|
209
|
-
pub attempt_count: u32,
|
|
210
|
-
pub retry_count: u32,
|
|
211
|
-
/// The instant, in unix seconds, before which this workload must not be
|
|
212
|
-
/// claimed. `None` means "eligible as soon as it is `Ready`".
|
|
213
|
-
///
|
|
214
|
-
/// This is the single temporal eligibility concept in the workload model.
|
|
215
|
-
/// It is what retry backoff sets after a failure, what a timeout sets before
|
|
216
|
-
/// re-arming, and what [`CreateWorkload::not_before`] sets at creation, so a
|
|
217
|
-
/// workload that is not yet executable is always not-yet-executable for the
|
|
218
|
-
/// same reason and is refused by the same check in [`WorkloadStore::claim`].
|
|
219
|
-
pub next_attempt_at: Option<i64>,
|
|
220
|
-
pub last_failure: Option<String>,
|
|
221
|
-
pub current_claim: Option<WorkloadClaim>,
|
|
222
|
-
pub fencing_token: u64,
|
|
223
|
-
#[serde(default)]
|
|
224
|
-
pub dependencies: Vec<WorkloadDependency>,
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
impl Workload {
|
|
228
|
-
/// Has this workload's eligibility instant arrived?
|
|
229
|
-
///
|
|
230
|
-
/// Eligibility is only half of claimability — the state must also be
|
|
231
|
-
/// `Ready`, and [`WorkloadStore::claim`] checks both. This answers the
|
|
232
|
-
/// temporal half alone so a scheduler can skip a workload that is merely
|
|
233
|
-
/// early without attempting a claim it knows will be refused.
|
|
234
|
-
pub fn is_eligible_at(&self, now: i64) -> bool {
|
|
235
|
-
self.next_attempt_at.is_none_or(|instant| instant <= now)
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
|
-
|
|
239
|
-
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
240
|
-
pub struct CreateWorkload {
|
|
241
|
-
pub tenant_id: String,
|
|
242
|
-
pub application_id: String,
|
|
243
|
-
pub environment: String,
|
|
244
|
-
pub definition_id: String,
|
|
245
|
-
pub definition_revision: String,
|
|
246
|
-
pub trigger: String,
|
|
247
|
-
pub input: Value,
|
|
248
|
-
pub requested_by: Subject,
|
|
249
|
-
#[serde(default)]
|
|
250
|
-
pub session_id: Option<String>,
|
|
251
|
-
pub authorization_snapshot: SignedGrant,
|
|
252
|
-
pub capability_snapshot: BTreeSet<String>,
|
|
253
|
-
#[serde(default)]
|
|
254
|
-
pub connection_scopes: BTreeSet<String>,
|
|
255
|
-
#[serde(default)]
|
|
256
|
-
pub priority: i32,
|
|
257
|
-
pub idempotency_key: String,
|
|
258
|
-
#[serde(default)]
|
|
259
|
-
pub retry_policy: WorkloadRetryPolicy,
|
|
260
|
-
#[serde(default)]
|
|
261
|
-
pub timeouts: WorkloadTimeouts,
|
|
262
|
-
#[serde(default)]
|
|
263
|
-
pub dependencies: Vec<WorkloadDependency>,
|
|
264
|
-
/// The instant, in unix seconds, before which this workload must not be
|
|
265
|
-
/// claimed. Omitted or `None` means it is claimable as soon as it is
|
|
266
|
-
/// `Ready`, which is what every workload did before this field existed.
|
|
267
|
-
///
|
|
268
|
-
/// A workload created with `not_before` is durable immediately and
|
|
269
|
-
/// executable later: it reaches `Ready`, survives restart, and is refused to
|
|
270
|
-
/// every worker until the instant arrives. It is *not* a schedule — there is
|
|
271
|
-
/// no recurrence, no expression, and no component that re-arms it. Creating
|
|
272
|
-
/// the next occurrence is the caller's business.
|
|
273
|
-
///
|
|
274
|
-
/// A `not_before` already in the past is not an error. It simply describes a
|
|
275
|
-
/// workload that is eligible now, which is the same thing as omitting it.
|
|
276
|
-
#[serde(default)]
|
|
277
|
-
pub not_before: Option<i64>,
|
|
278
|
-
pub created_at: i64,
|
|
279
|
-
}
|
|
280
|
-
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
|
281
|
-
pub struct WorkloadError {
|
|
282
|
-
pub code: String,
|
|
283
|
-
pub message: String,
|
|
284
|
-
/// Structured detail for the caller, e.g. which request fields conflict.
|
|
285
|
-
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
286
|
-
pub details: Option<Value>,
|
|
287
|
-
}
|
|
288
|
-
impl WorkloadError {
|
|
289
|
-
fn new(code: &str, message: impl Into<String>) -> Self {
|
|
290
|
-
Self {
|
|
291
|
-
code: code.into(),
|
|
292
|
-
message: message.into(),
|
|
293
|
-
details: None,
|
|
294
|
-
}
|
|
295
|
-
}
|
|
296
|
-
}
|
|
297
|
-
impl std::fmt::Display for WorkloadError {
|
|
298
|
-
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
299
|
-
write!(f, "{}: {}", self.code, self.message)
|
|
300
|
-
}
|
|
301
|
-
}
|
|
302
|
-
impl std::error::Error for WorkloadError {}
|
|
303
|
-
|
|
304
|
-
#[derive(Clone)]
|
|
305
|
-
pub struct WorkloadStore {
|
|
306
|
-
db: FeltDb,
|
|
307
|
-
pub workloads: BTreeMap<String, Workload>,
|
|
308
|
-
idempotency: BTreeMap<String, String>,
|
|
309
|
-
result_ids: BTreeMap<String, String>,
|
|
310
|
-
/// Rebuildable cache of FeltDB-owned document versions. This is never
|
|
311
|
-
/// serialized and never advances independently of a canonical commit.
|
|
312
|
-
document_versions: BTreeMap<String, u64>,
|
|
313
|
-
/// The store revision this projection last reconciled with. A store that
|
|
314
|
-
/// has moved past it (a commit by another writer, or a replicated
|
|
315
|
-
/// operation) is re-read before the projection answers.
|
|
316
|
-
synced_revision: u64,
|
|
317
|
-
/// Always false outside tests, which set it to act on the projection
|
|
318
|
-
/// without re-reading the store: that reproduces another writer
|
|
319
|
-
/// committing between a refresh and a commit — the window only the storage
|
|
320
|
-
/// guard closes.
|
|
321
|
-
refresh_disabled: bool,
|
|
322
|
-
}
|
|
323
|
-
|
|
324
|
-
#[derive(Serialize, Deserialize)]
|
|
325
|
-
struct LegacyWorkloadStore {
|
|
326
|
-
#[serde(default)]
|
|
327
|
-
workloads: BTreeMap<String, Workload>,
|
|
328
|
-
#[serde(default)]
|
|
329
|
-
idempotency: BTreeMap<String, String>,
|
|
330
|
-
#[serde(default)]
|
|
331
|
-
result_ids: BTreeMap<String, String>,
|
|
332
|
-
#[serde(default)]
|
|
333
|
-
event_sequence: u64,
|
|
334
|
-
}
|
|
335
|
-
|
|
336
|
-
impl WorkloadStore {
|
|
337
|
-
pub fn load(path: impl AsRef<Path>) -> Result<Self, WorkloadError> {
|
|
338
|
-
Self::from_db(FeltDb::open(path).map_err(io)?)
|
|
339
|
-
}
|
|
340
|
-
|
|
341
|
-
pub fn from_db(db: FeltDb) -> Result<Self, WorkloadError> {
|
|
342
|
-
// Read before listing: a commit that lands in between is re-read by
|
|
343
|
-
// the next refresh rather than missed.
|
|
344
|
-
let synced_revision = db.current_revision().map_err(io)?;
|
|
345
|
-
let mut workloads = BTreeMap::new();
|
|
346
|
-
let mut document_versions = BTreeMap::new();
|
|
347
|
-
for row in db.list_collection(DURABLE_WORK_COLLECTION).map_err(io)? {
|
|
348
|
-
let version = row
|
|
349
|
-
.value
|
|
350
|
-
.get("__version")
|
|
351
|
-
.and_then(Value::as_u64)
|
|
352
|
-
// Canonical FeltDB CAS defines a pre-versioned document as
|
|
353
|
-
// version 1. The next guarded mutation writes version 2, which
|
|
354
|
-
// upgrades rows created before Work adopted RecordPrecondition.
|
|
355
|
-
.unwrap_or(1);
|
|
356
|
-
let workload: Workload = serde_json::from_value(row.value).map_err(io)?;
|
|
357
|
-
document_versions.insert(workload.workload_id.clone(), version);
|
|
358
|
-
workloads.insert(workload.workload_id.clone(), workload);
|
|
359
|
-
}
|
|
360
|
-
let idempotency = idempotency_index(&workloads, idempotency_scope);
|
|
361
|
-
let mut result_ids = BTreeMap::new();
|
|
362
|
-
for workload in workloads.values() {
|
|
363
|
-
if let Some(result) = &workload.result {
|
|
364
|
-
result_ids.insert(result.result_id.clone(), workload.workload_id.clone());
|
|
365
|
-
}
|
|
366
|
-
}
|
|
367
|
-
Ok(Self {
|
|
368
|
-
db,
|
|
369
|
-
workloads,
|
|
370
|
-
idempotency,
|
|
371
|
-
result_ids,
|
|
372
|
-
document_versions,
|
|
373
|
-
synced_revision,
|
|
374
|
-
refresh_disabled: false,
|
|
375
|
-
})
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
/// Reconcile the projection with the committed store.
|
|
379
|
-
///
|
|
380
|
-
/// The store, not this projection, is authoritative. When its revision has
|
|
381
|
-
/// moved since the last reconciliation, every workload row whose document
|
|
382
|
-
/// version differs from the projection's is re-read; rows are borrowed and
|
|
383
|
-
/// only changed ones are cloned. Commits by another writer of the store and
|
|
384
|
-
/// operations replicated from a peer both become visible here, without a
|
|
385
|
-
/// restart.
|
|
386
|
-
pub fn refresh(&mut self) -> Result<(), WorkloadError> {
|
|
387
|
-
if self.refresh_disabled {
|
|
388
|
-
return Ok(());
|
|
389
|
-
}
|
|
390
|
-
let revision = self.db.current_revision().map_err(io)?;
|
|
391
|
-
if revision == self.synced_revision {
|
|
392
|
-
return Ok(());
|
|
393
|
-
}
|
|
394
|
-
let known = &self.document_versions;
|
|
395
|
-
let changed = self
|
|
396
|
-
.db
|
|
397
|
-
.query_collection(DURABLE_WORK_COLLECTION, None, |row| {
|
|
398
|
-
let version = row
|
|
399
|
-
.value
|
|
400
|
-
.get("__version")
|
|
401
|
-
.and_then(Value::as_u64)
|
|
402
|
-
.unwrap_or(1);
|
|
403
|
-
let id = row.key.rsplit('/').next().unwrap_or_default();
|
|
404
|
-
known.get(id) != Some(&version)
|
|
405
|
-
})
|
|
406
|
-
.map_err(io)?;
|
|
407
|
-
for row in changed {
|
|
408
|
-
self.adopt(row)?;
|
|
409
|
-
}
|
|
410
|
-
self.synced_revision = revision;
|
|
411
|
-
Ok(())
|
|
412
|
-
}
|
|
413
|
-
|
|
414
|
-
/// Re-read one workload from the store when its committed version differs
|
|
415
|
-
/// from the projection's: a keyed read, not a scan. An id the projection
|
|
416
|
-
/// has never seen (admitted by another writer) falls back to [`Self::refresh`].
|
|
417
|
-
fn refresh_id(&mut self, id: &str) -> Result<(), WorkloadError> {
|
|
418
|
-
match self.workloads.get(id).map(work_key) {
|
|
419
|
-
Some(key) => self.refresh_key(&key).map(|_| ()),
|
|
420
|
-
None => self.refresh(),
|
|
421
|
-
}
|
|
422
|
-
}
|
|
423
|
-
|
|
424
|
-
fn refresh_key(&mut self, key: &str) -> Result<Option<Workload>, WorkloadError> {
|
|
425
|
-
if self.refresh_disabled {
|
|
426
|
-
let id = key.rsplit('/').next().unwrap_or_default();
|
|
427
|
-
return Ok(self.workloads.get(id).cloned());
|
|
428
|
-
}
|
|
429
|
-
let Some(row) = self.db.live_row(DURABLE_WORK_COLLECTION, key) else {
|
|
430
|
-
return Ok(None);
|
|
431
|
-
};
|
|
432
|
-
let version = row
|
|
433
|
-
.value
|
|
434
|
-
.get("__version")
|
|
435
|
-
.and_then(Value::as_u64)
|
|
436
|
-
.unwrap_or(1);
|
|
437
|
-
let id = key.rsplit('/').next().unwrap_or_default();
|
|
438
|
-
if self.document_versions.get(id) == Some(&version) {
|
|
439
|
-
return Ok(self.workloads.get(id).cloned());
|
|
440
|
-
}
|
|
441
|
-
self.adopt(row).map(Some)
|
|
442
|
-
}
|
|
443
|
-
|
|
444
|
-
/// Install one committed row in the projection and its derived indexes.
|
|
445
|
-
fn adopt(&mut self, row: StoredRow) -> Result<Workload, WorkloadError> {
|
|
446
|
-
let version = row
|
|
447
|
-
.value
|
|
448
|
-
.get("__version")
|
|
449
|
-
.and_then(Value::as_u64)
|
|
450
|
-
.unwrap_or(1);
|
|
451
|
-
let workload: Workload = serde_json::from_value(row.value).map_err(io)?;
|
|
452
|
-
if let Some(result) = &workload.result {
|
|
453
|
-
self.result_ids
|
|
454
|
-
.insert(result.result_id.clone(), workload.workload_id.clone());
|
|
455
|
-
}
|
|
456
|
-
self.idempotency.insert(
|
|
457
|
-
idempotency_scope(
|
|
458
|
-
&workload.tenant_id,
|
|
459
|
-
&workload.application_id,
|
|
460
|
-
&workload.requested_by,
|
|
461
|
-
&workload.idempotency_key,
|
|
462
|
-
),
|
|
463
|
-
workload.workload_id.clone(),
|
|
464
|
-
);
|
|
465
|
-
self.document_versions
|
|
466
|
-
.insert(workload.workload_id.clone(), version);
|
|
467
|
-
self.workloads
|
|
468
|
-
.insert(workload.workload_id.clone(), workload.clone());
|
|
469
|
-
Ok(workload)
|
|
470
|
-
}
|
|
471
|
-
|
|
472
|
-
/// A workload as committed in the store (see [`Self::refresh`]).
|
|
473
|
-
pub fn fresh(&mut self, id: &str) -> Result<&Workload, WorkloadError> {
|
|
474
|
-
self.refresh_id(id)?;
|
|
475
|
-
self.get(id)
|
|
476
|
-
}
|
|
477
|
-
|
|
478
|
-
/// An application's workloads as committed in the store.
|
|
479
|
-
pub fn fresh_list(
|
|
480
|
-
&mut self,
|
|
481
|
-
tenant: &str,
|
|
482
|
-
application: &str,
|
|
483
|
-
) -> Result<Vec<Workload>, WorkloadError> {
|
|
484
|
-
self.refresh()?;
|
|
485
|
-
Ok(self.list(tenant, application))
|
|
486
|
-
}
|
|
487
|
-
|
|
488
|
-
pub fn migrate_legacy_file(db: FeltDb, path: &Path) -> Result<Self, WorkloadError> {
|
|
489
|
-
let bytes = std::fs::read(path).map_err(io)?;
|
|
490
|
-
let legacy: LegacyWorkloadStore = serde_json::from_slice(&bytes).map_err(io)?;
|
|
491
|
-
let mut store = Self::from_db(db)?;
|
|
492
|
-
for (id, workload) in legacy.workloads {
|
|
493
|
-
if id != workload.workload_id {
|
|
494
|
-
return Err(WorkloadError::new(
|
|
495
|
-
"WORKLOAD_MIGRATION_CONFLICT",
|
|
496
|
-
format!(
|
|
497
|
-
"legacy key {id} does not match workload identity {}",
|
|
498
|
-
workload.workload_id
|
|
499
|
-
),
|
|
500
|
-
));
|
|
501
|
-
}
|
|
502
|
-
if let Some(existing) = store.workloads.get(&id) {
|
|
503
|
-
if serde_json::to_value(existing).map_err(io)?
|
|
504
|
-
!= serde_json::to_value(&workload).map_err(io)?
|
|
505
|
-
{
|
|
506
|
-
return Err(WorkloadError::new(
|
|
507
|
-
"WORKLOAD_MIGRATION_CONFLICT",
|
|
508
|
-
format!("canonical workload {id} differs from legacy source"),
|
|
509
|
-
));
|
|
510
|
-
}
|
|
511
|
-
continue;
|
|
512
|
-
}
|
|
513
|
-
store.workloads.insert(id.clone(), workload);
|
|
514
|
-
store.persist_workload(&id, "workload_migration")?;
|
|
515
|
-
}
|
|
516
|
-
// The maps are verification evidence, not migrated authorities.
|
|
517
|
-
let rebuilt = Self::from_db(store.db.clone())?;
|
|
518
|
-
if legacy.idempotency != idempotency_index(&rebuilt.workloads, legacy_idempotency_scope)
|
|
519
|
-
|| legacy.result_ids != rebuilt.result_ids
|
|
520
|
-
|| legacy.event_sequence != derived_event_sequence(&rebuilt.workloads)
|
|
521
|
-
{
|
|
522
|
-
return Err(WorkloadError::new(
|
|
523
|
-
"WORKLOAD_MIGRATION_CONFLICT",
|
|
524
|
-
"legacy derived indexes do not match canonical work aggregates",
|
|
525
|
-
));
|
|
526
|
-
}
|
|
527
|
-
Ok(rebuilt)
|
|
528
|
-
}
|
|
529
|
-
|
|
530
|
-
fn persist_workload(&mut self, id: &str, action: &str) -> Result<(), WorkloadError> {
|
|
531
|
-
let workload = self
|
|
532
|
-
.workloads
|
|
533
|
-
.get(id)
|
|
534
|
-
.ok_or_else(|| WorkloadError::new("WORKLOAD_NOT_FOUND", "workload not found"))?;
|
|
535
|
-
let key = work_key(workload);
|
|
536
|
-
let expected_version = self.document_versions.get(id).copied();
|
|
537
|
-
let transaction_id = runtime_id("work_tx");
|
|
538
|
-
let aggregate_digest = hash(&serde_json::to_value(workload).map_err(io)?);
|
|
539
|
-
let commit = self
|
|
540
|
-
.db
|
|
541
|
-
.apply_atomic_transaction_guarded(
|
|
542
|
-
&transaction_id,
|
|
543
|
-
None,
|
|
544
|
-
None,
|
|
545
|
-
&[],
|
|
546
|
-
&[RecordPrecondition {
|
|
547
|
-
capability: DURABLE_WORK_COLLECTION.into(),
|
|
548
|
-
key: key.clone(),
|
|
549
|
-
require_absent: expected_version.is_none(),
|
|
550
|
-
expected_version,
|
|
551
|
-
expected_epoch: None,
|
|
552
|
-
expected_lease_id: None,
|
|
553
|
-
}],
|
|
554
|
-
&[AtomicMutation {
|
|
555
|
-
capability: DURABLE_WORK_COLLECTION.into(),
|
|
556
|
-
key,
|
|
557
|
-
value: Some(serde_json::to_value(workload).map_err(io)?),
|
|
558
|
-
}],
|
|
559
|
-
Some(serde_json::json!({
|
|
560
|
-
"kind": action,
|
|
561
|
-
"subject": workload.requested_by.id(),
|
|
562
|
-
"principal_id": workload.requested_by.id(),
|
|
563
|
-
"agent_id": (workload.requested_by.kind() == "agent").then(|| workload.requested_by.id()),
|
|
564
|
-
"session_id": workload.session_id,
|
|
565
|
-
"delegator_id": workload.authorization_snapshot.grant.issuer.id(),
|
|
566
|
-
"authorization_decision_id": workload.authorization_snapshot.grant.id,
|
|
567
|
-
"tenant": workload.tenant_id,
|
|
568
|
-
"application": workload.application_id,
|
|
569
|
-
"work_id": workload.workload_id,
|
|
570
|
-
"operation_id": transaction_id,
|
|
571
|
-
"attempt_id": workload.attempts.last().map(|attempt| attempt.attempt_id.as_str()),
|
|
572
|
-
"worker_id": workload.current_claim.as_ref().map(|claim| claim.worker_id.as_str()),
|
|
573
|
-
"result_id": workload.result.as_ref().map(|result| result.result_id.as_str()),
|
|
574
|
-
"source_kind": (action == "workload_migration").then_some("WorkloadStore"),
|
|
575
|
-
"source_digest": aggregate_digest,
|
|
576
|
-
"migration_outcome": (action == "workload_migration").then_some("imported"),
|
|
577
|
-
})),
|
|
578
|
-
)
|
|
579
|
-
.map_err(workload_storage_error)?;
|
|
580
|
-
let version = commit
|
|
581
|
-
.rows
|
|
582
|
-
.first()
|
|
583
|
-
.and_then(|row| row.value.get("__version"))
|
|
584
|
-
.and_then(Value::as_u64)
|
|
585
|
-
.ok_or_else(|| io("canonical work commit has no document version"))?;
|
|
586
|
-
self.document_versions.insert(id.into(), version);
|
|
587
|
-
Ok(())
|
|
588
|
-
}
|
|
589
|
-
pub fn create(&mut self, input: CreateWorkload) -> Result<Workload, WorkloadError> {
|
|
590
|
-
self.admit(input).map(|(workload, _)| workload)
|
|
591
|
-
}
|
|
592
|
-
|
|
593
|
-
/// Create a workload, or return the one this requester already created
|
|
594
|
-
/// under the same key.
|
|
595
|
-
///
|
|
596
|
-
/// The key is scoped to the tenant, the application **and the requester**,
|
|
597
|
-
/// so one requester's key can never return, reveal or block another's
|
|
598
|
-
/// workload. A repeated key is answered with the original workload only when
|
|
599
|
-
/// the request is the same — definition, revision, environment, input and
|
|
600
|
-
/// capability snapshot; otherwise it is refused with
|
|
601
|
-
/// `IDEMPOTENCY_KEY_REUSED` and nothing is created. The requester's
|
|
602
|
-
/// authorization snapshot is checked before any replay. The boolean is
|
|
603
|
-
/// `true` when this call created the workload.
|
|
604
|
-
pub fn admit(&mut self, input: CreateWorkload) -> Result<(Workload, bool), WorkloadError> {
|
|
605
|
-
ensure_safe(&input.input)?;
|
|
606
|
-
if input.idempotency_key.trim().is_empty() {
|
|
607
|
-
return Err(WorkloadError::new(
|
|
608
|
-
"WORKLOAD_INVALID",
|
|
609
|
-
"idempotency key is required",
|
|
610
|
-
));
|
|
611
|
-
}
|
|
612
|
-
let t = &input.timeouts;
|
|
613
|
-
if [
|
|
614
|
-
t.queue_ms,
|
|
615
|
-
t.claim_ms,
|
|
616
|
-
t.execution_ms,
|
|
617
|
-
t.heartbeat_ms,
|
|
618
|
-
t.overall_ms,
|
|
619
|
-
]
|
|
620
|
-
.contains(&Some(0))
|
|
621
|
-
{
|
|
622
|
-
return Err(WorkloadError::new(
|
|
623
|
-
"WORKLOAD_INVALID",
|
|
624
|
-
"a timeout must be at least 1 ms; omit it for no deadline",
|
|
625
|
-
));
|
|
626
|
-
}
|
|
627
|
-
if input.authorization_snapshot.grant.subject != input.requested_by
|
|
628
|
-
|| input.authorization_snapshot.grant.tenant_id != input.tenant_id
|
|
629
|
-
|| input.authorization_snapshot.grant.application_id != input.application_id
|
|
630
|
-
{
|
|
631
|
-
return Err(WorkloadError::new(
|
|
632
|
-
"WORKLOAD_UNAUTHORIZED",
|
|
633
|
-
"authorization snapshot does not bind the requester",
|
|
634
|
-
));
|
|
635
|
-
}
|
|
636
|
-
let idempotency = idempotency_scope(
|
|
637
|
-
&input.tenant_id,
|
|
638
|
-
&input.application_id,
|
|
639
|
-
&input.requested_by,
|
|
640
|
-
&input.idempotency_key,
|
|
641
|
-
);
|
|
642
|
-
// The scoped idempotency identity is the record identity. Competing
|
|
643
|
-
// admissions therefore contend on one canonical require-absent CAS
|
|
644
|
-
// even when their payloads differ; no separate durable uniqueness
|
|
645
|
-
// index is needed.
|
|
646
|
-
let id = format!("workload_{}", &hex(&Sha256::digest(&idempotency))[..24]);
|
|
647
|
-
// Another writer, or a peer, may have admitted this identity: the
|
|
648
|
-
// store is asked by key, not the projection.
|
|
649
|
-
let committed_key = format!("{}/{}/{}", input.tenant_id, input.application_id, id);
|
|
650
|
-
if let Some(existing) = self.refresh_key(&committed_key)? {
|
|
651
|
-
return replay(existing, &input);
|
|
652
|
-
}
|
|
653
|
-
if let Some(existing_id) = self.idempotency.get(&idempotency) {
|
|
654
|
-
let existing = self.workloads[existing_id].clone();
|
|
655
|
-
return replay(existing, &input);
|
|
656
|
-
}
|
|
657
|
-
let request = input.clone();
|
|
658
|
-
let input_hash = hash(&input.input);
|
|
659
|
-
let mut workload = Workload {
|
|
660
|
-
contract_version: DEADLINE_CONTRACT_VERSION,
|
|
661
|
-
workload_id: id.clone(),
|
|
662
|
-
tenant_id: input.tenant_id,
|
|
663
|
-
application_id: input.application_id,
|
|
664
|
-
environment: input.environment,
|
|
665
|
-
definition_id: input.definition_id,
|
|
666
|
-
definition_revision: input.definition_revision,
|
|
667
|
-
trigger: input.trigger,
|
|
668
|
-
input: input.input,
|
|
669
|
-
input_hash,
|
|
670
|
-
requested_by: input.requested_by,
|
|
671
|
-
session_id: input.session_id,
|
|
672
|
-
authorization_snapshot: input.authorization_snapshot,
|
|
673
|
-
capability_snapshot: input.capability_snapshot,
|
|
674
|
-
connection_scopes: input.connection_scopes,
|
|
675
|
-
priority: input.priority,
|
|
676
|
-
idempotency_key: input.idempotency_key,
|
|
677
|
-
created_at: input.created_at,
|
|
678
|
-
updated_at: input.created_at,
|
|
679
|
-
state: WorkloadState::Created,
|
|
680
|
-
execution_history: vec![],
|
|
681
|
-
attempts: vec![],
|
|
682
|
-
checkpoints: vec![],
|
|
683
|
-
result: None,
|
|
684
|
-
retry_policy: input.retry_policy,
|
|
685
|
-
timeouts: input.timeouts,
|
|
686
|
-
attempt_count: 0,
|
|
687
|
-
retry_count: 0,
|
|
688
|
-
// Creation-time eligibility and retry backoff are the same thing, so
|
|
689
|
-
// they are the same field. A workload with `not_before` is created
|
|
690
|
-
// exactly as any other and is simply not yet claimable.
|
|
691
|
-
next_attempt_at: input.not_before,
|
|
692
|
-
last_failure: None,
|
|
693
|
-
current_claim: None,
|
|
694
|
-
fencing_token: 0,
|
|
695
|
-
dependencies: input.dependencies,
|
|
696
|
-
};
|
|
697
|
-
// The durable history records the eligibility the workload was created
|
|
698
|
-
// with, so "why was this Ready and unclaimed for an hour" is answerable
|
|
699
|
-
// from the workload itself rather than from the caller's intent.
|
|
700
|
-
let evidence = match workload.next_attempt_at {
|
|
701
|
-
Some(instant) => [("not_before".into(), instant.to_string())].into(),
|
|
702
|
-
None => BTreeMap::new(),
|
|
703
|
-
};
|
|
704
|
-
self.transition(
|
|
705
|
-
&mut workload,
|
|
706
|
-
WorkloadState::Ready,
|
|
707
|
-
"rust",
|
|
708
|
-
input.created_at,
|
|
709
|
-
evidence,
|
|
710
|
-
)?;
|
|
711
|
-
let key = work_key(&workload);
|
|
712
|
-
self.idempotency.insert(idempotency.clone(), id.clone());
|
|
713
|
-
self.workloads.insert(id.clone(), workload.clone());
|
|
714
|
-
if let Err(error) = self.persist_workload(&id, "workload.created") {
|
|
715
|
-
self.workloads.remove(&id);
|
|
716
|
-
self.idempotency.remove(&idempotency);
|
|
717
|
-
// Another writer of the same store (a second process, or a replica)
|
|
718
|
-
// admitted this identity first. Converge on what it committed.
|
|
719
|
-
if error.code == "WORKLOAD_CONFLICT" {
|
|
720
|
-
if let Some(existing) = self.reload_committed(&key)? {
|
|
721
|
-
self.idempotency.insert(idempotency, id);
|
|
722
|
-
return replay(existing, &request);
|
|
723
|
-
}
|
|
724
|
-
}
|
|
725
|
-
return Err(error);
|
|
726
|
-
}
|
|
727
|
-
Ok((workload, true))
|
|
728
|
-
}
|
|
729
|
-
pub fn get(&self, id: &str) -> Result<&Workload, WorkloadError> {
|
|
730
|
-
self.workloads
|
|
731
|
-
.get(id)
|
|
732
|
-
.ok_or_else(|| WorkloadError::new("WORKLOAD_NOT_FOUND", "workload not found"))
|
|
733
|
-
}
|
|
734
|
-
pub fn list(&self, tenant: &str, application: &str) -> Vec<Workload> {
|
|
735
|
-
let mut values = self
|
|
736
|
-
.workloads
|
|
737
|
-
.values()
|
|
738
|
-
.filter(|v| v.tenant_id == tenant && v.application_id == application)
|
|
739
|
-
.cloned()
|
|
740
|
-
.collect::<Vec<_>>();
|
|
741
|
-
values.sort_by_key(|v| (std::cmp::Reverse(v.priority), v.created_at));
|
|
742
|
-
values
|
|
743
|
-
}
|
|
744
|
-
fn edit<R>(
|
|
745
|
-
&mut self,
|
|
746
|
-
id: &str,
|
|
747
|
-
action: impl FnOnce(&mut Self, &mut Workload) -> Result<R, WorkloadError>,
|
|
748
|
-
) -> Result<R, WorkloadError> {
|
|
749
|
-
// Act only on the committed version: a projection that missed another
|
|
750
|
-
// writer's commit would otherwise lose its guarded commit at best.
|
|
751
|
-
self.refresh_id(id)?;
|
|
752
|
-
let workload = self
|
|
753
|
-
.workloads
|
|
754
|
-
.get(id)
|
|
755
|
-
.cloned()
|
|
756
|
-
.ok_or_else(|| WorkloadError::new("WORKLOAD_NOT_FOUND", "workload not found"))?;
|
|
757
|
-
let mut edited = workload.clone();
|
|
758
|
-
let result = action(self, &mut edited)?;
|
|
759
|
-
self.workloads.insert(id.into(), edited);
|
|
760
|
-
if let Err(error) = self.persist_workload(id, "workload.mutated") {
|
|
761
|
-
self.workloads.insert(id.into(), workload.clone());
|
|
762
|
-
// Another writer of the same store committed this workload first.
|
|
763
|
-
// Nothing of this edit was committed; adopt what was, so this
|
|
764
|
-
// projection does not keep acting on a version that no longer
|
|
765
|
-
// exists (every later guarded commit from it would conflict).
|
|
766
|
-
if error.code == "WORKLOAD_CONFLICT" {
|
|
767
|
-
self.reload_committed(&work_key(&workload))?;
|
|
768
|
-
}
|
|
769
|
-
return Err(error);
|
|
770
|
-
}
|
|
771
|
-
Ok(result)
|
|
772
|
-
}
|
|
773
|
-
#[cfg(test)]
|
|
774
|
-
pub(crate) fn db_for_tests(&self) -> FeltDb {
|
|
775
|
-
self.db.clone()
|
|
776
|
-
}
|
|
777
|
-
/// See `refresh_disabled`.
|
|
778
|
-
#[cfg(test)]
|
|
779
|
-
pub(crate) fn without_refresh(mut self) -> Self {
|
|
780
|
-
self.refresh_disabled = true;
|
|
781
|
-
self
|
|
782
|
-
}
|
|
783
|
-
/// Replace this projection's copy of a workload with the committed row.
|
|
784
|
-
fn reload_committed(&mut self, key: &str) -> Result<Option<Workload>, WorkloadError> {
|
|
785
|
-
match self.db.live_row(DURABLE_WORK_COLLECTION, key) {
|
|
786
|
-
Some(row) => self.adopt(row).map(Some),
|
|
787
|
-
None => Ok(None),
|
|
788
|
-
}
|
|
789
|
-
}
|
|
790
|
-
fn transition(
|
|
791
|
-
&mut self,
|
|
792
|
-
w: &mut Workload,
|
|
793
|
-
to: WorkloadState,
|
|
794
|
-
actor: &str,
|
|
795
|
-
at: i64,
|
|
796
|
-
evidence: BTreeMap<String, String>,
|
|
797
|
-
) -> Result<(), WorkloadError> {
|
|
798
|
-
if !legal(w.state, to) {
|
|
799
|
-
return Err(WorkloadError::new(
|
|
800
|
-
"WORKLOAD_TRANSITION_DENIED",
|
|
801
|
-
format!("{:?} cannot transition to {:?}", w.state, to),
|
|
802
|
-
));
|
|
803
|
-
}
|
|
804
|
-
let from = w.state;
|
|
805
|
-
w.state = to;
|
|
806
|
-
w.updated_at = at;
|
|
807
|
-
let sequence = w
|
|
808
|
-
.execution_history
|
|
809
|
-
.last()
|
|
810
|
-
.map_or(1, |event| event.sequence + 1);
|
|
811
|
-
w.execution_history.push(WorkloadEvent {
|
|
812
|
-
sequence,
|
|
813
|
-
event_type: event_name(to).into(),
|
|
814
|
-
from: Some(from),
|
|
815
|
-
to,
|
|
816
|
-
actor: actor.into(),
|
|
817
|
-
at,
|
|
818
|
-
evidence,
|
|
819
|
-
});
|
|
820
|
-
Ok(())
|
|
821
|
-
}
|
|
822
|
-
pub fn claim(
|
|
823
|
-
&mut self,
|
|
824
|
-
id: &str,
|
|
825
|
-
worker: &str,
|
|
826
|
-
capabilities: &BTreeSet<String>,
|
|
827
|
-
now: i64,
|
|
828
|
-
lease_ms: u64,
|
|
829
|
-
) -> Result<WorkloadClaim, WorkloadError> {
|
|
830
|
-
self.enforce_deadline(id, now)?;
|
|
831
|
-
// A Ready workload that has already used its budget (an operator
|
|
832
|
-
// re-arm beyond it, or a record left by unbounded recovery before it
|
|
833
|
-
// was bounded) is dead-lettered here rather than claimed again.
|
|
834
|
-
let exhausted = self
|
|
835
|
-
.get(id)
|
|
836
|
-
.is_ok_and(|w| w.state == WorkloadState::Ready && w.attempt_count >= attempt_budget(w));
|
|
837
|
-
if exhausted {
|
|
838
|
-
self.edit(id, |store, w| {
|
|
839
|
-
let evidence = [
|
|
840
|
-
("reason".into(), "attempt_budget_exhausted".into()),
|
|
841
|
-
("attempts_used".into(), w.attempt_count.to_string()),
|
|
842
|
-
("attempt_budget".into(), attempt_budget(w).to_string()),
|
|
843
|
-
]
|
|
844
|
-
.into();
|
|
845
|
-
store.transition(w, WorkloadState::DeadLetter, "rust.claim", now, evidence)
|
|
846
|
-
})?;
|
|
847
|
-
return Err(WorkloadError::new(
|
|
848
|
-
"WORKLOAD_ATTEMPTS_EXHAUSTED",
|
|
849
|
-
"workload has used its attempt budget",
|
|
850
|
-
));
|
|
851
|
-
}
|
|
852
|
-
self.edit(id, |store, w| {
|
|
853
|
-
if w.state != WorkloadState::Ready {
|
|
854
|
-
return Err(WorkloadError::new(
|
|
855
|
-
"WORKLOAD_NOT_READY",
|
|
856
|
-
"workload is not eligible",
|
|
857
|
-
));
|
|
858
|
-
}
|
|
859
|
-
// Separated from the state check because the two mean opposite
|
|
860
|
-
// things to a worker: a workload in the wrong state may never become
|
|
861
|
-
// claimable, while one that is merely early certainly will, at an
|
|
862
|
-
// instant the error names. Collapsing them into one code left a
|
|
863
|
-
// polling worker unable to tell "come back" from "give up".
|
|
864
|
-
if !w.is_eligible_at(now) {
|
|
865
|
-
return Err(WorkloadError::new(
|
|
866
|
-
"WORKLOAD_NOT_YET_ELIGIBLE",
|
|
867
|
-
format!(
|
|
868
|
-
"workload is not claimable until {}",
|
|
869
|
-
w.next_attempt_at.unwrap_or(now)
|
|
870
|
-
),
|
|
871
|
-
));
|
|
872
|
-
}
|
|
873
|
-
if !w.capability_snapshot.is_subset(capabilities) {
|
|
874
|
-
return Err(WorkloadError::new(
|
|
875
|
-
"WORKLOAD_CAPABILITY_DENIED",
|
|
876
|
-
"worker lacks required capability snapshot",
|
|
877
|
-
));
|
|
878
|
-
}
|
|
879
|
-
w.fencing_token += 1;
|
|
880
|
-
w.attempt_count += 1;
|
|
881
|
-
let seed = format!("{}:{}:{}", w.workload_id, w.fencing_token, worker);
|
|
882
|
-
let claim = WorkloadClaim {
|
|
883
|
-
claim_id: format!("claim_{}", &hex(&Sha256::digest(seed.as_bytes()))[..20]),
|
|
884
|
-
workload_id: w.workload_id.clone(),
|
|
885
|
-
worker_id: worker.into(),
|
|
886
|
-
lease_id: format!("lease_{}", w.fencing_token),
|
|
887
|
-
fencing_token: w.fencing_token,
|
|
888
|
-
capability_snapshot: w.capability_snapshot.clone(),
|
|
889
|
-
claimed_at: now,
|
|
890
|
-
expires_at: now + (lease_ms / 1000) as i64,
|
|
891
|
-
last_heartbeat: now,
|
|
892
|
-
};
|
|
893
|
-
w.attempts.push(WorkloadAttempt {
|
|
894
|
-
attempt_id: format!("attempt_{}_{}", w.workload_id, w.attempt_count),
|
|
895
|
-
workload_id: w.workload_id.clone(),
|
|
896
|
-
worker_id: worker.into(),
|
|
897
|
-
claim_id: claim.claim_id.clone(),
|
|
898
|
-
fencing_token: claim.fencing_token,
|
|
899
|
-
started_at: None,
|
|
900
|
-
ended_at: None,
|
|
901
|
-
input_hash: w.input_hash.clone(),
|
|
902
|
-
result_hash: None,
|
|
903
|
-
error: None,
|
|
904
|
-
resource_usage: BTreeMap::new(),
|
|
905
|
-
status: AttemptStatus::Claimed,
|
|
906
|
-
});
|
|
907
|
-
w.current_claim = Some(claim.clone());
|
|
908
|
-
store.transition(
|
|
909
|
-
w,
|
|
910
|
-
WorkloadState::Claimed,
|
|
911
|
-
worker,
|
|
912
|
-
now,
|
|
913
|
-
[("fencing_token".into(), claim.fencing_token.to_string())].into(),
|
|
914
|
-
)?;
|
|
915
|
-
Ok(claim)
|
|
916
|
-
})
|
|
917
|
-
}
|
|
918
|
-
fn fence(
|
|
919
|
-
w: &Workload,
|
|
920
|
-
claim: &str,
|
|
921
|
-
worker: &str,
|
|
922
|
-
token: u64,
|
|
923
|
-
now: i64,
|
|
924
|
-
) -> Result<(), WorkloadError> {
|
|
925
|
-
let current = w
|
|
926
|
-
.current_claim
|
|
927
|
-
.as_ref()
|
|
928
|
-
.ok_or_else(|| WorkloadError::new("WORKLOAD_CLAIM_INVALID", "no active claim"))?;
|
|
929
|
-
if current.claim_id != claim
|
|
930
|
-
|| current.worker_id != worker
|
|
931
|
-
|| current.fencing_token != token
|
|
932
|
-
{
|
|
933
|
-
return Err(WorkloadError::new(
|
|
934
|
-
"WORKLOAD_STALE_FENCE",
|
|
935
|
-
"claim or fencing token is stale",
|
|
936
|
-
));
|
|
937
|
-
}
|
|
938
|
-
if current.expires_at <= now {
|
|
939
|
-
return Err(WorkloadError::new(
|
|
940
|
-
"WORKLOAD_LEASE_EXPIRED",
|
|
941
|
-
"claim lease expired",
|
|
942
|
-
));
|
|
943
|
-
}
|
|
944
|
-
Ok(())
|
|
945
|
-
}
|
|
946
|
-
pub fn start(
|
|
947
|
-
&mut self,
|
|
948
|
-
id: &str,
|
|
949
|
-
claim: &str,
|
|
950
|
-
worker: &str,
|
|
951
|
-
token: u64,
|
|
952
|
-
now: i64,
|
|
953
|
-
) -> Result<(), WorkloadError> {
|
|
954
|
-
self.enforce_deadline(id, now)?;
|
|
955
|
-
self.edit(id, |store, w| {
|
|
956
|
-
Self::fence(w, claim, worker, token, now)?;
|
|
957
|
-
let attempt = w.attempts.last_mut().unwrap();
|
|
958
|
-
attempt.status = AttemptStatus::Running;
|
|
959
|
-
attempt.started_at = Some(now);
|
|
960
|
-
store.transition(w, WorkloadState::Running, worker, now, BTreeMap::new())
|
|
961
|
-
})
|
|
962
|
-
}
|
|
963
|
-
pub fn heartbeat(
|
|
964
|
-
&mut self,
|
|
965
|
-
id: &str,
|
|
966
|
-
claim: &str,
|
|
967
|
-
worker: &str,
|
|
968
|
-
token: u64,
|
|
969
|
-
now: i64,
|
|
970
|
-
extend_ms: u64,
|
|
971
|
-
) -> Result<WorkloadClaim, WorkloadError> {
|
|
972
|
-
self.enforce_deadline(id, now)?;
|
|
973
|
-
self.edit(id, |_, w| {
|
|
974
|
-
Self::fence(w, claim, worker, token, now)?;
|
|
975
|
-
let current = w.current_claim.as_mut().unwrap();
|
|
976
|
-
current.last_heartbeat = now;
|
|
977
|
-
current.expires_at = now + (extend_ms / 1000) as i64;
|
|
978
|
-
Ok(current.clone())
|
|
979
|
-
})
|
|
980
|
-
}
|
|
981
|
-
pub fn execution_context(
|
|
982
|
-
&mut self,
|
|
983
|
-
id: &str,
|
|
984
|
-
claim: &str,
|
|
985
|
-
worker: &str,
|
|
986
|
-
token: u64,
|
|
987
|
-
now: i64,
|
|
988
|
-
secret_references: Vec<String>,
|
|
989
|
-
) -> Result<WorkloadExecutionContext, WorkloadError> {
|
|
990
|
-
self.enforce_deadline(id, now)?;
|
|
991
|
-
let workload = self.fresh(id)?;
|
|
992
|
-
Self::fence(workload, claim, worker, token, now)?;
|
|
993
|
-
if secret_references
|
|
994
|
-
.iter()
|
|
995
|
-
.any(|v| !v.starts_with("secret://"))
|
|
996
|
-
{
|
|
997
|
-
return Err(WorkloadError::new(
|
|
998
|
-
"WORKLOAD_SECRET_REJECTED",
|
|
999
|
-
"execution context accepts secret references only",
|
|
1000
|
-
));
|
|
1001
|
-
}
|
|
1002
|
-
let current = workload.current_claim.as_ref().unwrap();
|
|
1003
|
-
Ok(WorkloadExecutionContext {
|
|
1004
|
-
workload_id: id.into(),
|
|
1005
|
-
attempt_id: workload.attempts.last().unwrap().attempt_id.clone(),
|
|
1006
|
-
worker_id: worker.into(),
|
|
1007
|
-
capabilities: current.capability_snapshot.clone(),
|
|
1008
|
-
connection_scopes: workload.connection_scopes.clone(),
|
|
1009
|
-
secret_references,
|
|
1010
|
-
expires_at: current.expires_at,
|
|
1011
|
-
fencing_token: token,
|
|
1012
|
-
input: workload.input.clone(),
|
|
1013
|
-
checkpoints: workload.checkpoints.clone(),
|
|
1014
|
-
})
|
|
1015
|
-
}
|
|
1016
|
-
pub fn checkpoint(
|
|
1017
|
-
&mut self,
|
|
1018
|
-
id: &str,
|
|
1019
|
-
claim: &str,
|
|
1020
|
-
worker: &str,
|
|
1021
|
-
token: u64,
|
|
1022
|
-
state: Value,
|
|
1023
|
-
now: i64,
|
|
1024
|
-
) -> Result<WorkloadCheckpoint, WorkloadError> {
|
|
1025
|
-
ensure_safe(&state)?;
|
|
1026
|
-
self.enforce_deadline(id, now)?;
|
|
1027
|
-
self.edit(id, |store, w| {
|
|
1028
|
-
Self::fence(w, claim, worker, token, now)?;
|
|
1029
|
-
if !matches!(
|
|
1030
|
-
w.state,
|
|
1031
|
-
WorkloadState::Running | WorkloadState::Checkpointed
|
|
1032
|
-
) {
|
|
1033
|
-
return Err(WorkloadError::new(
|
|
1034
|
-
"WORKLOAD_TRANSITION_DENIED",
|
|
1035
|
-
"checkpoint requires a running workload",
|
|
1036
|
-
));
|
|
1037
|
-
}
|
|
1038
|
-
let attempt = w.attempts.last_mut().unwrap();
|
|
1039
|
-
attempt.status = AttemptStatus::Checkpointed;
|
|
1040
|
-
let sequence = w.checkpoints.len() as u64 + 1;
|
|
1041
|
-
let checkpoint = WorkloadCheckpoint {
|
|
1042
|
-
checkpoint_id: format!("checkpoint_{}_{}", w.workload_id, sequence),
|
|
1043
|
-
workload_id: w.workload_id.clone(),
|
|
1044
|
-
attempt_id: attempt.attempt_id.clone(),
|
|
1045
|
-
sequence,
|
|
1046
|
-
state_hash: hash(&state),
|
|
1047
|
-
state,
|
|
1048
|
-
created_at: now,
|
|
1049
|
-
produced_by: worker.into(),
|
|
1050
|
-
};
|
|
1051
|
-
w.checkpoints.push(checkpoint.clone());
|
|
1052
|
-
if w.state == WorkloadState::Running {
|
|
1053
|
-
store.transition(w, WorkloadState::Checkpointed, worker, now, BTreeMap::new())?;
|
|
1054
|
-
}
|
|
1055
|
-
Ok(checkpoint)
|
|
1056
|
-
})
|
|
1057
|
-
}
|
|
1058
|
-
pub fn complete(
|
|
1059
|
-
&mut self,
|
|
1060
|
-
id: &str,
|
|
1061
|
-
claim: &str,
|
|
1062
|
-
worker: &str,
|
|
1063
|
-
token: u64,
|
|
1064
|
-
result_id: &str,
|
|
1065
|
-
output: Value,
|
|
1066
|
-
artifacts: Vec<String>,
|
|
1067
|
-
schema_version: u64,
|
|
1068
|
-
now: i64,
|
|
1069
|
-
) -> Result<WorkloadResult, WorkloadError> {
|
|
1070
|
-
ensure_safe(&output)?;
|
|
1071
|
-
if let Some(workload_id) = self.result_ids.get(result_id) {
|
|
1072
|
-
if workload_id == id {
|
|
1073
|
-
// A replay is the reporting attempt retrying the same report
|
|
1074
|
-
// (e.g. after a lost response). Anything else naming this
|
|
1075
|
-
// result id did not produce it and must not be told it landed.
|
|
1076
|
-
let result = self.get(id)?.result.clone().unwrap();
|
|
1077
|
-
let same_report = result.produced_by == worker
|
|
1078
|
-
&& result.provenance.get("claim_id").map(String::as_str) == Some(claim)
|
|
1079
|
-
&& result.provenance.get("fencing_token") == Some(&token.to_string())
|
|
1080
|
-
&& result.output_hash == hash(&output);
|
|
1081
|
-
if same_report {
|
|
1082
|
-
return Ok(result);
|
|
1083
|
-
}
|
|
1084
|
-
return Err(WorkloadError::new(
|
|
1085
|
-
"WORKLOAD_RESULT_CONFLICT",
|
|
1086
|
-
"the result id was already recorded by a different report",
|
|
1087
|
-
));
|
|
1088
|
-
}
|
|
1089
|
-
return Err(WorkloadError::new(
|
|
1090
|
-
"WORKLOAD_RESULT_DUPLICATE",
|
|
1091
|
-
"result id belongs to another workload",
|
|
1092
|
-
));
|
|
1093
|
-
}
|
|
1094
|
-
self.enforce_deadline(id, now)?;
|
|
1095
|
-
let result = self.edit(id, |store, w| {
|
|
1096
|
-
Self::fence(w, claim, worker, token, now)?;
|
|
1097
|
-
if !matches!(
|
|
1098
|
-
w.state,
|
|
1099
|
-
WorkloadState::Running | WorkloadState::Checkpointed
|
|
1100
|
-
) {
|
|
1101
|
-
return Err(WorkloadError::new(
|
|
1102
|
-
"WORKLOAD_TRANSITION_DENIED",
|
|
1103
|
-
"completion requires running state",
|
|
1104
|
-
));
|
|
1105
|
-
}
|
|
1106
|
-
let attempt = w.attempts.last_mut().unwrap();
|
|
1107
|
-
let output_hash = hash(&output);
|
|
1108
|
-
attempt.status = AttemptStatus::Succeeded;
|
|
1109
|
-
attempt.ended_at = Some(now);
|
|
1110
|
-
attempt.result_hash = Some(output_hash.clone());
|
|
1111
|
-
let result = WorkloadResult {
|
|
1112
|
-
result_id: result_id.into(),
|
|
1113
|
-
workload_id: w.workload_id.clone(),
|
|
1114
|
-
attempt_id: attempt.attempt_id.clone(),
|
|
1115
|
-
status: "succeeded".into(),
|
|
1116
|
-
output,
|
|
1117
|
-
output_hash,
|
|
1118
|
-
artifacts,
|
|
1119
|
-
provenance: [
|
|
1120
|
-
("claim_id".into(), claim.into()),
|
|
1121
|
-
("fencing_token".into(), token.to_string()),
|
|
1122
|
-
]
|
|
1123
|
-
.into(),
|
|
1124
|
-
produced_by: worker.into(),
|
|
1125
|
-
created_at: now,
|
|
1126
|
-
schema_version,
|
|
1127
|
-
};
|
|
1128
|
-
w.result = Some(result.clone());
|
|
1129
|
-
w.current_claim = None;
|
|
1130
|
-
store.transition(w, WorkloadState::Succeeded, worker, now, BTreeMap::new())?;
|
|
1131
|
-
Ok(result)
|
|
1132
|
-
})?;
|
|
1133
|
-
self.result_ids.insert(result_id.into(), id.into());
|
|
1134
|
-
Ok(result)
|
|
1135
|
-
}
|
|
1136
|
-
pub fn fail(
|
|
1137
|
-
&mut self,
|
|
1138
|
-
id: &str,
|
|
1139
|
-
claim: &str,
|
|
1140
|
-
worker: &str,
|
|
1141
|
-
token: u64,
|
|
1142
|
-
kind: &str,
|
|
1143
|
-
error: &str,
|
|
1144
|
-
now: i64,
|
|
1145
|
-
) -> Result<WorkloadState, WorkloadError> {
|
|
1146
|
-
self.fail_with_effect(id, claim, worker, token, kind, error, None, now)
|
|
1147
|
-
}
|
|
1148
|
-
/// A worker's failure report for its current attempt.
|
|
1149
|
-
///
|
|
1150
|
-
/// Accepted from `CLAIMED` (the worker could not start) as well as from a
|
|
1151
|
-
/// started attempt, fenced like every worker call. `effect` says what the
|
|
1152
|
-
/// worker knows about external effects: `none` — it failed before causing
|
|
1153
|
-
/// any (implied for an attempt that never started); `unknown` — an effect
|
|
1154
|
-
/// may have happened. Without it, the policy decides by `kind`, as before.
|
|
1155
|
-
/// An `unknown` effect is decided like a lost worker: retried only when the
|
|
1156
|
-
/// policy retries `worker_failure`, whatever `kind` says. Repeating the same
|
|
1157
|
-
/// report replays its outcome; any other report from an attempt that
|
|
1158
|
-
/// already reported gets the fence's answer (`CLAIM_INVALID`/`STALE_FENCE`).
|
|
1159
|
-
#[allow(clippy::too_many_arguments)]
|
|
1160
|
-
pub fn fail_with_effect(
|
|
1161
|
-
&mut self,
|
|
1162
|
-
id: &str,
|
|
1163
|
-
claim: &str,
|
|
1164
|
-
worker: &str,
|
|
1165
|
-
token: u64,
|
|
1166
|
-
kind: &str,
|
|
1167
|
-
error: &str,
|
|
1168
|
-
effect: Option<&str>,
|
|
1169
|
-
now: i64,
|
|
1170
|
-
) -> Result<WorkloadState, WorkloadError> {
|
|
1171
|
-
if sensitive_text(error) {
|
|
1172
|
-
return Err(WorkloadError::new(
|
|
1173
|
-
"WORKLOAD_SECRET_REJECTED",
|
|
1174
|
-
"errors may not contain secret material",
|
|
1175
|
-
));
|
|
1176
|
-
}
|
|
1177
|
-
if effect.is_some_and(|e| !matches!(e, "none" | "unknown")) {
|
|
1178
|
-
return Err(WorkloadError::new(
|
|
1179
|
-
"WORKLOAD_INVALID",
|
|
1180
|
-
"effect must be \"none\" or \"unknown\"",
|
|
1181
|
-
));
|
|
1182
|
-
}
|
|
1183
|
-
// The same report repeated (a lost response) replays its outcome. Any
|
|
1184
|
-
// other report from an attempt that already reported is answered by
|
|
1185
|
-
// the fence, as before: its claim is no longer current.
|
|
1186
|
-
let current = self.fresh(id)?.clone();
|
|
1187
|
-
if let Some(attempt) = reported_attempt(¤t, claim, worker, token) {
|
|
1188
|
-
let same = attempt.status == AttemptStatus::Failed
|
|
1189
|
-
&& attempt.error.as_deref() == Some(error)
|
|
1190
|
-
&& failure_event(¤t, &attempt.attempt_id).is_some_and(|event| {
|
|
1191
|
-
let recorded_effect = event.evidence.get("effect").map(String::as_str);
|
|
1192
|
-
event.evidence.get("failure_kind").map(String::as_str) == Some(kind)
|
|
1193
|
-
&& (effect.is_none() || effect == recorded_effect)
|
|
1194
|
-
});
|
|
1195
|
-
if same {
|
|
1196
|
-
return Ok(current.state);
|
|
1197
|
-
}
|
|
1198
|
-
}
|
|
1199
|
-
self.enforce_deadline(id, now)?;
|
|
1200
|
-
self.edit(id, |store, w| {
|
|
1201
|
-
Self::fence(w, claim, worker, token, now)?;
|
|
1202
|
-
let started = w.state != WorkloadState::Claimed;
|
|
1203
|
-
let effect = effect.or((!started).then_some("none"));
|
|
1204
|
-
let attempt = w.attempts.last_mut().unwrap();
|
|
1205
|
-
attempt.status = AttemptStatus::Failed;
|
|
1206
|
-
attempt.ended_at = Some(now);
|
|
1207
|
-
attempt.error = Some(error.into());
|
|
1208
|
-
let attempt_id = attempt.attempt_id.clone();
|
|
1209
|
-
w.last_failure = Some(format!("{kind}: {error}"));
|
|
1210
|
-
w.current_claim = None;
|
|
1211
|
-
let mut evidence: BTreeMap<String, String> = [
|
|
1212
|
-
("failure_kind".into(), kind.into()),
|
|
1213
|
-
("attempt_id".into(), attempt_id),
|
|
1214
|
-
]
|
|
1215
|
-
.into();
|
|
1216
|
-
if let Some(effect) = effect {
|
|
1217
|
-
evidence.insert("effect".into(), effect.into());
|
|
1218
|
-
}
|
|
1219
|
-
store.transition(w, WorkloadState::Failed, worker, now, evidence)?;
|
|
1220
|
-
let unknown = effect == Some("unknown");
|
|
1221
|
-
let allowed =
|
|
1222
|
-
w.retry_policy
|
|
1223
|
-
.retry_on
|
|
1224
|
-
.contains(if unknown { "worker_failure" } else { kind });
|
|
1225
|
-
let decision = if !allowed && unknown {
|
|
1226
|
-
"effect_unknown_not_retryable"
|
|
1227
|
-
} else if !allowed {
|
|
1228
|
-
"failure_kind_not_retryable"
|
|
1229
|
-
} else if w.attempt_count >= attempt_budget(w) {
|
|
1230
|
-
"attempt_budget_exhausted"
|
|
1231
|
-
} else {
|
|
1232
|
-
"retry"
|
|
1233
|
-
};
|
|
1234
|
-
let retryable = decision == "retry";
|
|
1235
|
-
let evidence: BTreeMap<String, String> = [("decision".into(), decision.into())].into();
|
|
1236
|
-
if retryable {
|
|
1237
|
-
w.retry_count += 1;
|
|
1238
|
-
let multiplier = if w.retry_policy.backoff == BackoffPolicy::Exponential {
|
|
1239
|
-
2u64.saturating_pow(w.retry_count.saturating_sub(1))
|
|
1240
|
-
} else {
|
|
1241
|
-
1
|
|
1242
|
-
};
|
|
1243
|
-
let delay = w
|
|
1244
|
-
.retry_policy
|
|
1245
|
-
.initial_delay_ms
|
|
1246
|
-
.saturating_mul(multiplier)
|
|
1247
|
-
.min(w.retry_policy.max_delay_ms);
|
|
1248
|
-
w.next_attempt_at = Some(now + (delay / 1000) as i64);
|
|
1249
|
-
store.transition(w, WorkloadState::Ready, "rust.retry", now, evidence)?;
|
|
1250
|
-
} else {
|
|
1251
|
-
store.transition(w, WorkloadState::DeadLetter, "rust.retry", now, evidence)?;
|
|
1252
|
-
}
|
|
1253
|
-
Ok(w.state)
|
|
1254
|
-
})
|
|
1255
|
-
}
|
|
1256
|
-
pub fn cancel(
|
|
1257
|
-
&mut self,
|
|
1258
|
-
id: &str,
|
|
1259
|
-
actor: &str,
|
|
1260
|
-
now: i64,
|
|
1261
|
-
) -> Result<WorkloadState, WorkloadError> {
|
|
1262
|
-
self.edit(id, |store, w| {
|
|
1263
|
-
match w.state {
|
|
1264
|
-
WorkloadState::Ready
|
|
1265
|
-
| WorkloadState::Created
|
|
1266
|
-
| WorkloadState::Failed
|
|
1267
|
-
| WorkloadState::DeadLetter => {
|
|
1268
|
-
w.current_claim = None;
|
|
1269
|
-
store.transition(w, WorkloadState::Cancelled, actor, now, BTreeMap::new())?
|
|
1270
|
-
}
|
|
1271
|
-
WorkloadState::Claimed | WorkloadState::Running | WorkloadState::Checkpointed => {
|
|
1272
|
-
store.transition(
|
|
1273
|
-
w,
|
|
1274
|
-
WorkloadState::CancellationRequested,
|
|
1275
|
-
actor,
|
|
1276
|
-
now,
|
|
1277
|
-
BTreeMap::new(),
|
|
1278
|
-
)?
|
|
1279
|
-
}
|
|
1280
|
-
WorkloadState::CancellationRequested => {}
|
|
1281
|
-
_ => {
|
|
1282
|
-
return Err(WorkloadError::new(
|
|
1283
|
-
"WORKLOAD_TRANSITION_DENIED",
|
|
1284
|
-
"terminal workload cannot be cancelled",
|
|
1285
|
-
))
|
|
1286
|
-
}
|
|
1287
|
-
}
|
|
1288
|
-
Ok(w.state)
|
|
1289
|
-
})
|
|
1290
|
-
}
|
|
1291
|
-
pub fn confirm_cancel(
|
|
1292
|
-
&mut self,
|
|
1293
|
-
id: &str,
|
|
1294
|
-
claim: &str,
|
|
1295
|
-
worker: &str,
|
|
1296
|
-
token: u64,
|
|
1297
|
-
now: i64,
|
|
1298
|
-
) -> Result<(), WorkloadError> {
|
|
1299
|
-
self.edit(id, |store, w| {
|
|
1300
|
-
Self::fence(w, claim, worker, token, now)?;
|
|
1301
|
-
if w.state != WorkloadState::CancellationRequested {
|
|
1302
|
-
return Err(WorkloadError::new(
|
|
1303
|
-
"WORKLOAD_TRANSITION_DENIED",
|
|
1304
|
-
"cancellation was not requested",
|
|
1305
|
-
));
|
|
1306
|
-
}
|
|
1307
|
-
if let Some(attempt) = w.attempts.last_mut() {
|
|
1308
|
-
attempt.status = AttemptStatus::Cancelled;
|
|
1309
|
-
attempt.ended_at = Some(now);
|
|
1310
|
-
}
|
|
1311
|
-
w.current_claim = None;
|
|
1312
|
-
store.transition(w, WorkloadState::Cancelled, worker, now, BTreeMap::new())
|
|
1313
|
-
})
|
|
1314
|
-
}
|
|
1315
|
-
pub fn timeout(
|
|
1316
|
-
&mut self,
|
|
1317
|
-
id: &str,
|
|
1318
|
-
actor: &str,
|
|
1319
|
-
now: i64,
|
|
1320
|
-
) -> Result<WorkloadState, WorkloadError> {
|
|
1321
|
-
self.edit(id, |store, w| {
|
|
1322
|
-
if !matches!(
|
|
1323
|
-
w.state,
|
|
1324
|
-
WorkloadState::Ready
|
|
1325
|
-
| WorkloadState::Claimed
|
|
1326
|
-
| WorkloadState::Running
|
|
1327
|
-
| WorkloadState::Checkpointed
|
|
1328
|
-
) {
|
|
1329
|
-
return Err(WorkloadError::new(
|
|
1330
|
-
"WORKLOAD_TRANSITION_DENIED",
|
|
1331
|
-
"workload cannot time out in its current state",
|
|
1332
|
-
));
|
|
1333
|
-
}
|
|
1334
|
-
// A Ready workload has no attempt in progress: the last attempt's
|
|
1335
|
-
// recorded outcome is not this timeout's to overwrite.
|
|
1336
|
-
if w.state != WorkloadState::Ready {
|
|
1337
|
-
if let Some(attempt) = w.attempts.last_mut() {
|
|
1338
|
-
attempt.status = AttemptStatus::TimedOut;
|
|
1339
|
-
attempt.ended_at = Some(now);
|
|
1340
|
-
}
|
|
1341
|
-
}
|
|
1342
|
-
w.current_claim = None;
|
|
1343
|
-
store.transition(w, WorkloadState::TimedOut, actor, now, BTreeMap::new())?;
|
|
1344
|
-
if w.attempt_count < attempt_budget(w) && w.retry_policy.retry_on.contains("timeout")
|
|
1345
|
-
{
|
|
1346
|
-
w.retry_count += 1;
|
|
1347
|
-
w.next_attempt_at = Some(now + (w.retry_policy.initial_delay_ms / 1000) as i64);
|
|
1348
|
-
store.transition(w, WorkloadState::Ready, "rust.retry", now, BTreeMap::new())?;
|
|
1349
|
-
} else {
|
|
1350
|
-
store.transition(
|
|
1351
|
-
w,
|
|
1352
|
-
WorkloadState::DeadLetter,
|
|
1353
|
-
"rust.timeout",
|
|
1354
|
-
now,
|
|
1355
|
-
BTreeMap::new(),
|
|
1356
|
-
)?;
|
|
1357
|
-
}
|
|
1358
|
-
Ok(w.state)
|
|
1359
|
-
})
|
|
1360
|
-
}
|
|
1361
|
-
pub fn retry(&mut self, id: &str, actor: &str, now: i64) -> Result<(), WorkloadError> {
|
|
1362
|
-
self.retry_with_reason(id, actor, now, None)
|
|
1363
|
-
}
|
|
1364
|
-
/// An operator's re-arm of failed, dead-lettered or timed-out work.
|
|
1365
|
-
///
|
|
1366
|
-
/// Each re-arm grants one more attempt, as it always has, and is recorded
|
|
1367
|
-
/// in the history with its actor and reason. A workload may be re-armed at
|
|
1368
|
-
/// most `retry_policy.max_attempts` times, so operator retries cannot
|
|
1369
|
-
/// become an unbounded loop.
|
|
1370
|
-
pub fn retry_with_reason(
|
|
1371
|
-
&mut self,
|
|
1372
|
-
id: &str,
|
|
1373
|
-
actor: &str,
|
|
1374
|
-
now: i64,
|
|
1375
|
-
reason: Option<&str>,
|
|
1376
|
-
) -> Result<(), WorkloadError> {
|
|
1377
|
-
self.edit(id, |store, w| {
|
|
1378
|
-
if !matches!(
|
|
1379
|
-
w.state,
|
|
1380
|
-
WorkloadState::Failed | WorkloadState::DeadLetter | WorkloadState::TimedOut
|
|
1381
|
-
) {
|
|
1382
|
-
return Err(WorkloadError::new(
|
|
1383
|
-
"WORKLOAD_TRANSITION_DENIED",
|
|
1384
|
-
"workload is not retryable",
|
|
1385
|
-
));
|
|
1386
|
-
}
|
|
1387
|
-
let previous = operator_retries(w);
|
|
1388
|
-
if previous >= w.retry_policy.max_attempts {
|
|
1389
|
-
return Err(WorkloadError::new(
|
|
1390
|
-
"WORKLOAD_RETRY_EXHAUSTED",
|
|
1391
|
-
"workload has used every operator retry its policy allows",
|
|
1392
|
-
));
|
|
1393
|
-
}
|
|
1394
|
-
w.next_attempt_at = Some(now);
|
|
1395
|
-
w.current_claim = None;
|
|
1396
|
-
let mut evidence: BTreeMap<String, String> = [
|
|
1397
|
-
("retry".into(), "operator".into()),
|
|
1398
|
-
("operator_retry".into(), (previous + 1).to_string()),
|
|
1399
|
-
(
|
|
1400
|
-
"attempt_budget".into(),
|
|
1401
|
-
w.retry_policy
|
|
1402
|
-
.max_attempts
|
|
1403
|
-
.saturating_add(previous + 1)
|
|
1404
|
-
.to_string(),
|
|
1405
|
-
),
|
|
1406
|
-
]
|
|
1407
|
-
.into();
|
|
1408
|
-
if let Some(reason) = reason.filter(|reason| !reason.trim().is_empty()) {
|
|
1409
|
-
evidence.insert("reason".into(), reason.chars().take(256).collect());
|
|
1410
|
-
}
|
|
1411
|
-
store.transition(w, WorkloadState::Ready, actor, now, evidence)
|
|
1412
|
-
})
|
|
1413
|
-
}
|
|
1414
|
-
/// Time out every workload past one of its deadlines (see
|
|
1415
|
-
/// [`deadline_breach`]), as the recovery pass does each second. Deadlines
|
|
1416
|
-
/// are computed from the workload's committed timestamps, so the decision
|
|
1417
|
-
/// is the same after a restart and on any writer of the store.
|
|
1418
|
-
pub fn enforce_deadlines(&mut self, now: i64) -> Result<Vec<String>, WorkloadError> {
|
|
1419
|
-
self.refresh()?;
|
|
1420
|
-
let due = self
|
|
1421
|
-
.workloads
|
|
1422
|
-
.values()
|
|
1423
|
-
.filter_map(|w| deadline_breach(w, now).map(|breach| (w.workload_id.clone(), breach)))
|
|
1424
|
-
.collect::<Vec<_>>();
|
|
1425
|
-
let mut expired = Vec::with_capacity(due.len());
|
|
1426
|
-
for (id, (deadline, at)) in due {
|
|
1427
|
-
match self.expire(&id, deadline, at, now) {
|
|
1428
|
-
Ok(()) => expired.push(id),
|
|
1429
|
-
Err(error) if error.code == "WORKLOAD_CONFLICT" => {}
|
|
1430
|
-
Err(error) => return Err(error),
|
|
1431
|
-
}
|
|
1432
|
-
}
|
|
1433
|
-
Ok(expired)
|
|
1434
|
-
}
|
|
1435
|
-
/// Refuse a worker call on a workload past a deadline, timing it out
|
|
1436
|
-
/// first, in the same way the recovery pass would have.
|
|
1437
|
-
fn enforce_deadline(&mut self, id: &str, now: i64) -> Result<(), WorkloadError> {
|
|
1438
|
-
let Some((deadline, at)) = deadline_breach(self.fresh(id)?, now) else {
|
|
1439
|
-
return Ok(());
|
|
1440
|
-
};
|
|
1441
|
-
self.expire(id, deadline, at, now)?;
|
|
1442
|
-
let mut error = WorkloadError::new(
|
|
1443
|
-
"WORKLOAD_TIMED_OUT",
|
|
1444
|
-
format!("the workload's {deadline} deadline passed"),
|
|
1445
|
-
);
|
|
1446
|
-
error.details = Some(serde_json::json!({ "deadline": deadline, "deadline_at": at }));
|
|
1447
|
-
Err(error)
|
|
1448
|
-
}
|
|
1449
|
-
fn expire(&mut self, id: &str, deadline: &str, at: i64, now: i64) -> Result<(), WorkloadError> {
|
|
1450
|
-
self.edit(id, |store, w| {
|
|
1451
|
-
// Re-check on the committed version: a heartbeat or completion that
|
|
1452
|
-
// committed first decides, and this changes nothing.
|
|
1453
|
-
if deadline_breach(w, now).map(|(kind, _)| kind) != Some(deadline) {
|
|
1454
|
-
return Err(WorkloadError::new(
|
|
1455
|
-
"WORKLOAD_CONFLICT",
|
|
1456
|
-
"the workload changed before its deadline was enforced",
|
|
1457
|
-
));
|
|
1458
|
-
}
|
|
1459
|
-
let lost = w.current_claim.take();
|
|
1460
|
-
if w.state != WorkloadState::Ready {
|
|
1461
|
-
if let Some(attempt) = w.attempts.last_mut() {
|
|
1462
|
-
attempt.status = AttemptStatus::TimedOut;
|
|
1463
|
-
attempt.ended_at = Some(now);
|
|
1464
|
-
attempt.error = Some(format!("timeout: {deadline} deadline passed"));
|
|
1465
|
-
}
|
|
1466
|
-
}
|
|
1467
|
-
let mut evidence: BTreeMap<String, String> = [
|
|
1468
|
-
("deadline".into(), deadline.into()),
|
|
1469
|
-
("deadline_at".into(), at.to_string()),
|
|
1470
|
-
]
|
|
1471
|
-
.into();
|
|
1472
|
-
if let Some(claim) = &lost {
|
|
1473
|
-
evidence.insert("claim_id".into(), claim.claim_id.clone());
|
|
1474
|
-
evidence.insert("worker_id".into(), claim.worker_id.clone());
|
|
1475
|
-
}
|
|
1476
|
-
store.transition(w, WorkloadState::TimedOut, "rust.deadline", now, evidence)?;
|
|
1477
|
-
// The overall and queue deadlines bound the work, not an attempt:
|
|
1478
|
-
// another attempt cannot meet them.
|
|
1479
|
-
let decision = if matches!(deadline, "overall" | "queue") {
|
|
1480
|
-
"deadline_not_retryable"
|
|
1481
|
-
} else if !w.retry_policy.retry_on.contains("timeout") {
|
|
1482
|
-
"timeout_not_retryable"
|
|
1483
|
-
} else if w.attempt_count >= attempt_budget(w) {
|
|
1484
|
-
"attempt_budget_exhausted"
|
|
1485
|
-
} else {
|
|
1486
|
-
"retry"
|
|
1487
|
-
};
|
|
1488
|
-
let evidence: BTreeMap<String, String> = [("decision".into(), decision.into())].into();
|
|
1489
|
-
if decision == "retry" {
|
|
1490
|
-
w.retry_count += 1;
|
|
1491
|
-
w.next_attempt_at = Some(now + (w.retry_policy.initial_delay_ms / 1000) as i64);
|
|
1492
|
-
store.transition(w, WorkloadState::Ready, "rust.deadline", now, evidence)?;
|
|
1493
|
-
} else {
|
|
1494
|
-
store.transition(w, WorkloadState::DeadLetter, "rust.deadline", now, evidence)?;
|
|
1495
|
-
}
|
|
1496
|
-
Ok(())
|
|
1497
|
-
})
|
|
1498
|
-
}
|
|
1499
|
-
pub fn recover_expired(&mut self, now: i64) -> Result<Vec<String>, WorkloadError> {
|
|
1500
|
-
self.refresh()?;
|
|
1501
|
-
let ids = self
|
|
1502
|
-
.workloads
|
|
1503
|
-
.values()
|
|
1504
|
-
.filter(|w| {
|
|
1505
|
-
w.current_claim
|
|
1506
|
-
.as_ref()
|
|
1507
|
-
.is_some_and(|c| c.expires_at <= now)
|
|
1508
|
-
&& matches!(
|
|
1509
|
-
w.state,
|
|
1510
|
-
WorkloadState::Claimed
|
|
1511
|
-
| WorkloadState::Running
|
|
1512
|
-
| WorkloadState::Checkpointed
|
|
1513
|
-
| WorkloadState::CancellationRequested
|
|
1514
|
-
)
|
|
1515
|
-
})
|
|
1516
|
-
.map(|w| w.workload_id.clone())
|
|
1517
|
-
.collect::<Vec<_>>();
|
|
1518
|
-
let mut recovered = Vec::with_capacity(ids.len());
|
|
1519
|
-
for id in ids {
|
|
1520
|
-
let outcome = self.edit(&id, |store, w| {
|
|
1521
|
-
// Decide on the committed version: another writer may have
|
|
1522
|
-
// recovered (or the worker completed) it since it was listed.
|
|
1523
|
-
let still_expired = w
|
|
1524
|
-
.current_claim
|
|
1525
|
-
.as_ref()
|
|
1526
|
-
.is_some_and(|c| c.expires_at <= now)
|
|
1527
|
-
&& matches!(
|
|
1528
|
-
w.state,
|
|
1529
|
-
WorkloadState::Claimed
|
|
1530
|
-
| WorkloadState::Running
|
|
1531
|
-
| WorkloadState::Checkpointed
|
|
1532
|
-
| WorkloadState::CancellationRequested
|
|
1533
|
-
);
|
|
1534
|
-
if !still_expired {
|
|
1535
|
-
return Err(WorkloadError::new(
|
|
1536
|
-
"WORKLOAD_CONFLICT",
|
|
1537
|
-
"the workload changed before it was recovered",
|
|
1538
|
-
));
|
|
1539
|
-
}
|
|
1540
|
-
let lost = w.current_claim.take();
|
|
1541
|
-
if let Some(attempt) = w.attempts.last_mut() {
|
|
1542
|
-
attempt.status = AttemptStatus::WorkerLost;
|
|
1543
|
-
attempt.ended_at = Some(now);
|
|
1544
|
-
attempt.error = Some("worker_lost: claim lease expired".into());
|
|
1545
|
-
}
|
|
1546
|
-
let mut evidence: BTreeMap<String, String> = [
|
|
1547
|
-
("recovery_reason".into(), "lease_expired".into()),
|
|
1548
|
-
("attempts_used".into(), w.attempt_count.to_string()),
|
|
1549
|
-
("attempt_budget".into(), attempt_budget(w).to_string()),
|
|
1550
|
-
]
|
|
1551
|
-
.into();
|
|
1552
|
-
if let Some(claim) = &lost {
|
|
1553
|
-
evidence.insert("claim_id".into(), claim.claim_id.clone());
|
|
1554
|
-
evidence.insert("worker_id".into(), claim.worker_id.clone());
|
|
1555
|
-
evidence.insert("lease_expired_at".into(), claim.expires_at.to_string());
|
|
1556
|
-
}
|
|
1557
|
-
if let Some(attempt) = w.attempts.last() {
|
|
1558
|
-
evidence.insert("attempt_id".into(), attempt.attempt_id.clone());
|
|
1559
|
-
}
|
|
1560
|
-
if w.state == WorkloadState::CancellationRequested {
|
|
1561
|
-
store.transition(w, WorkloadState::Cancelled, "rust.recovery", now, evidence)?;
|
|
1562
|
-
} else {
|
|
1563
|
-
store.transition(
|
|
1564
|
-
w,
|
|
1565
|
-
WorkloadState::Recovering,
|
|
1566
|
-
"rust.recovery",
|
|
1567
|
-
now,
|
|
1568
|
-
evidence,
|
|
1569
|
-
)?;
|
|
1570
|
-
// A lost worker is the policy's `worker_failure`. Whether
|
|
1571
|
-
// the lost attempt had effects is unknown to FeltDB, so a
|
|
1572
|
-
// retry re-executes it: at-least-once, never exactly-once.
|
|
1573
|
-
let (to, decision) = if !w.retry_policy.retry_on.contains("worker_failure") {
|
|
1574
|
-
(WorkloadState::DeadLetter, "worker_failure_not_retryable")
|
|
1575
|
-
} else if w.attempt_count >= attempt_budget(w) {
|
|
1576
|
-
(WorkloadState::DeadLetter, "attempt_budget_exhausted")
|
|
1577
|
-
} else {
|
|
1578
|
-
(WorkloadState::Ready, "retry")
|
|
1579
|
-
};
|
|
1580
|
-
store.transition(
|
|
1581
|
-
w,
|
|
1582
|
-
to,
|
|
1583
|
-
"rust.recovery",
|
|
1584
|
-
now,
|
|
1585
|
-
[("decision".into(), decision.into())].into(),
|
|
1586
|
-
)?;
|
|
1587
|
-
}
|
|
1588
|
-
Ok(())
|
|
1589
|
-
});
|
|
1590
|
-
match outcome {
|
|
1591
|
-
Ok(()) => recovered.push(id),
|
|
1592
|
-
// Another writer of this store committed the workload first —
|
|
1593
|
-
// in a race, its own recovery. Nothing of this attempt was
|
|
1594
|
-
// committed and `edit` has adopted the committed workload, so
|
|
1595
|
-
// the attempt is recovered once, by that writer. If it still
|
|
1596
|
-
// needs recovery, the next pass sees it.
|
|
1597
|
-
Err(error) if error.code == "WORKLOAD_CONFLICT" => {}
|
|
1598
|
-
Err(error) => return Err(error),
|
|
1599
|
-
}
|
|
1600
|
-
}
|
|
1601
|
-
Ok(recovered)
|
|
1602
|
-
}
|
|
1603
|
-
}
|
|
1604
|
-
|
|
1605
|
-
pub trait WorkScheduler {
|
|
1606
|
-
fn enqueue(&mut self, workload_id: &str);
|
|
1607
|
-
fn claim(
|
|
1608
|
-
&mut self,
|
|
1609
|
-
store: &mut WorkloadStore,
|
|
1610
|
-
tenant: &str,
|
|
1611
|
-
application: &str,
|
|
1612
|
-
worker: &str,
|
|
1613
|
-
capabilities: &BTreeSet<String>,
|
|
1614
|
-
now: i64,
|
|
1615
|
-
lease_ms: u64,
|
|
1616
|
-
) -> Result<Option<WorkloadClaim>, WorkloadError>;
|
|
1617
|
-
fn release(&mut self, claim_id: &str);
|
|
1618
|
-
}
|
|
1619
|
-
#[derive(Default)]
|
|
1620
|
-
pub struct LocalDurableScheduler {
|
|
1621
|
-
queue: BTreeSet<String>,
|
|
1622
|
-
}
|
|
1623
|
-
impl WorkScheduler for LocalDurableScheduler {
|
|
1624
|
-
fn enqueue(&mut self, id: &str) {
|
|
1625
|
-
self.queue.insert(id.into());
|
|
1626
|
-
}
|
|
1627
|
-
fn claim(
|
|
1628
|
-
&mut self,
|
|
1629
|
-
store: &mut WorkloadStore,
|
|
1630
|
-
tenant: &str,
|
|
1631
|
-
application: &str,
|
|
1632
|
-
worker: &str,
|
|
1633
|
-
capabilities: &BTreeSet<String>,
|
|
1634
|
-
now: i64,
|
|
1635
|
-
lease_ms: u64,
|
|
1636
|
-
) -> Result<Option<WorkloadClaim>, WorkloadError> {
|
|
1637
|
-
let selected = store
|
|
1638
|
-
.list(tenant, application)
|
|
1639
|
-
.into_iter()
|
|
1640
|
-
.filter(|w| {
|
|
1641
|
-
// Eligibility is part of selection, not a reason to fail. A
|
|
1642
|
-
// workload that is merely early is skipped so the scheduler can
|
|
1643
|
-
// hand out work that is ready now; without this it would pick an
|
|
1644
|
-
// ineligible workload, have `store.claim` refuse it, and return
|
|
1645
|
-
// an error while claimable work sat behind it.
|
|
1646
|
-
w.state == WorkloadState::Ready
|
|
1647
|
-
&& w.is_eligible_at(now)
|
|
1648
|
-
&& w.capability_snapshot.is_subset(capabilities)
|
|
1649
|
-
})
|
|
1650
|
-
.find(|w| {
|
|
1651
|
-
w.dependencies.iter().all(|d| {
|
|
1652
|
-
store
|
|
1653
|
-
.workloads
|
|
1654
|
-
.get(&d.workload_id)
|
|
1655
|
-
.is_some_and(|dependency| dependency.state == d.required_state)
|
|
1656
|
-
})
|
|
1657
|
-
})
|
|
1658
|
-
.map(|w| w.workload_id);
|
|
1659
|
-
match selected {
|
|
1660
|
-
Some(id) => store
|
|
1661
|
-
.claim(&id, worker, capabilities, now, lease_ms)
|
|
1662
|
-
.map(Some),
|
|
1663
|
-
None => Ok(None),
|
|
1664
|
-
}
|
|
1665
|
-
}
|
|
1666
|
-
fn release(&mut self, _: &str) {}
|
|
1667
|
-
}
|
|
1668
|
-
fn legal(from: WorkloadState, to: WorkloadState) -> bool {
|
|
1669
|
-
use WorkloadState::*;
|
|
1670
|
-
matches!(
|
|
1671
|
-
(from, to),
|
|
1672
|
-
(Created, Ready)
|
|
1673
|
-
| (Ready, Claimed)
|
|
1674
|
-
| (Claimed, Running)
|
|
1675
|
-
| (Running, Checkpointed)
|
|
1676
|
-
| (Checkpointed, Checkpointed)
|
|
1677
|
-
| (Running, Succeeded)
|
|
1678
|
-
| (Checkpointed, Succeeded)
|
|
1679
|
-
| (Claimed, CancellationRequested)
|
|
1680
|
-
| (Running, CancellationRequested)
|
|
1681
|
-
| (Checkpointed, CancellationRequested)
|
|
1682
|
-
| (CancellationRequested, Cancelled)
|
|
1683
|
-
| (Ready, Cancelled)
|
|
1684
|
-
| (Created, Cancelled)
|
|
1685
|
-
| (Failed, Cancelled)
|
|
1686
|
-
| (DeadLetter, Cancelled)
|
|
1687
|
-
| (Claimed, Failed)
|
|
1688
|
-
| (Running, Failed)
|
|
1689
|
-
| (Checkpointed, Failed)
|
|
1690
|
-
| (Claimed, Recovering)
|
|
1691
|
-
| (Running, Recovering)
|
|
1692
|
-
| (Checkpointed, Recovering)
|
|
1693
|
-
| (Recovering, Ready)
|
|
1694
|
-
| (Failed, Ready)
|
|
1695
|
-
| (TimedOut, Ready)
|
|
1696
|
-
| (DeadLetter, Ready)
|
|
1697
|
-
| (Failed, DeadLetter)
|
|
1698
|
-
| (Running, TimedOut)
|
|
1699
|
-
| (Claimed, TimedOut)
|
|
1700
|
-
| (Checkpointed, TimedOut)
|
|
1701
|
-
| (Ready, TimedOut)
|
|
1702
|
-
| (TimedOut, DeadLetter)
|
|
1703
|
-
| (Recovering, DeadLetter)
|
|
1704
|
-
| (Ready, DeadLetter)
|
|
1705
|
-
)
|
|
1706
|
-
}
|
|
1707
|
-
pub fn can_transition(from: WorkloadState, to: WorkloadState) -> bool {
|
|
1708
|
-
legal(from, to)
|
|
1709
|
-
}
|
|
1710
|
-
fn event_name(state: WorkloadState) -> &'static str {
|
|
1711
|
-
use WorkloadState::*;
|
|
1712
|
-
match state {
|
|
1713
|
-
Created => "workload.created",
|
|
1714
|
-
Ready => "workload.ready",
|
|
1715
|
-
Claimed => "workload.claimed",
|
|
1716
|
-
Running => "workload.started",
|
|
1717
|
-
Checkpointed => "workload.checkpointed",
|
|
1718
|
-
Recovering => "workload.recovered",
|
|
1719
|
-
CancellationRequested => "workload.cancellation_requested",
|
|
1720
|
-
Succeeded => "workload.completed",
|
|
1721
|
-
Failed => "workload.failed",
|
|
1722
|
-
Cancelled => "workload.cancelled",
|
|
1723
|
-
TimedOut => "workload.timed_out",
|
|
1724
|
-
DeadLetter => "workload.dead_lettered",
|
|
1725
|
-
}
|
|
1726
|
-
}
|
|
1727
|
-
fn ensure_safe(value: &Value) -> Result<(), WorkloadError> {
|
|
1728
|
-
fn walk(value: &Value) -> bool {
|
|
1729
|
-
match value {
|
|
1730
|
-
Value::Object(map) => map.iter().any(|(k, v)| {
|
|
1731
|
-
matches!(
|
|
1732
|
-
k.to_ascii_lowercase().as_str(),
|
|
1733
|
-
"password" | "secret" | "token" | "api_key" | "private_key"
|
|
1734
|
-
) || walk(v)
|
|
1735
|
-
}),
|
|
1736
|
-
Value::Array(values) => values.iter().any(walk),
|
|
1737
|
-
Value::String(v) => sensitive_text(v),
|
|
1738
|
-
_ => false,
|
|
1739
|
-
}
|
|
1740
|
-
}
|
|
1741
|
-
if walk(value) {
|
|
1742
|
-
Err(WorkloadError::new(
|
|
1743
|
-
"WORKLOAD_SECRET_REJECTED",
|
|
1744
|
-
"plaintext secrets are forbidden in workload state",
|
|
1745
|
-
))
|
|
1746
|
-
} else {
|
|
1747
|
-
Ok(())
|
|
1748
|
-
}
|
|
1749
|
-
}
|
|
1750
|
-
fn sensitive_text(v: &str) -> bool {
|
|
1751
|
-
v.contains("BEGIN PRIVATE KEY") || v.starts_with("sk-") || v.starts_with("fdb_live_")
|
|
1752
|
-
}
|
|
1753
|
-
/// The identity a caller's idempotency key is unique within: tenant,
|
|
1754
|
-
/// application and requester. Encoded as a JSON array, so no choice of key or
|
|
1755
|
-
/// identifier can make two different scopes coincide.
|
|
1756
|
-
fn idempotency_scope(tenant: &str, application: &str, requester: &Subject, key: &str) -> String {
|
|
1757
|
-
serde_json::json!([tenant, application, requester.kind(), requester.id(), key]).to_string()
|
|
1758
|
-
}
|
|
1759
|
-
/// The scope before keys were bound to their requester, kept only to verify
|
|
1760
|
-
/// legacy store files during migration.
|
|
1761
|
-
fn legacy_idempotency_scope(
|
|
1762
|
-
tenant: &str,
|
|
1763
|
-
application: &str,
|
|
1764
|
-
_requester: &Subject,
|
|
1765
|
-
key: &str,
|
|
1766
|
-
) -> String {
|
|
1767
|
-
format!("{tenant}:{application}:{key}")
|
|
1768
|
-
}
|
|
1769
|
-
fn idempotency_index(
|
|
1770
|
-
workloads: &BTreeMap<String, Workload>,
|
|
1771
|
-
scope: fn(&str, &str, &Subject, &str) -> String,
|
|
1772
|
-
) -> BTreeMap<String, String> {
|
|
1773
|
-
workloads
|
|
1774
|
-
.values()
|
|
1775
|
-
.map(|w| {
|
|
1776
|
-
(
|
|
1777
|
-
scope(&w.tenant_id, &w.application_id, &w.requested_by, &w.idempotency_key),
|
|
1778
|
-
w.workload_id.clone(),
|
|
1779
|
-
)
|
|
1780
|
-
})
|
|
1781
|
-
.collect()
|
|
1782
|
-
}
|
|
1783
|
-
/// Answer a repeated key: the original workload for the same request, a
|
|
1784
|
-
/// refusal for a different one. Never the original for a different request.
|
|
1785
|
-
/// The execution-affecting part of an admission, normalized, field by field.
|
|
1786
|
-
///
|
|
1787
|
-
/// A replay under the same key is answered with the original workload only when
|
|
1788
|
-
/// every one of these is equal. Optional fields are compared after their
|
|
1789
|
-
/// defaults apply (serde fills an omitted `retry_policy`, `timeouts`,
|
|
1790
|
-
/// `priority`, `connection_scopes` or `dependencies`); sets compare as sets and
|
|
1791
|
-
/// dependencies regardless of order; the input compares by its canonical hash,
|
|
1792
|
-
/// so member order is not part of it. Provenance that does not change what runs
|
|
1793
|
-
/// — `trigger`, `session_id`, `created_at`, and the authorization snapshot,
|
|
1794
|
-
/// which is verified before any replay — is not compared.
|
|
1795
|
-
fn execution_fields(
|
|
1796
|
-
definition_id: &str,
|
|
1797
|
-
definition_revision: &str,
|
|
1798
|
-
environment: &str,
|
|
1799
|
-
input_hash: &str,
|
|
1800
|
-
capability_snapshot: &BTreeSet<String>,
|
|
1801
|
-
connection_scopes: &BTreeSet<String>,
|
|
1802
|
-
priority: i32,
|
|
1803
|
-
retry_policy: &WorkloadRetryPolicy,
|
|
1804
|
-
timeouts: &WorkloadTimeouts,
|
|
1805
|
-
dependencies: &[WorkloadDependency],
|
|
1806
|
-
not_before: Option<i64>,
|
|
1807
|
-
) -> Vec<(&'static str, Value)> {
|
|
1808
|
-
let mut dependencies = dependencies
|
|
1809
|
-
.iter()
|
|
1810
|
-
.map(|d| serde_json::json!([d.workload_id, d.required_state]))
|
|
1811
|
-
.collect::<Vec<_>>();
|
|
1812
|
-
dependencies.sort_by_key(|d| d.to_string());
|
|
1813
|
-
vec![
|
|
1814
|
-
("definition_id", definition_id.into()),
|
|
1815
|
-
("definition_revision", definition_revision.into()),
|
|
1816
|
-
("environment", environment.into()),
|
|
1817
|
-
("input", input_hash.into()),
|
|
1818
|
-
(
|
|
1819
|
-
"capability_snapshot",
|
|
1820
|
-
serde_json::json!(capability_snapshot),
|
|
1821
|
-
),
|
|
1822
|
-
("connection_scopes", serde_json::json!(connection_scopes)),
|
|
1823
|
-
("priority", priority.into()),
|
|
1824
|
-
("retry_policy", serde_json::json!(retry_policy)),
|
|
1825
|
-
("timeouts", serde_json::json!(timeouts)),
|
|
1826
|
-
("dependencies", Value::Array(dependencies)),
|
|
1827
|
-
("not_before", serde_json::json!(not_before)),
|
|
1828
|
-
]
|
|
1829
|
-
}
|
|
1830
|
-
/// The `not_before` a workload was admitted with, from its creation event
|
|
1831
|
-
/// (`next_attempt_at` moves with retries; the admission evidence does not).
|
|
1832
|
-
fn admitted_not_before(w: &Workload) -> Option<i64> {
|
|
1833
|
-
w.execution_history
|
|
1834
|
-
.iter()
|
|
1835
|
-
.find(|event| event.to == WorkloadState::Ready)
|
|
1836
|
-
.and_then(|event| event.evidence.get("not_before"))
|
|
1837
|
-
.and_then(|value| value.parse().ok())
|
|
1838
|
-
}
|
|
1839
|
-
fn replay(existing: Workload, input: &CreateWorkload) -> Result<(Workload, bool), WorkloadError> {
|
|
1840
|
-
let recorded = execution_fields(
|
|
1841
|
-
&existing.definition_id,
|
|
1842
|
-
&existing.definition_revision,
|
|
1843
|
-
&existing.environment,
|
|
1844
|
-
&existing.input_hash,
|
|
1845
|
-
&existing.capability_snapshot,
|
|
1846
|
-
&existing.connection_scopes,
|
|
1847
|
-
existing.priority,
|
|
1848
|
-
&existing.retry_policy,
|
|
1849
|
-
&existing.timeouts,
|
|
1850
|
-
&existing.dependencies,
|
|
1851
|
-
admitted_not_before(&existing),
|
|
1852
|
-
);
|
|
1853
|
-
let requested = execution_fields(
|
|
1854
|
-
&input.definition_id,
|
|
1855
|
-
&input.definition_revision,
|
|
1856
|
-
&input.environment,
|
|
1857
|
-
&hash(&input.input),
|
|
1858
|
-
&input.capability_snapshot,
|
|
1859
|
-
&input.connection_scopes,
|
|
1860
|
-
input.priority,
|
|
1861
|
-
&input.retry_policy,
|
|
1862
|
-
&input.timeouts,
|
|
1863
|
-
&input.dependencies,
|
|
1864
|
-
input.not_before,
|
|
1865
|
-
);
|
|
1866
|
-
let mut conflicting = recorded
|
|
1867
|
-
.iter()
|
|
1868
|
-
.zip(&requested)
|
|
1869
|
-
.filter(|((_, left), (_, right))| left != right)
|
|
1870
|
-
.map(|((name, _), _)| *name)
|
|
1871
|
-
.collect::<Vec<_>>();
|
|
1872
|
-
if existing.requested_by != input.requested_by {
|
|
1873
|
-
conflicting.insert(0, "requested_by");
|
|
1874
|
-
}
|
|
1875
|
-
if conflicting.is_empty() {
|
|
1876
|
-
return Ok((existing, false));
|
|
1877
|
-
}
|
|
1878
|
-
let mut error = WorkloadError::new(
|
|
1879
|
-
"IDEMPOTENCY_KEY_REUSED",
|
|
1880
|
-
"idempotency key was already used for a different request",
|
|
1881
|
-
);
|
|
1882
|
-
error.details = Some(serde_json::json!({ "fields": conflicting }));
|
|
1883
|
-
Err(error)
|
|
1884
|
-
}
|
|
1885
|
-
/// Seconds a millisecond timeout lasts, rounded up so a deadline is never early.
|
|
1886
|
-
fn deadline_seconds(ms: u64) -> i64 {
|
|
1887
|
-
ms.div_ceil(1000) as i64
|
|
1888
|
-
}
|
|
1889
|
-
/// The first deadline a workload has passed at `now`, and when it fell.
|
|
1890
|
-
///
|
|
1891
|
-
/// | Deadline | Measured from | While |
|
|
1892
|
-
/// | --- | --- | --- |
|
|
1893
|
-
/// | `overall` | admission (`created_at`) | not terminal |
|
|
1894
|
-
/// | `queue` | becoming `READY` (or its `next_attempt_at`, if later) | `READY` |
|
|
1895
|
-
/// | `claim` | the claim | `CLAIMED` |
|
|
1896
|
-
/// | `execution` | the attempt's start | `RUNNING`, `CHECKPOINTED` |
|
|
1897
|
-
/// | `heartbeat` | the last heartbeat (or the claim) | claimed |
|
|
1898
|
-
///
|
|
1899
|
-
/// Only workloads admitted at [`DEADLINE_CONTRACT_VERSION`] or later.
|
|
1900
|
-
pub fn deadline_breach(w: &Workload, now: i64) -> Option<(&'static str, i64)> {
|
|
1901
|
-
use WorkloadState::*;
|
|
1902
|
-
if w.contract_version < DEADLINE_CONTRACT_VERSION {
|
|
1903
|
-
return None;
|
|
1904
|
-
}
|
|
1905
|
-
let active = matches!(w.state, Ready | Claimed | Running | Checkpointed);
|
|
1906
|
-
if !active {
|
|
1907
|
-
return None;
|
|
1908
|
-
}
|
|
1909
|
-
let t = &w.timeouts;
|
|
1910
|
-
let claim = w.current_claim.as_ref();
|
|
1911
|
-
let candidates = [
|
|
1912
|
-
(
|
|
1913
|
-
"overall",
|
|
1914
|
-
t.overall_ms.map(|ms| w.created_at + deadline_seconds(ms)),
|
|
1915
|
-
),
|
|
1916
|
-
(
|
|
1917
|
-
"queue",
|
|
1918
|
-
t.queue_ms.filter(|_| w.state == Ready).map(|ms| {
|
|
1919
|
-
let ready_since = w
|
|
1920
|
-
.execution_history
|
|
1921
|
-
.iter()
|
|
1922
|
-
.rev()
|
|
1923
|
-
.find(|event| event.to == Ready)
|
|
1924
|
-
.map_or(w.created_at, |event| event.at);
|
|
1925
|
-
ready_since.max(w.next_attempt_at.unwrap_or(i64::MIN)) + deadline_seconds(ms)
|
|
1926
|
-
}),
|
|
1927
|
-
),
|
|
1928
|
-
(
|
|
1929
|
-
"claim",
|
|
1930
|
-
t.claim_ms
|
|
1931
|
-
.filter(|_| w.state == Claimed)
|
|
1932
|
-
.and_then(|ms| claim.map(|c| c.claimed_at + deadline_seconds(ms))),
|
|
1933
|
-
),
|
|
1934
|
-
(
|
|
1935
|
-
"execution",
|
|
1936
|
-
t.execution_ms
|
|
1937
|
-
.filter(|_| matches!(w.state, Running | Checkpointed))
|
|
1938
|
-
.and_then(|ms| {
|
|
1939
|
-
w.attempts
|
|
1940
|
-
.last()
|
|
1941
|
-
.and_then(|a| a.started_at)
|
|
1942
|
-
.map(|started| started + deadline_seconds(ms))
|
|
1943
|
-
}),
|
|
1944
|
-
),
|
|
1945
|
-
(
|
|
1946
|
-
"heartbeat",
|
|
1947
|
-
t.heartbeat_ms
|
|
1948
|
-
.and_then(|ms| claim.map(|c| c.last_heartbeat + deadline_seconds(ms))),
|
|
1949
|
-
),
|
|
1950
|
-
];
|
|
1951
|
-
candidates
|
|
1952
|
-
.into_iter()
|
|
1953
|
-
.find_map(|(kind, at)| at.filter(|at| now >= *at).map(|at| (kind, at)))
|
|
1954
|
-
}
|
|
1955
|
-
/// The attempt this fence named, when that attempt has already reported its
|
|
1956
|
-
/// outcome (failed or succeeded). Lost and timed-out attempts are not reports.
|
|
1957
|
-
fn reported_attempt<'w>(
|
|
1958
|
-
w: &'w Workload,
|
|
1959
|
-
claim: &str,
|
|
1960
|
-
worker: &str,
|
|
1961
|
-
token: u64,
|
|
1962
|
-
) -> Option<&'w WorkloadAttempt> {
|
|
1963
|
-
w.attempts.iter().find(|attempt| {
|
|
1964
|
-
attempt.claim_id == claim
|
|
1965
|
-
&& attempt.worker_id == worker
|
|
1966
|
-
&& attempt.fencing_token == token
|
|
1967
|
-
&& matches!(
|
|
1968
|
-
attempt.status,
|
|
1969
|
-
AttemptStatus::Failed | AttemptStatus::Succeeded
|
|
1970
|
-
)
|
|
1971
|
-
})
|
|
1972
|
-
}
|
|
1973
|
-
/// The `FAILED` event a failure report of `attempt_id` recorded.
|
|
1974
|
-
fn failure_event<'w>(w: &'w Workload, attempt_id: &str) -> Option<&'w WorkloadEvent> {
|
|
1975
|
-
w.execution_history.iter().rev().find(|event| {
|
|
1976
|
-
event.to == WorkloadState::Failed
|
|
1977
|
-
&& event.evidence.get("attempt_id").map(String::as_str) == Some(attempt_id)
|
|
1978
|
-
})
|
|
1979
|
-
}
|
|
1980
|
-
/// Operator re-arms recorded in the workload's own history.
|
|
1981
|
-
fn operator_retries(w: &Workload) -> u32 {
|
|
1982
|
-
w.execution_history
|
|
1983
|
-
.iter()
|
|
1984
|
-
.filter(|event| event.evidence.get("retry").map(String::as_str) == Some("operator"))
|
|
1985
|
-
.count() as u32
|
|
1986
|
-
}
|
|
1987
|
-
/// How many claims the workload may have in total: its policy's
|
|
1988
|
-
/// `max_attempts`, plus one for each operator re-arm (which is what an operator
|
|
1989
|
-
/// retry has always granted).
|
|
1990
|
-
fn attempt_budget(w: &Workload) -> u32 {
|
|
1991
|
-
w.retry_policy
|
|
1992
|
-
.max_attempts
|
|
1993
|
-
.saturating_add(operator_retries(w))
|
|
1994
|
-
}
|
|
1995
|
-
fn work_key(workload: &Workload) -> String {
|
|
1996
|
-
format!(
|
|
1997
|
-
"{}/{}/{}",
|
|
1998
|
-
workload.tenant_id, workload.application_id, workload.workload_id
|
|
1999
|
-
)
|
|
2000
|
-
}
|
|
2001
|
-
fn derived_event_sequence(workloads: &BTreeMap<String, Workload>) -> u64 {
|
|
2002
|
-
workloads
|
|
2003
|
-
.values()
|
|
2004
|
-
.flat_map(|workload| workload.execution_history.iter())
|
|
2005
|
-
.map(|event| event.sequence)
|
|
2006
|
-
.max()
|
|
2007
|
-
.unwrap_or_default()
|
|
2008
|
-
}
|
|
2009
|
-
fn runtime_id(prefix: &str) -> String {
|
|
2010
|
-
let value: String = rand::thread_rng()
|
|
2011
|
-
.sample_iter(&Alphanumeric)
|
|
2012
|
-
.take(24)
|
|
2013
|
-
.map(char::from)
|
|
2014
|
-
.collect();
|
|
2015
|
-
format!("{prefix}_{value}")
|
|
2016
|
-
}
|
|
2017
|
-
fn io(e: impl std::fmt::Display) -> WorkloadError {
|
|
2018
|
-
WorkloadError::new("WORKLOAD_STORAGE_FAILURE", e.to_string())
|
|
2019
|
-
}
|
|
2020
|
-
fn workload_storage_error(error: FlowError) -> WorkloadError {
|
|
2021
|
-
match error {
|
|
2022
|
-
FlowError::PreconditionFailed(_) | FlowError::RevisionConflict { .. } => {
|
|
2023
|
-
WorkloadError::new("WORKLOAD_CONFLICT", error.to_string())
|
|
2024
|
-
}
|
|
2025
|
-
FlowError::CapabilityError(ref message) if message.starts_with("PRECONDITION_FAILED:") => {
|
|
2026
|
-
WorkloadError::new("WORKLOAD_CONFLICT", error.to_string())
|
|
2027
|
-
}
|
|
2028
|
-
_ => io(error),
|
|
2029
|
-
}
|
|
2030
|
-
}
|
|
2031
|
-
fn hash(value: &Value) -> String {
|
|
2032
|
-
hex(&Sha256::digest(
|
|
2033
|
-
serde_json::to_vec(value).unwrap_or_default(),
|
|
2034
|
-
))
|
|
2035
|
-
}
|
|
2036
|
-
fn hex(bytes: &[u8]) -> String {
|
|
2037
|
-
bytes.iter().map(|v| format!("{v:02x}")).collect()
|
|
2038
|
-
}
|
|
2039
|
-
|
|
2040
|
-
#[cfg(test)]
|
|
2041
|
-
mod tests {
|
|
2042
|
-
use super::*;
|
|
2043
|
-
use crate::authorization::{
|
|
2044
|
-
Grant, GrantConstraints, GrantSigner, GrantStatus, ResourceScope, ResourceScopeKind,
|
|
2045
|
-
};
|
|
2046
|
-
use std::{fs, path::PathBuf};
|
|
2047
|
-
fn create() -> CreateWorkload {
|
|
2048
|
-
create_as("a1")
|
|
2049
|
-
}
|
|
2050
|
-
/// The fixture request, made by agent `agent`.
|
|
2051
|
-
fn create_as(agent: &str) -> CreateWorkload {
|
|
2052
|
-
let signer = GrantSigner::new("k", vec![1; 32]).unwrap();
|
|
2053
|
-
let subject = Subject::Agent(agent.into());
|
|
2054
|
-
let grant = Grant {
|
|
2055
|
-
contract_version: 1,
|
|
2056
|
-
id: "g".into(),
|
|
2057
|
-
issuer: Subject::Human("h".into()),
|
|
2058
|
-
subject: subject.clone(),
|
|
2059
|
-
tenant_id: "t".into(),
|
|
2060
|
-
application_id: "app".into(),
|
|
2061
|
-
capabilities: ["workload:execute".into()].into(),
|
|
2062
|
-
resources: vec![ResourceScope {
|
|
2063
|
-
kind: ResourceScopeKind::Workload,
|
|
2064
|
-
uri: "flow://app/workloads/*".into(),
|
|
2065
|
-
}],
|
|
2066
|
-
constraints: GrantConstraints::default(),
|
|
2067
|
-
issued_at: 1,
|
|
2068
|
-
expires_at: 9999,
|
|
2069
|
-
parent_grant: None,
|
|
2070
|
-
revision: 1,
|
|
2071
|
-
status: GrantStatus::Active,
|
|
2072
|
-
};
|
|
2073
|
-
CreateWorkload {
|
|
2074
|
-
tenant_id: "t".into(),
|
|
2075
|
-
application_id: "app".into(),
|
|
2076
|
-
environment: "production".into(),
|
|
2077
|
-
definition_id: "send".into(),
|
|
2078
|
-
definition_revision: "r1".into(),
|
|
2079
|
-
trigger: "manual".into(),
|
|
2080
|
-
input: serde_json::json!({"id":1}),
|
|
2081
|
-
requested_by: subject,
|
|
2082
|
-
session_id: Some("session-1".into()),
|
|
2083
|
-
authorization_snapshot: signer.sign(grant).unwrap(),
|
|
2084
|
-
capability_snapshot: ["email:send".into()].into(),
|
|
2085
|
-
connection_scopes: BTreeSet::new(),
|
|
2086
|
-
priority: 0,
|
|
2087
|
-
idempotency_key: "one".into(),
|
|
2088
|
-
retry_policy: Default::default(),
|
|
2089
|
-
timeouts: Default::default(),
|
|
2090
|
-
dependencies: vec![],
|
|
2091
|
-
not_before: None,
|
|
2092
|
-
created_at: 10,
|
|
2093
|
-
}
|
|
2094
|
-
}
|
|
2095
|
-
fn capabilities() -> BTreeSet<String> {
|
|
2096
|
-
["email:send".into()].into()
|
|
2097
|
-
}
|
|
2098
|
-
|
|
2099
|
-
#[test]
|
|
2100
|
-
fn idempotency_keys_are_bound_to_their_requester_and_request() {
|
|
2101
|
-
let (path, mut store) = store("requester-scoped-keys");
|
|
2102
|
-
let (original, created) = store.admit(create_as("a1")).unwrap();
|
|
2103
|
-
assert!(created);
|
|
2104
|
-
|
|
2105
|
-
// The same requester and request: the original, not a second workload.
|
|
2106
|
-
let (replayed, created) = store.admit(create_as("a1")).unwrap();
|
|
2107
|
-
assert!(!created);
|
|
2108
|
-
assert_eq!(replayed.workload_id, original.workload_id);
|
|
2109
|
-
|
|
2110
|
-
// The same key for a different request is refused and creates nothing.
|
|
2111
|
-
let mut different = create_as("a1");
|
|
2112
|
-
different.input = serde_json::json!({"id": 2});
|
|
2113
|
-
assert_eq!(store.admit(different).unwrap_err().code, "IDEMPOTENCY_KEY_REUSED");
|
|
2114
|
-
let mut other_definition = create_as("a1");
|
|
2115
|
-
other_definition.definition_id = "other".into();
|
|
2116
|
-
assert_eq!(
|
|
2117
|
-
store.admit(other_definition).unwrap_err().code,
|
|
2118
|
-
"IDEMPOTENCY_KEY_REUSED"
|
|
2119
|
-
);
|
|
2120
|
-
|
|
2121
|
-
// Another requester's identical key is its own workload: it neither
|
|
2122
|
-
// returns, reveals nor blocks the first requester's.
|
|
2123
|
-
let (theirs, created) = store.admit(create_as("a2")).unwrap();
|
|
2124
|
-
assert!(created);
|
|
2125
|
-
assert_ne!(theirs.workload_id, original.workload_id);
|
|
2126
|
-
assert_eq!(store.workloads.len(), 2);
|
|
2127
|
-
|
|
2128
|
-
// A snapshot that does not bind the requester is refused before any
|
|
2129
|
-
// replay, so a key cannot be used to fetch a workload.
|
|
2130
|
-
let mut unbound = create_as("a1");
|
|
2131
|
-
unbound.requested_by = Subject::Agent("a2".into());
|
|
2132
|
-
assert_eq!(store.admit(unbound).unwrap_err().code, "WORKLOAD_UNAUTHORIZED");
|
|
2133
|
-
let _ = fs::remove_file(path);
|
|
2134
|
-
}
|
|
2135
|
-
|
|
2136
|
-
#[test]
|
|
2137
|
-
fn lease_recovery_records_the_lost_worker_and_is_bounded_by_the_attempt_budget() {
|
|
2138
|
-
let (path, mut store) = store("bounded-recovery");
|
|
2139
|
-
let workload = store.create(create()).unwrap();
|
|
2140
|
-
let id = workload.workload_id.clone();
|
|
2141
|
-
for attempt in 1..=3u32 {
|
|
2142
|
-
let now = 100 * attempt as i64;
|
|
2143
|
-
let claim = store.claim(&id, "worker-a", &capabilities(), now, 10_000).unwrap();
|
|
2144
|
-
assert_eq!(store.recover_expired(claim.expires_at).unwrap(), vec![id.clone()]);
|
|
2145
|
-
let recovered = store.get(&id).unwrap();
|
|
2146
|
-
let lost = recovered.attempts.last().unwrap();
|
|
2147
|
-
assert_eq!(lost.status, AttemptStatus::WorkerLost);
|
|
2148
|
-
assert_eq!(lost.claim_id, claim.claim_id);
|
|
2149
|
-
let recovering = recovered
|
|
2150
|
-
.execution_history
|
|
2151
|
-
.iter()
|
|
2152
|
-
.rev()
|
|
2153
|
-
.find(|event| event.to == WorkloadState::Recovering)
|
|
2154
|
-
.unwrap();
|
|
2155
|
-
assert_eq!(recovering.evidence["recovery_reason"], "lease_expired");
|
|
2156
|
-
assert_eq!(recovering.evidence["worker_id"], "worker-a");
|
|
2157
|
-
assert_eq!(recovering.evidence["claim_id"], claim.claim_id);
|
|
2158
|
-
assert_eq!(recovering.evidence["attempts_used"], attempt.to_string());
|
|
2159
|
-
let decision = &recovered.execution_history.last().unwrap();
|
|
2160
|
-
if attempt < 3 {
|
|
2161
|
-
assert_eq!(recovered.state, WorkloadState::Ready);
|
|
2162
|
-
assert_eq!(decision.evidence["decision"], "retry");
|
|
2163
|
-
} else {
|
|
2164
|
-
assert_eq!(recovered.state, WorkloadState::DeadLetter);
|
|
2165
|
-
assert_eq!(decision.evidence["decision"], "attempt_budget_exhausted");
|
|
2166
|
-
}
|
|
2167
|
-
}
|
|
2168
|
-
assert_eq!(store.get(&id).unwrap().attempt_count, 3);
|
|
2169
|
-
assert_eq!(
|
|
2170
|
-
store.claim(&id, "worker-b", &capabilities(), 1_000, 10_000).unwrap_err().code,
|
|
2171
|
-
"WORKLOAD_NOT_READY"
|
|
2172
|
-
);
|
|
2173
|
-
let _ = fs::remove_file(path);
|
|
2174
|
-
}
|
|
2175
|
-
|
|
2176
|
-
/// The recovery history of `id` as committed, read by a fresh projection.
|
|
2177
|
-
fn committed_workload(db: &FeltDb, id: &str) -> Workload {
|
|
2178
|
-
WorkloadStore::from_db(db.clone()).unwrap().get(id).unwrap().clone()
|
|
2179
|
-
}
|
|
2180
|
-
|
|
2181
|
-
fn recoveries(workload: &Workload) -> usize {
|
|
2182
|
-
workload
|
|
2183
|
-
.execution_history
|
|
2184
|
-
.iter()
|
|
2185
|
-
.filter(|event| event.to == WorkloadState::Recovering)
|
|
2186
|
-
.count()
|
|
2187
|
-
}
|
|
2188
|
-
|
|
2189
|
-
/// Two writers of one store recover the same expired workload at once.
|
|
2190
|
-
fn race_recovery(db: &FeltDb, now: i64) -> (WorkloadStore, WorkloadStore, Vec<Vec<String>>) {
|
|
2191
|
-
let stores = [
|
|
2192
|
-
WorkloadStore::from_db(db.clone()).unwrap(),
|
|
2193
|
-
WorkloadStore::from_db(db.clone()).unwrap(),
|
|
2194
|
-
];
|
|
2195
|
-
let barrier = std::sync::Barrier::new(2);
|
|
2196
|
-
let results = std::thread::scope(|scope| {
|
|
2197
|
-
let handles = stores
|
|
2198
|
-
.into_iter()
|
|
2199
|
-
.map(|mut store| {
|
|
2200
|
-
let barrier = &barrier;
|
|
2201
|
-
scope.spawn(move || {
|
|
2202
|
-
barrier.wait();
|
|
2203
|
-
let recovered = store.recover_expired(now).unwrap();
|
|
2204
|
-
(store, recovered)
|
|
2205
|
-
})
|
|
2206
|
-
})
|
|
2207
|
-
.collect::<Vec<_>>();
|
|
2208
|
-
handles
|
|
2209
|
-
.into_iter()
|
|
2210
|
-
.map(|handle| handle.join().unwrap())
|
|
2211
|
-
.collect::<Vec<_>>()
|
|
2212
|
-
});
|
|
2213
|
-
let mut results = results.into_iter();
|
|
2214
|
-
let (first, first_recovered) = results.next().unwrap();
|
|
2215
|
-
let (second, second_recovered) = results.next().unwrap();
|
|
2216
|
-
(first, second, vec![first_recovered, second_recovered])
|
|
2217
|
-
}
|
|
2218
|
-
|
|
2219
|
-
#[test]
|
|
2220
|
-
fn concurrent_recovery_commits_one_transition_and_the_loser_converges() {
|
|
2221
|
-
for round in 0..20 {
|
|
2222
|
-
let (path, mut store) = store(&format!("concurrent-recovery-{round}"));
|
|
2223
|
-
let db = store.db.clone();
|
|
2224
|
-
let id = store.create(create()).unwrap().workload_id;
|
|
2225
|
-
let claim = store.claim(&id, "worker-a", &capabilities(), 100, 10_000).unwrap();
|
|
2226
|
-
// A writer that loaded the workload before either recovery.
|
|
2227
|
-
// Commits land in this projection's window between refresh and
|
|
2228
|
-
// commit: only the storage guard can refuse them.
|
|
2229
|
-
let mut stale = WorkloadStore::from_db(db.clone())
|
|
2230
|
-
.unwrap()
|
|
2231
|
-
.without_refresh();
|
|
2232
|
-
|
|
2233
|
-
let (mut first, mut second, recovered) = race_recovery(&db, claim.expires_at);
|
|
2234
|
-
// Exactly one writer committed the recovery; the other reported none.
|
|
2235
|
-
let mut outcomes = recovered.clone();
|
|
2236
|
-
outcomes.sort();
|
|
2237
|
-
assert_eq!(outcomes, vec![vec![], vec![id.clone()]], "round {round}");
|
|
2238
|
-
|
|
2239
|
-
let committed = committed_workload(&db, &id);
|
|
2240
|
-
assert_eq!(recoveries(&committed), 1, "one recovery transition");
|
|
2241
|
-
assert_eq!(committed.state, WorkloadState::Ready);
|
|
2242
|
-
assert_eq!(committed.attempt_count, 1);
|
|
2243
|
-
assert_eq!(committed.attempts.len(), 1);
|
|
2244
|
-
assert_eq!(committed.attempts[0].status, AttemptStatus::WorkerLost);
|
|
2245
|
-
assert!(committed.current_claim.is_none());
|
|
2246
|
-
let recovering = committed
|
|
2247
|
-
.execution_history
|
|
2248
|
-
.iter()
|
|
2249
|
-
.find(|event| event.to == WorkloadState::Recovering)
|
|
2250
|
-
.unwrap();
|
|
2251
|
-
assert_eq!(recovering.evidence["claim_id"], claim.claim_id);
|
|
2252
|
-
assert_eq!(recovering.evidence["attempts_used"], "1");
|
|
2253
|
-
|
|
2254
|
-
// The losing writer adopted the committed workload instead of
|
|
2255
|
-
// keeping its own, uncommitted view.
|
|
2256
|
-
let expected = serde_json::to_value(&committed).unwrap();
|
|
2257
|
-
assert_eq!(serde_json::to_value(first.get(&id).unwrap()).unwrap(), expected);
|
|
2258
|
-
assert_eq!(serde_json::to_value(second.get(&id).unwrap()).unwrap(), expected);
|
|
2259
|
-
// And neither recovers it again, or commits anything more.
|
|
2260
|
-
assert!(first.recover_expired(claim.expires_at + 1).unwrap().is_empty());
|
|
2261
|
-
assert!(second.recover_expired(claim.expires_at + 1).unwrap().is_empty());
|
|
2262
|
-
assert_eq!(
|
|
2263
|
-
serde_json::to_value(committed_workload(&db, &id)).unwrap(),
|
|
2264
|
-
expected,
|
|
2265
|
-
"no further commit"
|
|
2266
|
-
);
|
|
2267
|
-
|
|
2268
|
-
// The lost claimant cannot complete, even through a projection
|
|
2269
|
-
// that still shows its claim as current and unexpired: the guarded
|
|
2270
|
-
// commit refuses the stale version.
|
|
2271
|
-
let refused = stale
|
|
2272
|
-
.start(&id, &claim.claim_id, "worker-a", claim.fencing_token, claim.expires_at - 1)
|
|
2273
|
-
.unwrap_err();
|
|
2274
|
-
assert_eq!(refused.code, "WORKLOAD_CONFLICT");
|
|
2275
|
-
assert_eq!(stale.get(&id).unwrap().state, WorkloadState::Ready);
|
|
2276
|
-
let refused = stale
|
|
2277
|
-
.complete(
|
|
2278
|
-
&id,
|
|
2279
|
-
&claim.claim_id,
|
|
2280
|
-
"worker-a",
|
|
2281
|
-
claim.fencing_token,
|
|
2282
|
-
"late-result",
|
|
2283
|
-
serde_json::json!({"late": true}),
|
|
2284
|
-
vec![],
|
|
2285
|
-
1,
|
|
2286
|
-
claim.expires_at - 1,
|
|
2287
|
-
)
|
|
2288
|
-
.unwrap_err();
|
|
2289
|
-
assert_eq!(refused.code, "WORKLOAD_CLAIM_INVALID");
|
|
2290
|
-
// Through a current projection its fence no longer matches.
|
|
2291
|
-
let refused = first
|
|
2292
|
-
.heartbeat(&id, &claim.claim_id, "worker-a", claim.fencing_token, claim.expires_at - 1, 10_000)
|
|
2293
|
-
.unwrap_err();
|
|
2294
|
-
assert_eq!(refused.code, "WORKLOAD_CLAIM_INVALID");
|
|
2295
|
-
assert_eq!(serde_json::to_value(committed_workload(&db, &id)).unwrap(), expected);
|
|
2296
|
-
drop((store, first, second, stale, db));
|
|
2297
|
-
let _ = fs::remove_file(path);
|
|
2298
|
-
}
|
|
2299
|
-
}
|
|
2300
|
-
|
|
2301
|
-
#[test]
|
|
2302
|
-
fn concurrent_recovery_never_exceeds_the_attempt_budget() {
|
|
2303
|
-
let (path, mut store) = store("concurrent-recovery-budget");
|
|
2304
|
-
let db = store.db.clone();
|
|
2305
|
-
let id = store.create(create()).unwrap().workload_id;
|
|
2306
|
-
for attempt in 1..=3i64 {
|
|
2307
|
-
// Each attempt is claimed by an up-to-date writer and lost.
|
|
2308
|
-
let mut claimer = WorkloadStore::from_db(db.clone()).unwrap();
|
|
2309
|
-
let claim = claimer
|
|
2310
|
-
.claim(&id, "worker-a", &capabilities(), 100 * attempt, 10_000)
|
|
2311
|
-
.unwrap();
|
|
2312
|
-
let (_, _, recovered) = race_recovery(&db, claim.expires_at);
|
|
2313
|
-
assert_eq!(recovered.concat(), vec![id.clone()], "attempt {attempt}");
|
|
2314
|
-
let committed = committed_workload(&db, &id);
|
|
2315
|
-
assert_eq!(recoveries(&committed), attempt as usize);
|
|
2316
|
-
assert_eq!(committed.attempt_count, attempt as u32);
|
|
2317
|
-
}
|
|
2318
|
-
let committed = committed_workload(&db, &id);
|
|
2319
|
-
assert_eq!(committed.state, WorkloadState::DeadLetter);
|
|
2320
|
-
assert_eq!(
|
|
2321
|
-
committed.execution_history.last().unwrap().evidence["decision"],
|
|
2322
|
-
"attempt_budget_exhausted"
|
|
2323
|
-
);
|
|
2324
|
-
assert!(committed
|
|
2325
|
-
.attempts
|
|
2326
|
-
.iter()
|
|
2327
|
-
.all(|attempt| attempt.status == AttemptStatus::WorkerLost));
|
|
2328
|
-
let refused = WorkloadStore::from_db(db.clone())
|
|
2329
|
-
.unwrap()
|
|
2330
|
-
.claim(&id, "worker-b", &capabilities(), 10_000, 10_000)
|
|
2331
|
-
.unwrap_err();
|
|
2332
|
-
assert_eq!(refused.code, "WORKLOAD_NOT_READY");
|
|
2333
|
-
assert_eq!(committed_workload(&db, &id).attempt_count, 3);
|
|
2334
|
-
drop((store, db));
|
|
2335
|
-
let _ = fs::remove_file(path);
|
|
2336
|
-
}
|
|
2337
|
-
|
|
2338
|
-
#[test]
|
|
2339
|
-
fn a_policy_that_does_not_retry_worker_failure_dead_letters_a_lost_attempt() {
|
|
2340
|
-
let (path, mut store) = store("no-worker-failure-retry");
|
|
2341
|
-
let mut request = create();
|
|
2342
|
-
request.retry_policy.retry_on = ["timeout".into()].into();
|
|
2343
|
-
let id = store.create(request).unwrap().workload_id;
|
|
2344
|
-
let claim = store.claim(&id, "worker-a", &capabilities(), 100, 10_000).unwrap();
|
|
2345
|
-
store.recover_expired(claim.expires_at).unwrap();
|
|
2346
|
-
let workload = store.get(&id).unwrap();
|
|
2347
|
-
assert_eq!(workload.state, WorkloadState::DeadLetter);
|
|
2348
|
-
assert_eq!(
|
|
2349
|
-
workload.execution_history.last().unwrap().evidence["decision"],
|
|
2350
|
-
"worker_failure_not_retryable"
|
|
2351
|
-
);
|
|
2352
|
-
let _ = fs::remove_file(path);
|
|
2353
|
-
}
|
|
2354
|
-
|
|
2355
|
-
#[test]
|
|
2356
|
-
fn claiming_exhausted_ready_work_dead_letters_it_instead() {
|
|
2357
|
-
let (path, mut store) = store("exhausted-claim");
|
|
2358
|
-
let id = store.create(create()).unwrap().workload_id;
|
|
2359
|
-
// What unbounded recovery used to leave behind: Ready, budget spent.
|
|
2360
|
-
let mut legacy = store.get(&id).unwrap().clone();
|
|
2361
|
-
legacy.attempt_count = 7;
|
|
2362
|
-
store.workloads.insert(id.clone(), legacy);
|
|
2363
|
-
store.persist_workload(&id, "fixture.legacy").unwrap();
|
|
2364
|
-
let refused = store.claim(&id, "worker-a", &capabilities(), 100, 10_000).unwrap_err();
|
|
2365
|
-
assert_eq!(refused.code, "WORKLOAD_ATTEMPTS_EXHAUSTED");
|
|
2366
|
-
let workload = store.get(&id).unwrap();
|
|
2367
|
-
assert_eq!(workload.state, WorkloadState::DeadLetter);
|
|
2368
|
-
assert_eq!(
|
|
2369
|
-
workload.execution_history.last().unwrap().evidence["reason"],
|
|
2370
|
-
"attempt_budget_exhausted"
|
|
2371
|
-
);
|
|
2372
|
-
// Durable: a restart sees the same outcome.
|
|
2373
|
-
let reopened = WorkloadStore::from_db(store.db.clone()).unwrap();
|
|
2374
|
-
assert_eq!(reopened.get(&id).unwrap().state, WorkloadState::DeadLetter);
|
|
2375
|
-
let _ = fs::remove_file(path);
|
|
2376
|
-
}
|
|
2377
|
-
|
|
2378
|
-
#[test]
|
|
2379
|
-
fn operator_retry_grants_one_attempt_is_recorded_and_is_bounded() {
|
|
2380
|
-
let (path, mut store) = store("operator-retry");
|
|
2381
|
-
let id = store.create(create()).unwrap().workload_id;
|
|
2382
|
-
// Claim and lose leases until the workload stops being claimable.
|
|
2383
|
-
fn exhaust(store: &mut WorkloadStore, id: &str, now: &mut i64) {
|
|
2384
|
-
loop {
|
|
2385
|
-
*now += 100;
|
|
2386
|
-
match store.claim(id, "worker-a", &capabilities(), *now, 10_000) {
|
|
2387
|
-
Ok(claim) => {
|
|
2388
|
-
store.recover_expired(claim.expires_at).unwrap();
|
|
2389
|
-
}
|
|
2390
|
-
Err(error) => {
|
|
2391
|
-
assert!(
|
|
2392
|
-
matches!(
|
|
2393
|
-
error.code.as_str(),
|
|
2394
|
-
"WORKLOAD_NOT_READY" | "WORKLOAD_ATTEMPTS_EXHAUSTED"
|
|
2395
|
-
),
|
|
2396
|
-
"{error:?}"
|
|
2397
|
-
);
|
|
2398
|
-
return;
|
|
2399
|
-
}
|
|
2400
|
-
}
|
|
2401
|
-
}
|
|
2402
|
-
}
|
|
2403
|
-
let mut now = 100;
|
|
2404
|
-
exhaust(&mut store, &id, &mut now);
|
|
2405
|
-
assert_eq!(store.get(&id).unwrap().attempt_count, 3);
|
|
2406
|
-
for retry in 1..=3u32 {
|
|
2407
|
-
now += 100;
|
|
2408
|
-
store
|
|
2409
|
-
.retry_with_reason(&id, "operator-1", now, Some("dependency restored"))
|
|
2410
|
-
.unwrap();
|
|
2411
|
-
let event = store.get(&id).unwrap().execution_history.last().unwrap().clone();
|
|
2412
|
-
assert_eq!(event.actor, "operator-1");
|
|
2413
|
-
assert_eq!(event.evidence["retry"], "operator");
|
|
2414
|
-
assert_eq!(event.evidence["reason"], "dependency restored");
|
|
2415
|
-
assert_eq!(event.evidence["operator_retry"], retry.to_string());
|
|
2416
|
-
exhaust(&mut store, &id, &mut now);
|
|
2417
|
-
assert_eq!(
|
|
2418
|
-
store.get(&id).unwrap().attempt_count,
|
|
2419
|
-
3 + retry,
|
|
2420
|
-
"each operator retry grants exactly one attempt"
|
|
2421
|
-
);
|
|
2422
|
-
assert_eq!(store.get(&id).unwrap().state, WorkloadState::DeadLetter);
|
|
2423
|
-
}
|
|
2424
|
-
assert_eq!(
|
|
2425
|
-
store.retry(&id, "operator-1", now + 100).unwrap_err().code,
|
|
2426
|
-
"WORKLOAD_RETRY_EXHAUSTED"
|
|
2427
|
-
);
|
|
2428
|
-
assert_eq!(store.get(&id).unwrap().state, WorkloadState::DeadLetter);
|
|
2429
|
-
let _ = fs::remove_file(path);
|
|
2430
|
-
}
|
|
2431
|
-
|
|
2432
|
-
fn store(name: &str) -> (PathBuf, WorkloadStore) {
|
|
2433
|
-
let path =
|
|
2434
|
-
std::env::temp_dir().join(format!("workload-{name}-{}.json", std::process::id()));
|
|
2435
|
-
let store = WorkloadStore::load(&path).unwrap();
|
|
2436
|
-
(path, store)
|
|
2437
|
-
}
|
|
2438
|
-
#[test]
|
|
2439
|
-
fn legacy_migration_is_idempotent_conflict_detecting_and_source_independent() {
|
|
2440
|
-
let (source_db, mut source) = store("legacy-source-builder");
|
|
2441
|
-
let workload = source.create(create()).unwrap();
|
|
2442
|
-
let legacy = LegacyWorkloadStore {
|
|
2443
|
-
workloads: source.workloads.clone(),
|
|
2444
|
-
// What a pre-requester-scope release wrote.
|
|
2445
|
-
idempotency: idempotency_index(&source.workloads, legacy_idempotency_scope),
|
|
2446
|
-
result_ids: source.result_ids.clone(),
|
|
2447
|
-
event_sequence: derived_event_sequence(&source.workloads),
|
|
2448
|
-
};
|
|
2449
|
-
let legacy_path = source_db.with_extension("legacy-workloads.json");
|
|
2450
|
-
fs::write(&legacy_path, serde_json::to_vec(&legacy).unwrap()).unwrap();
|
|
2451
|
-
let canonical_path = source_db.with_extension("canonical.log");
|
|
2452
|
-
let db = FeltDb::open(&canonical_path).unwrap();
|
|
2453
|
-
let migrated = WorkloadStore::migrate_legacy_file(db.clone(), &legacy_path).unwrap();
|
|
2454
|
-
assert_eq!(
|
|
2455
|
-
migrated.get(&workload.workload_id).unwrap().input_hash,
|
|
2456
|
-
workload.input_hash
|
|
2457
|
-
);
|
|
2458
|
-
WorkloadStore::migrate_legacy_file(db.clone(), &legacy_path).unwrap();
|
|
2459
|
-
fs::remove_file(&legacy_path).unwrap();
|
|
2460
|
-
let reopened = WorkloadStore::from_db(FeltDb::open(&canonical_path).unwrap()).unwrap();
|
|
2461
|
-
assert_eq!(
|
|
2462
|
-
reopened.get(&workload.workload_id).unwrap().workload_id,
|
|
2463
|
-
workload.workload_id
|
|
2464
|
-
);
|
|
2465
|
-
|
|
2466
|
-
let mut conflicting = legacy;
|
|
2467
|
-
conflicting
|
|
2468
|
-
.workloads
|
|
2469
|
-
.get_mut(&workload.workload_id)
|
|
2470
|
-
.unwrap()
|
|
2471
|
-
.priority += 1;
|
|
2472
|
-
fs::write(&legacy_path, serde_json::to_vec(&conflicting).unwrap()).unwrap();
|
|
2473
|
-
assert_eq!(
|
|
2474
|
-
WorkloadStore::migrate_legacy_file(db, &legacy_path)
|
|
2475
|
-
.err()
|
|
2476
|
-
.unwrap()
|
|
2477
|
-
.code,
|
|
2478
|
-
"WORKLOAD_MIGRATION_CONFLICT"
|
|
2479
|
-
);
|
|
2480
|
-
let _ = fs::remove_file(source_db);
|
|
2481
|
-
let _ = fs::remove_file(canonical_path);
|
|
2482
|
-
let _ = fs::remove_file(legacy_path);
|
|
2483
|
-
}
|
|
2484
|
-
|
|
2485
|
-
#[test]
|
|
2486
|
-
fn canonical_history_reconstructs_work_identity_provenance() {
|
|
2487
|
-
let (path, mut store) = store("provenance");
|
|
2488
|
-
let workload = store.create(create()).unwrap();
|
|
2489
|
-
let transaction = store
|
|
2490
|
-
.db
|
|
2491
|
-
.committed_transactions_after(0, usize::MAX)
|
|
2492
|
-
.unwrap()
|
|
2493
|
-
.into_iter()
|
|
2494
|
-
.last()
|
|
2495
|
-
.unwrap();
|
|
2496
|
-
let audit = transaction.audit.unwrap();
|
|
2497
|
-
assert_eq!(audit["work_id"], workload.workload_id);
|
|
2498
|
-
assert_eq!(audit["session_id"], workload.session_id.as_deref().unwrap());
|
|
2499
|
-
assert_eq!(audit["agent_id"], workload.requested_by.id());
|
|
2500
|
-
assert_eq!(
|
|
2501
|
-
audit["delegator_id"],
|
|
2502
|
-
workload.authorization_snapshot.grant.issuer.id()
|
|
2503
|
-
);
|
|
2504
|
-
assert!(transaction.commit_revision > transaction.base_revision);
|
|
2505
|
-
assert_eq!(transaction.work_id.as_deref(), Some(workload.workload_id.as_str()));
|
|
2506
|
-
assert_eq!(transaction.agent_id.as_deref(), Some(workload.requested_by.id()));
|
|
2507
|
-
assert_eq!(transaction.session_id, workload.session_id);
|
|
2508
|
-
assert_eq!(
|
|
2509
|
-
transaction.delegator_id.as_deref(),
|
|
2510
|
-
Some(workload.authorization_snapshot.grant.issuer.id())
|
|
2511
|
-
);
|
|
2512
|
-
assert_eq!(transaction.operation_ids.len(), 1);
|
|
2513
|
-
assert_ne!(transaction.operation_ids[0], transaction.transaction_id);
|
|
2514
|
-
drop(store);
|
|
2515
|
-
let reopened = WorkloadStore::load(&path).unwrap();
|
|
2516
|
-
assert_eq!(
|
|
2517
|
-
reopened.get(&workload.workload_id).unwrap().workload_id,
|
|
2518
|
-
workload.workload_id
|
|
2519
|
-
);
|
|
2520
|
-
let _ = fs::remove_file(path);
|
|
2521
|
-
}
|
|
2522
|
-
|
|
2523
|
-
#[test]
|
|
2524
|
-
fn competing_writers_use_canonical_document_cas_before_and_after_restart() {
|
|
2525
|
-
let (path, mut seed) = store("canonical-cas");
|
|
2526
|
-
let workload = seed.create(create()).unwrap();
|
|
2527
|
-
let id = workload.workload_id.clone();
|
|
2528
|
-
let db = seed.db.clone();
|
|
2529
|
-
let caps = ["email:send".into()].into();
|
|
2530
|
-
|
|
2531
|
-
let mut winner = WorkloadStore::from_db(db.clone()).unwrap();
|
|
2532
|
-
// The loser's claim races the winner's inside the window between its
|
|
2533
|
-
// refresh and its commit, which only the storage guard closes.
|
|
2534
|
-
let mut loser = WorkloadStore::from_db(db.clone())
|
|
2535
|
-
.unwrap()
|
|
2536
|
-
.without_refresh();
|
|
2537
|
-
winner.claim(&id, "worker-a", &caps, 11, 1_000).unwrap();
|
|
2538
|
-
let conflict = loser.claim(&id, "worker-b", &caps, 11, 1_000).unwrap_err();
|
|
2539
|
-
assert_eq!(conflict.code, "WORKLOAD_CONFLICT");
|
|
2540
|
-
// The loser committed nothing and now projects the winner's claim.
|
|
2541
|
-
let projected = loser.get(&id).unwrap();
|
|
2542
|
-
assert_eq!(projected.state, WorkloadState::Claimed);
|
|
2543
|
-
assert_eq!(projected.current_claim.as_ref().unwrap().worker_id, "worker-a");
|
|
2544
|
-
|
|
2545
|
-
let row = db
|
|
2546
|
-
.list_collection(DURABLE_WORK_COLLECTION)
|
|
2547
|
-
.unwrap()
|
|
2548
|
-
.into_iter()
|
|
2549
|
-
.find(|row| row.key.ends_with(&id))
|
|
2550
|
-
.unwrap();
|
|
2551
|
-
assert_eq!(row.value["__version"], 2);
|
|
2552
|
-
|
|
2553
|
-
drop((seed, winner, loser, db));
|
|
2554
|
-
let restarted_db = FeltDb::open(&path).unwrap();
|
|
2555
|
-
let mut recovery = WorkloadStore::from_db(restarted_db.clone()).unwrap();
|
|
2556
|
-
recovery.recover_expired(13).unwrap();
|
|
2557
|
-
let mut restarted_winner = WorkloadStore::from_db(restarted_db.clone()).unwrap();
|
|
2558
|
-
let mut restarted_loser = WorkloadStore::from_db(restarted_db.clone())
|
|
2559
|
-
.unwrap()
|
|
2560
|
-
.without_refresh();
|
|
2561
|
-
restarted_winner
|
|
2562
|
-
.claim(&id, "worker-c", &caps, 13, 10_000)
|
|
2563
|
-
.unwrap();
|
|
2564
|
-
let conflict = restarted_loser
|
|
2565
|
-
.claim(&id, "worker-d", &caps, 13, 10_000)
|
|
2566
|
-
.unwrap_err();
|
|
2567
|
-
assert_eq!(conflict.code, "WORKLOAD_CONFLICT");
|
|
2568
|
-
let projected = restarted_loser.get(&id).unwrap();
|
|
2569
|
-
assert_eq!(projected.state, WorkloadState::Claimed);
|
|
2570
|
-
assert_eq!(projected.current_claim.as_ref().unwrap().worker_id, "worker-c");
|
|
2571
|
-
drop((recovery, restarted_winner, restarted_loser, restarted_db));
|
|
2572
|
-
let _ = fs::remove_file(path);
|
|
2573
|
-
}
|
|
2574
|
-
|
|
2575
|
-
#[test]
|
|
2576
|
-
fn competing_admissions_share_one_canonical_idempotency_identity() {
|
|
2577
|
-
let path = std::env::temp_dir().join(format!(
|
|
2578
|
-
"workload-idempotency-cas-{}.log",
|
|
2579
|
-
std::process::id()
|
|
2580
|
-
));
|
|
2581
|
-
let db = FeltDb::open(&path).unwrap();
|
|
2582
|
-
let mut first = WorkloadStore::from_db(db.clone()).unwrap();
|
|
2583
|
-
let mut second = WorkloadStore::from_db(db.clone()).unwrap();
|
|
2584
|
-
let accepted = first.create(create()).unwrap();
|
|
2585
|
-
// A second writer of the same store loses the require-absent CAS and
|
|
2586
|
-
// converges on what was committed: a different request is refused as a
|
|
2587
|
-
// reused key, never answered with the original.
|
|
2588
|
-
let mut competing = create();
|
|
2589
|
-
competing.input = serde_json::json!({"id": 2});
|
|
2590
|
-
let conflict = second.create(competing).unwrap_err();
|
|
2591
|
-
assert_eq!(conflict.code, "IDEMPOTENCY_KEY_REUSED");
|
|
2592
|
-
assert_eq!(
|
|
2593
|
-
second.workloads.keys().collect::<Vec<_>>(),
|
|
2594
|
-
vec![&accepted.workload_id],
|
|
2595
|
-
"only the committed workload is projected"
|
|
2596
|
-
);
|
|
2597
|
-
// The same request from a third, equally stale writer converges on it.
|
|
2598
|
-
let mut third = WorkloadStore::from_db(db.clone()).unwrap();
|
|
2599
|
-
third.workloads.clear();
|
|
2600
|
-
third.idempotency.clear();
|
|
2601
|
-
third.document_versions.clear();
|
|
2602
|
-
let (converged, created) = third.admit(create()).unwrap();
|
|
2603
|
-
assert!(!created);
|
|
2604
|
-
assert_eq!(converged.workload_id, accepted.workload_id);
|
|
2605
|
-
|
|
2606
|
-
drop((first, second, third, db));
|
|
2607
|
-
let restarted = WorkloadStore::from_db(FeltDb::open(&path).unwrap()).unwrap();
|
|
2608
|
-
assert_eq!(restarted.workloads.len(), 1);
|
|
2609
|
-
assert_eq!(
|
|
2610
|
-
restarted.idempotency[&idempotency_scope("t", "app", &accepted.requested_by, "one")],
|
|
2611
|
-
accepted.workload_id
|
|
2612
|
-
);
|
|
2613
|
-
let _ = fs::remove_file(path);
|
|
2614
|
-
}
|
|
2615
|
-
|
|
2616
|
-
#[test]
|
|
2617
|
-
fn idempotency_scopes_cannot_be_made_to_coincide() {
|
|
2618
|
-
let (path, mut store) = store("scope-separators");
|
|
2619
|
-
let mut first = create_as("x:k1");
|
|
2620
|
-
first.idempotency_key = "k2".into();
|
|
2621
|
-
let mut second = create_as("x");
|
|
2622
|
-
second.idempotency_key = "k1:k2".into();
|
|
2623
|
-
let first = store.create(first).unwrap();
|
|
2624
|
-
let second = store.create(second).unwrap();
|
|
2625
|
-
assert_ne!(first.workload_id, second.workload_id);
|
|
2626
|
-
let _ = fs::remove_file(path);
|
|
2627
|
-
}
|
|
2628
|
-
|
|
2629
|
-
#[test]
|
|
2630
|
-
fn concurrent_admissions_by_competing_writers_converge_or_conflict() {
|
|
2631
|
-
for round in 0..20 {
|
|
2632
|
-
let (path, seed) = store(&format!("concurrent-admission-{round}"));
|
|
2633
|
-
let db = seed.db.clone();
|
|
2634
|
-
let admit_concurrently = |requests: [CreateWorkload; 2]| {
|
|
2635
|
-
let barrier = std::sync::Barrier::new(2);
|
|
2636
|
-
std::thread::scope(|scope| {
|
|
2637
|
-
requests
|
|
2638
|
-
.into_iter()
|
|
2639
|
-
.map(|request| {
|
|
2640
|
-
let mut writer = WorkloadStore::from_db(db.clone()).unwrap();
|
|
2641
|
-
let barrier = &barrier;
|
|
2642
|
-
scope.spawn(move || {
|
|
2643
|
-
barrier.wait();
|
|
2644
|
-
writer.admit(request)
|
|
2645
|
-
})
|
|
2646
|
-
})
|
|
2647
|
-
.collect::<Vec<_>>()
|
|
2648
|
-
.into_iter()
|
|
2649
|
-
.map(|handle| handle.join().unwrap())
|
|
2650
|
-
.collect::<Vec<_>>()
|
|
2651
|
-
})
|
|
2652
|
-
};
|
|
2653
|
-
// The same request: both writers answer with one workload, and
|
|
2654
|
-
// exactly one of them created it.
|
|
2655
|
-
let same = admit_concurrently([create(), create()]);
|
|
2656
|
-
let (first, first_created) = same[0].as_ref().unwrap().clone();
|
|
2657
|
-
let (second, second_created) = same[1].as_ref().unwrap().clone();
|
|
2658
|
-
assert_eq!(first.workload_id, second.workload_id);
|
|
2659
|
-
assert!(first_created ^ second_created, "round {round}");
|
|
2660
|
-
|
|
2661
|
-
// Different requests under one key: one is admitted, the other is
|
|
2662
|
-
// refused as a reused key and never answered with the winner.
|
|
2663
|
-
let mut left = create();
|
|
2664
|
-
left.idempotency_key = "contested".into();
|
|
2665
|
-
let mut right = left.clone();
|
|
2666
|
-
right.input = serde_json::json!({"id": 2});
|
|
2667
|
-
let contested = admit_concurrently([left, right]);
|
|
2668
|
-
let admitted = contested.iter().filter(|outcome| outcome.is_ok()).count();
|
|
2669
|
-
assert_eq!(admitted, 1, "round {round}");
|
|
2670
|
-
assert!(contested.iter().any(|outcome| outcome
|
|
2671
|
-
.as_ref()
|
|
2672
|
-
.is_err_and(|error| error.code == "IDEMPOTENCY_KEY_REUSED")));
|
|
2673
|
-
let (winner, _) = contested.iter().find_map(|o| o.as_ref().ok()).unwrap();
|
|
2674
|
-
let stored = WorkloadStore::from_db(db.clone()).unwrap();
|
|
2675
|
-
assert_eq!(stored.workloads.len(), 2);
|
|
2676
|
-
assert_eq!(stored.get(&winner.workload_id).unwrap().input, winner.input);
|
|
2677
|
-
drop((seed, stored, db));
|
|
2678
|
-
let _ = fs::remove_file(path);
|
|
2679
|
-
}
|
|
2680
|
-
}
|
|
2681
|
-
|
|
2682
|
-
#[test]
|
|
2683
|
-
fn pre_versioned_canonical_rows_upgrade_through_the_standard_cas_contract() {
|
|
2684
|
-
let (path, mut store) = store("pre-versioned-upgrade");
|
|
2685
|
-
let workload = store.create(create()).unwrap();
|
|
2686
|
-
let db = store.db.clone();
|
|
2687
|
-
db.apply_atomic_transaction(
|
|
2688
|
-
"fixture:pre-versioned-work",
|
|
2689
|
-
None,
|
|
2690
|
-
&[],
|
|
2691
|
-
&[AtomicMutation {
|
|
2692
|
-
capability: DURABLE_WORK_COLLECTION.into(),
|
|
2693
|
-
key: work_key(&workload),
|
|
2694
|
-
value: Some(serde_json::to_value(&workload).unwrap()),
|
|
2695
|
-
}],
|
|
2696
|
-
None,
|
|
2697
|
-
)
|
|
2698
|
-
.unwrap();
|
|
2699
|
-
|
|
2700
|
-
let mut upgraded = WorkloadStore::from_db(db.clone()).unwrap();
|
|
2701
|
-
upgraded
|
|
2702
|
-
.claim(
|
|
2703
|
-
&workload.workload_id,
|
|
2704
|
-
"worker",
|
|
2705
|
-
&["email:send".into()].into(),
|
|
2706
|
-
11,
|
|
2707
|
-
10_000,
|
|
2708
|
-
)
|
|
2709
|
-
.unwrap();
|
|
2710
|
-
let row = db
|
|
2711
|
-
.list_collection(DURABLE_WORK_COLLECTION)
|
|
2712
|
-
.unwrap()
|
|
2713
|
-
.into_iter()
|
|
2714
|
-
.find(|row| row.key.ends_with(&workload.workload_id))
|
|
2715
|
-
.unwrap();
|
|
2716
|
-
assert_eq!(row.value["__version"], 2);
|
|
2717
|
-
let _ = fs::remove_file(path);
|
|
2718
|
-
}
|
|
2719
|
-
|
|
2720
|
-
#[test]
|
|
2721
|
-
fn derived_indexes_rebuild_exclusively_from_canonical_work() {
|
|
2722
|
-
let (path, mut store) = store("derived-index-rebuild");
|
|
2723
|
-
let workload = store.create(create()).unwrap();
|
|
2724
|
-
let caps = ["email:send".into()].into();
|
|
2725
|
-
let claim = store
|
|
2726
|
-
.claim(&workload.workload_id, "worker", &caps, 11, 10_000)
|
|
2727
|
-
.unwrap();
|
|
2728
|
-
store
|
|
2729
|
-
.start(
|
|
2730
|
-
&workload.workload_id,
|
|
2731
|
-
&claim.claim_id,
|
|
2732
|
-
"worker",
|
|
2733
|
-
claim.fencing_token,
|
|
2734
|
-
12,
|
|
2735
|
-
)
|
|
2736
|
-
.unwrap();
|
|
2737
|
-
store
|
|
2738
|
-
.complete(
|
|
2739
|
-
&workload.workload_id,
|
|
2740
|
-
&claim.claim_id,
|
|
2741
|
-
"worker",
|
|
2742
|
-
claim.fencing_token,
|
|
2743
|
-
"result-rebuilt",
|
|
2744
|
-
serde_json::json!({"ok": true}),
|
|
2745
|
-
vec![],
|
|
2746
|
-
1,
|
|
2747
|
-
13,
|
|
2748
|
-
)
|
|
2749
|
-
.unwrap();
|
|
2750
|
-
|
|
2751
|
-
store.idempotency.clear();
|
|
2752
|
-
store.result_ids.clear();
|
|
2753
|
-
store.document_versions.clear();
|
|
2754
|
-
let rebuilt = WorkloadStore::from_db(store.db.clone()).unwrap();
|
|
2755
|
-
assert_eq!(
|
|
2756
|
-
rebuilt
|
|
2757
|
-
.idempotency
|
|
2758
|
-
.get(&idempotency_scope("t", "app", &workload.requested_by, "one")),
|
|
2759
|
-
Some(&workload.workload_id)
|
|
2760
|
-
);
|
|
2761
|
-
assert_eq!(
|
|
2762
|
-
rebuilt.result_ids.get("result-rebuilt"),
|
|
2763
|
-
Some(&workload.workload_id)
|
|
2764
|
-
);
|
|
2765
|
-
assert!(rebuilt.document_versions[&workload.workload_id] > 1);
|
|
2766
|
-
assert_eq!(
|
|
2767
|
-
rebuilt
|
|
2768
|
-
.get(&workload.workload_id)
|
|
2769
|
-
.unwrap()
|
|
2770
|
-
.execution_history
|
|
2771
|
-
.iter()
|
|
2772
|
-
.map(|event| event.sequence)
|
|
2773
|
-
.collect::<Vec<_>>(),
|
|
2774
|
-
vec![1, 2, 3, 4]
|
|
2775
|
-
);
|
|
2776
|
-
let duplicate = WorkloadStore::from_db(store.db.clone())
|
|
2777
|
-
.unwrap()
|
|
2778
|
-
.create(create())
|
|
2779
|
-
.unwrap();
|
|
2780
|
-
assert_eq!(duplicate.workload_id, workload.workload_id);
|
|
2781
|
-
let _ = fs::remove_file(path);
|
|
2782
|
-
}
|
|
2783
|
-
#[test]
|
|
2784
|
-
fn idempotent_create_and_legal_lifecycle() {
|
|
2785
|
-
let (path, mut store) = store("lifecycle");
|
|
2786
|
-
let one = store.create(create()).unwrap();
|
|
2787
|
-
let two = store.create(create()).unwrap();
|
|
2788
|
-
assert_eq!(one.workload_id, two.workload_id);
|
|
2789
|
-
let caps = ["email:send".into()].into();
|
|
2790
|
-
let claim = store
|
|
2791
|
-
.claim(&one.workload_id, "worker", &caps, 11, 10_000)
|
|
2792
|
-
.unwrap();
|
|
2793
|
-
store
|
|
2794
|
-
.start(
|
|
2795
|
-
&one.workload_id,
|
|
2796
|
-
&claim.claim_id,
|
|
2797
|
-
"worker",
|
|
2798
|
-
claim.fencing_token,
|
|
2799
|
-
12,
|
|
2800
|
-
)
|
|
2801
|
-
.unwrap();
|
|
2802
|
-
let result = store
|
|
2803
|
-
.complete(
|
|
2804
|
-
&one.workload_id,
|
|
2805
|
-
&claim.claim_id,
|
|
2806
|
-
"worker",
|
|
2807
|
-
claim.fencing_token,
|
|
2808
|
-
"result-1",
|
|
2809
|
-
serde_json::json!({"ok":true}),
|
|
2810
|
-
vec![],
|
|
2811
|
-
1,
|
|
2812
|
-
13,
|
|
2813
|
-
)
|
|
2814
|
-
.unwrap();
|
|
2815
|
-
assert_eq!(result.result_id, "result-1");
|
|
2816
|
-
assert!(store
|
|
2817
|
-
.start(
|
|
2818
|
-
&one.workload_id,
|
|
2819
|
-
&claim.claim_id,
|
|
2820
|
-
"worker",
|
|
2821
|
-
claim.fencing_token,
|
|
2822
|
-
14
|
|
2823
|
-
)
|
|
2824
|
-
.is_err());
|
|
2825
|
-
let recovered = WorkloadStore::load(&path).unwrap();
|
|
2826
|
-
assert_eq!(
|
|
2827
|
-
recovered.get(&one.workload_id).unwrap().state,
|
|
2828
|
-
WorkloadState::Succeeded
|
|
2829
|
-
);
|
|
2830
|
-
let _ = fs::remove_file(path);
|
|
2831
|
-
}
|
|
2832
|
-
#[test]
|
|
2833
|
-
fn stale_fence_and_lease_recovery() {
|
|
2834
|
-
let (path, mut store) = store("fence");
|
|
2835
|
-
let w = store.create(create()).unwrap();
|
|
2836
|
-
let caps = ["email:send".into()].into();
|
|
2837
|
-
let old = store.claim(&w.workload_id, "a", &caps, 11, 1000).unwrap();
|
|
2838
|
-
store
|
|
2839
|
-
.start(&w.workload_id, &old.claim_id, "a", old.fencing_token, 11)
|
|
2840
|
-
.unwrap();
|
|
2841
|
-
store.recover_expired(13).unwrap();
|
|
2842
|
-
let new = store.claim(&w.workload_id, "b", &caps, 13, 10_000).unwrap();
|
|
2843
|
-
assert!(new.fencing_token > old.fencing_token);
|
|
2844
|
-
assert_eq!(
|
|
2845
|
-
store
|
|
2846
|
-
.complete(
|
|
2847
|
-
&w.workload_id,
|
|
2848
|
-
&old.claim_id,
|
|
2849
|
-
"a",
|
|
2850
|
-
old.fencing_token,
|
|
2851
|
-
"stale",
|
|
2852
|
-
Value::Null,
|
|
2853
|
-
vec![],
|
|
2854
|
-
1,
|
|
2855
|
-
14
|
|
2856
|
-
)
|
|
2857
|
-
.unwrap_err()
|
|
2858
|
-
.code,
|
|
2859
|
-
"WORKLOAD_STALE_FENCE"
|
|
2860
|
-
);
|
|
2861
|
-
let _ = fs::remove_file(path);
|
|
2862
|
-
}
|
|
2863
|
-
#[test]
|
|
2864
|
-
fn retry_dead_letter_checkpoint_and_secret_safety() {
|
|
2865
|
-
let (path, mut store) = store("retry");
|
|
2866
|
-
let mut input = create();
|
|
2867
|
-
input.retry_policy.max_attempts = 1;
|
|
2868
|
-
let w = store.create(input).unwrap();
|
|
2869
|
-
let caps = ["email:send".into()].into();
|
|
2870
|
-
let claim = store.claim(&w.workload_id, "a", &caps, 11, 10_000).unwrap();
|
|
2871
|
-
store
|
|
2872
|
-
.start(
|
|
2873
|
-
&w.workload_id,
|
|
2874
|
-
&claim.claim_id,
|
|
2875
|
-
"a",
|
|
2876
|
-
claim.fencing_token,
|
|
2877
|
-
12,
|
|
2878
|
-
)
|
|
2879
|
-
.unwrap();
|
|
2880
|
-
assert!(store
|
|
2881
|
-
.checkpoint(
|
|
2882
|
-
&w.workload_id,
|
|
2883
|
-
&claim.claim_id,
|
|
2884
|
-
"a",
|
|
2885
|
-
claim.fencing_token,
|
|
2886
|
-
serde_json::json!({"password":"bad"}),
|
|
2887
|
-
13
|
|
2888
|
-
)
|
|
2889
|
-
.is_err());
|
|
2890
|
-
assert_eq!(
|
|
2891
|
-
store
|
|
2892
|
-
.fail(
|
|
2893
|
-
&w.workload_id,
|
|
2894
|
-
&claim.claim_id,
|
|
2895
|
-
"a",
|
|
2896
|
-
claim.fencing_token,
|
|
2897
|
-
"worker_failure",
|
|
2898
|
-
"crashed",
|
|
2899
|
-
13
|
|
2900
|
-
)
|
|
2901
|
-
.unwrap(),
|
|
2902
|
-
WorkloadState::DeadLetter
|
|
2903
|
-
);
|
|
2904
|
-
let _ = fs::remove_file(path);
|
|
2905
|
-
}
|
|
2906
|
-
#[test]
|
|
2907
|
-
fn a_repeated_completion_replays_only_for_the_claim_and_output_that_produced_it() {
|
|
2908
|
-
let (path, mut store) = store("complete-replay");
|
|
2909
|
-
let w = store.create(create()).unwrap();
|
|
2910
|
-
let id = w.workload_id.clone();
|
|
2911
|
-
let caps = capabilities();
|
|
2912
|
-
// Worker a loses its lease; worker b completes the work.
|
|
2913
|
-
let lost = store.claim(&id, "a", &caps, 11, 1000).unwrap();
|
|
2914
|
-
store
|
|
2915
|
-
.start(&id, &lost.claim_id, "a", lost.fencing_token, 11)
|
|
2916
|
-
.unwrap();
|
|
2917
|
-
store.recover_expired(13).unwrap();
|
|
2918
|
-
let won = store.claim(&id, "b", &caps, 13, 10_000).unwrap();
|
|
2919
|
-
store
|
|
2920
|
-
.start(&id, &won.claim_id, "b", won.fencing_token, 13)
|
|
2921
|
-
.unwrap();
|
|
2922
|
-
let output = serde_json::json!({"sent": 1});
|
|
2923
|
-
let first = store
|
|
2924
|
-
.complete(
|
|
2925
|
-
&id,
|
|
2926
|
-
&won.claim_id,
|
|
2927
|
-
"b",
|
|
2928
|
-
won.fencing_token,
|
|
2929
|
-
"r1",
|
|
2930
|
-
output.clone(),
|
|
2931
|
-
vec![],
|
|
2932
|
-
1,
|
|
2933
|
-
14,
|
|
2934
|
-
)
|
|
2935
|
-
.unwrap();
|
|
2936
|
-
// The reporting attempt retrying after a lost response gets the same result.
|
|
2937
|
-
let replay = store
|
|
2938
|
-
.complete(
|
|
2939
|
-
&id,
|
|
2940
|
-
&won.claim_id,
|
|
2941
|
-
"b",
|
|
2942
|
-
won.fencing_token,
|
|
2943
|
-
"r1",
|
|
2944
|
-
output.clone(),
|
|
2945
|
-
vec![],
|
|
2946
|
-
1,
|
|
2947
|
-
15,
|
|
2948
|
-
)
|
|
2949
|
-
.unwrap();
|
|
2950
|
-
assert_eq!(replay.output_hash, first.output_hash);
|
|
2951
|
-
assert_eq!(replay.attempt_id, first.attempt_id);
|
|
2952
|
-
// The same result id with a different output is a conflict, not a replay.
|
|
2953
|
-
assert_eq!(
|
|
2954
|
-
store
|
|
2955
|
-
.complete(
|
|
2956
|
-
&id,
|
|
2957
|
-
&won.claim_id,
|
|
2958
|
-
"b",
|
|
2959
|
-
won.fencing_token,
|
|
2960
|
-
"r1",
|
|
2961
|
-
serde_json::json!({"sent": 2}),
|
|
2962
|
-
vec![],
|
|
2963
|
-
1,
|
|
2964
|
-
15
|
|
2965
|
-
)
|
|
2966
|
-
.unwrap_err()
|
|
2967
|
-
.code,
|
|
2968
|
-
"WORKLOAD_RESULT_CONFLICT"
|
|
2969
|
-
);
|
|
2970
|
-
// A stale worker naming the winner's result id learns nothing landed for it.
|
|
2971
|
-
assert_eq!(
|
|
2972
|
-
store
|
|
2973
|
-
.complete(
|
|
2974
|
-
&id,
|
|
2975
|
-
&lost.claim_id,
|
|
2976
|
-
"a",
|
|
2977
|
-
lost.fencing_token,
|
|
2978
|
-
"r1",
|
|
2979
|
-
output.clone(),
|
|
2980
|
-
vec![],
|
|
2981
|
-
1,
|
|
2982
|
-
15
|
|
2983
|
-
)
|
|
2984
|
-
.unwrap_err()
|
|
2985
|
-
.code,
|
|
2986
|
-
"WORKLOAD_RESULT_CONFLICT"
|
|
2987
|
-
);
|
|
2988
|
-
let committed = store.get(&id).unwrap();
|
|
2989
|
-
assert_eq!(committed.state, WorkloadState::Succeeded);
|
|
2990
|
-
assert_eq!(committed.result.as_ref().unwrap().output, output);
|
|
2991
|
-
assert_eq!(committed.result.as_ref().unwrap().produced_by, "b");
|
|
2992
|
-
let _ = fs::remove_file(path);
|
|
2993
|
-
}
|
|
2994
|
-
#[test]
|
|
2995
|
-
fn a_resumed_attempt_reads_the_input_and_the_lost_attempts_checkpoints_through_its_fence() {
|
|
2996
|
-
let (path, mut store) = store("resume-context");
|
|
2997
|
-
let w = store.create(create()).unwrap();
|
|
2998
|
-
let id = w.workload_id.clone();
|
|
2999
|
-
let caps = capabilities();
|
|
3000
|
-
let lost = store.claim(&id, "a", &caps, 11, 1000).unwrap();
|
|
3001
|
-
store
|
|
3002
|
-
.start(&id, &lost.claim_id, "a", lost.fencing_token, 11)
|
|
3003
|
-
.unwrap();
|
|
3004
|
-
store
|
|
3005
|
-
.checkpoint(
|
|
3006
|
-
&id,
|
|
3007
|
-
&lost.claim_id,
|
|
3008
|
-
"a",
|
|
3009
|
-
lost.fencing_token,
|
|
3010
|
-
serde_json::json!({"step": "reserve", "status": "done"}),
|
|
3011
|
-
11,
|
|
3012
|
-
)
|
|
3013
|
-
.unwrap();
|
|
3014
|
-
store.recover_expired(13).unwrap();
|
|
3015
|
-
let next = store.claim(&id, "b", &caps, 13, 10_000).unwrap();
|
|
3016
|
-
let context = store
|
|
3017
|
-
.execution_context(&id, &next.claim_id, "b", next.fencing_token, 13, vec![])
|
|
3018
|
-
.unwrap();
|
|
3019
|
-
assert_eq!(context.input, serde_json::json!({"id": 1}));
|
|
3020
|
-
assert_eq!(context.checkpoints.len(), 1);
|
|
3021
|
-
assert_eq!(context.checkpoints[0].state["step"], "reserve");
|
|
3022
|
-
assert_ne!(
|
|
3023
|
-
context.checkpoints[0].attempt_id, context.attempt_id,
|
|
3024
|
-
"written by the lost attempt"
|
|
3025
|
-
);
|
|
3026
|
-
// The lost worker's fence reads nothing.
|
|
3027
|
-
assert_eq!(
|
|
3028
|
-
store
|
|
3029
|
-
.execution_context(&id, &lost.claim_id, "a", lost.fencing_token, 13, vec![])
|
|
3030
|
-
.unwrap_err()
|
|
3031
|
-
.code,
|
|
3032
|
-
"WORKLOAD_STALE_FENCE"
|
|
3033
|
-
);
|
|
3034
|
-
let _ = fs::remove_file(path);
|
|
3035
|
-
}
|
|
3036
|
-
#[test]
|
|
3037
|
-
fn cancellation_race_and_duplicate_completion_are_deterministic() {
|
|
3038
|
-
let (path, mut store) = store("cancel");
|
|
3039
|
-
let w = store.create(create()).unwrap();
|
|
3040
|
-
let caps = ["email:send".into()].into();
|
|
3041
|
-
let claim = store.claim(&w.workload_id, "a", &caps, 11, 10_000).unwrap();
|
|
3042
|
-
store
|
|
3043
|
-
.start(
|
|
3044
|
-
&w.workload_id,
|
|
3045
|
-
&claim.claim_id,
|
|
3046
|
-
"a",
|
|
3047
|
-
claim.fencing_token,
|
|
3048
|
-
12,
|
|
3049
|
-
)
|
|
3050
|
-
.unwrap();
|
|
3051
|
-
assert_eq!(
|
|
3052
|
-
store.cancel(&w.workload_id, "user", 13).unwrap(),
|
|
3053
|
-
WorkloadState::CancellationRequested
|
|
3054
|
-
);
|
|
3055
|
-
assert_eq!(
|
|
3056
|
-
store
|
|
3057
|
-
.complete(
|
|
3058
|
-
&w.workload_id,
|
|
3059
|
-
&claim.claim_id,
|
|
3060
|
-
"a",
|
|
3061
|
-
claim.fencing_token,
|
|
3062
|
-
"r",
|
|
3063
|
-
Value::Null,
|
|
3064
|
-
vec![],
|
|
3065
|
-
1,
|
|
3066
|
-
14
|
|
3067
|
-
)
|
|
3068
|
-
.unwrap_err()
|
|
3069
|
-
.code,
|
|
3070
|
-
"WORKLOAD_TRANSITION_DENIED"
|
|
3071
|
-
);
|
|
3072
|
-
store
|
|
3073
|
-
.confirm_cancel(
|
|
3074
|
-
&w.workload_id,
|
|
3075
|
-
&claim.claim_id,
|
|
3076
|
-
"a",
|
|
3077
|
-
claim.fencing_token,
|
|
3078
|
-
14,
|
|
3079
|
-
)
|
|
3080
|
-
.unwrap();
|
|
3081
|
-
assert_eq!(
|
|
3082
|
-
store.get(&w.workload_id).unwrap().state,
|
|
3083
|
-
WorkloadState::Cancelled
|
|
3084
|
-
);
|
|
3085
|
-
let _ = fs::remove_file(path);
|
|
3086
|
-
}
|
|
3087
|
-
#[test]
|
|
3088
|
-
fn scheduler_respects_dependencies_and_tenant_application_isolation() {
|
|
3089
|
-
let (path, mut store) = store("dependency");
|
|
3090
|
-
let parent = store.create(create()).unwrap();
|
|
3091
|
-
let mut child = create();
|
|
3092
|
-
child.idempotency_key = "child".into();
|
|
3093
|
-
child.dependencies = vec![WorkloadDependency {
|
|
3094
|
-
workload_id: parent.workload_id.clone(),
|
|
3095
|
-
required_state: WorkloadState::Succeeded,
|
|
3096
|
-
}];
|
|
3097
|
-
let child = store.create(child).unwrap();
|
|
3098
|
-
assert_eq!(store.list("other", "app").len(), 0);
|
|
3099
|
-
let caps = ["email:send".into()].into();
|
|
3100
|
-
let mut scheduler = LocalDurableScheduler::default();
|
|
3101
|
-
let first = scheduler
|
|
3102
|
-
.claim(&mut store, "t", "app", "worker", &caps, 11, 10_000)
|
|
3103
|
-
.unwrap()
|
|
3104
|
-
.unwrap();
|
|
3105
|
-
assert_eq!(first.workload_id, parent.workload_id);
|
|
3106
|
-
assert_ne!(first.workload_id, child.workload_id);
|
|
3107
|
-
let _ = fs::remove_file(path);
|
|
3108
|
-
}
|
|
3109
|
-
|
|
3110
|
-
// ------------------------------------------------------------------
|
|
3111
|
-
// Temporal work eligibility.
|
|
3112
|
-
//
|
|
3113
|
-
// The invariant under test: a workload may exist durably before it is
|
|
3114
|
-
// executable. Every test below drives the real store — creating, claiming,
|
|
3115
|
-
// reloading from disk — rather than asserting that a field or a function
|
|
3116
|
-
// name exists.
|
|
3117
|
-
// ------------------------------------------------------------------
|
|
3118
|
-
|
|
3119
|
-
/// A store on a path unique to this test, cleared first so a recycled
|
|
3120
|
-
/// process id cannot hand a test a previous run's file.
|
|
3121
|
-
fn temporal_store(name: &str) -> (PathBuf, WorkloadStore) {
|
|
3122
|
-
let path = std::env::temp_dir().join(format!(
|
|
3123
|
-
"workload-temporal-{name}-{}-{}.json",
|
|
3124
|
-
std::process::id(),
|
|
3125
|
-
rand::random::<u64>()
|
|
3126
|
-
));
|
|
3127
|
-
let _ = fs::remove_file(&path);
|
|
3128
|
-
(path.clone(), WorkloadStore::load(&path).unwrap())
|
|
3129
|
-
}
|
|
3130
|
-
|
|
3131
|
-
fn scheduled(not_before: i64) -> CreateWorkload {
|
|
3132
|
-
let mut input = create();
|
|
3133
|
-
input.not_before = Some(not_before);
|
|
3134
|
-
input
|
|
3135
|
-
}
|
|
3136
|
-
|
|
3137
|
-
fn caps() -> BTreeSet<String> {
|
|
3138
|
-
["email:send".into()].into()
|
|
3139
|
-
}
|
|
3140
|
-
|
|
3141
|
-
#[test]
|
|
3142
|
-
fn scheduled_work_is_not_claimable_before_not_before() {
|
|
3143
|
-
let (path, mut store) = temporal_store("before");
|
|
3144
|
-
let workload = store.create(scheduled(2_000)).unwrap();
|
|
3145
|
-
|
|
3146
|
-
// Durable and Ready immediately: existing before it is executable is
|
|
3147
|
-
// the whole point.
|
|
3148
|
-
assert_eq!(workload.state, WorkloadState::Ready);
|
|
3149
|
-
assert_eq!(workload.next_attempt_at, Some(2_000));
|
|
3150
|
-
|
|
3151
|
-
let refused = store
|
|
3152
|
-
.claim(&workload.workload_id, "worker", &caps(), 1_999, 10_000)
|
|
3153
|
-
.unwrap_err();
|
|
3154
|
-
assert_eq!(refused.code, "WORKLOAD_NOT_YET_ELIGIBLE");
|
|
3155
|
-
assert!(
|
|
3156
|
-
refused.message.contains("2000"),
|
|
3157
|
-
"the refusal names the instant the worker should return at: {}",
|
|
3158
|
-
refused.message
|
|
3159
|
-
);
|
|
3160
|
-
|
|
3161
|
-
// Refusing a claim is not a state change. Nothing executed.
|
|
3162
|
-
let after = store.get(&workload.workload_id).unwrap();
|
|
3163
|
-
assert_eq!(after.state, WorkloadState::Ready);
|
|
3164
|
-
assert_eq!(after.attempt_count, 0);
|
|
3165
|
-
assert!(after.attempts.is_empty());
|
|
3166
|
-
assert!(after.current_claim.is_none());
|
|
3167
|
-
let _ = fs::remove_file(path);
|
|
3168
|
-
}
|
|
3169
|
-
|
|
3170
|
-
#[test]
|
|
3171
|
-
fn scheduled_work_becomes_claimable_after_not_before() {
|
|
3172
|
-
let (path, mut store) = temporal_store("after");
|
|
3173
|
-
let workload = store.create(scheduled(2_000)).unwrap();
|
|
3174
|
-
|
|
3175
|
-
// The boundary belongs to the eligible side: at exactly `not_before`
|
|
3176
|
-
// the workload is claimable, so a worker polling on the second does not
|
|
3177
|
-
// have to wait a further tick.
|
|
3178
|
-
let claim = store
|
|
3179
|
-
.claim(&workload.workload_id, "worker", &caps(), 2_000, 10_000)
|
|
3180
|
-
.expect("eligible at the instant itself");
|
|
3181
|
-
assert_eq!(claim.worker_id, "worker");
|
|
3182
|
-
assert_eq!(
|
|
3183
|
-
store.get(&workload.workload_id).unwrap().state,
|
|
3184
|
-
WorkloadState::Claimed
|
|
3185
|
-
);
|
|
3186
|
-
let _ = fs::remove_file(path);
|
|
3187
|
-
}
|
|
3188
|
-
|
|
3189
|
-
#[test]
|
|
3190
|
-
fn scheduled_work_survives_restart_before_eligibility() {
|
|
3191
|
-
let (path, mut store) = temporal_store("restart-before");
|
|
3192
|
-
let id = store.create(scheduled(2_000)).unwrap().workload_id;
|
|
3193
|
-
drop(store);
|
|
3194
|
-
|
|
3195
|
-
// Restart the authority. The workload is still here, still Ready, still
|
|
3196
|
-
// carrying its eligibility, and still refused.
|
|
3197
|
-
let mut reopened = WorkloadStore::load(&path).unwrap();
|
|
3198
|
-
let recovered = reopened.get(&id).unwrap();
|
|
3199
|
-
assert_eq!(recovered.state, WorkloadState::Ready);
|
|
3200
|
-
assert_eq!(recovered.next_attempt_at, Some(2_000));
|
|
3201
|
-
|
|
3202
|
-
assert_eq!(
|
|
3203
|
-
reopened
|
|
3204
|
-
.claim(&id, "worker", &caps(), 1_999, 10_000)
|
|
3205
|
-
.unwrap_err()
|
|
3206
|
-
.code,
|
|
3207
|
-
"WORKLOAD_NOT_YET_ELIGIBLE"
|
|
3208
|
-
);
|
|
3209
|
-
let _ = fs::remove_file(path);
|
|
3210
|
-
}
|
|
3211
|
-
|
|
3212
|
-
#[test]
|
|
3213
|
-
fn scheduled_work_executes_after_restart() {
|
|
3214
|
-
let (path, mut store) = temporal_store("restart-after");
|
|
3215
|
-
let id = store.create(scheduled(2_000)).unwrap().workload_id;
|
|
3216
|
-
drop(store);
|
|
3217
|
-
|
|
3218
|
-
let mut reopened = WorkloadStore::load(&path).unwrap();
|
|
3219
|
-
let claim = reopened
|
|
3220
|
-
.claim(&id, "worker", &caps(), 2_001, 10_000)
|
|
3221
|
-
.expect("claimable after restart once eligible");
|
|
3222
|
-
reopened
|
|
3223
|
-
.start(&id, &claim.claim_id, "worker", claim.fencing_token, 2_002)
|
|
3224
|
-
.unwrap();
|
|
3225
|
-
reopened
|
|
3226
|
-
.complete(
|
|
3227
|
-
&id,
|
|
3228
|
-
&claim.claim_id,
|
|
3229
|
-
"worker",
|
|
3230
|
-
claim.fencing_token,
|
|
3231
|
-
"result-scheduled",
|
|
3232
|
-
serde_json::json!({"ok": true}),
|
|
3233
|
-
vec![],
|
|
3234
|
-
1,
|
|
3235
|
-
2_003,
|
|
3236
|
-
)
|
|
3237
|
-
.unwrap();
|
|
3238
|
-
|
|
3239
|
-
// And the outcome is durable across a second restart.
|
|
3240
|
-
let again = WorkloadStore::load(&path).unwrap();
|
|
3241
|
-
let finished = again.get(&id).unwrap();
|
|
3242
|
-
assert_eq!(finished.state, WorkloadState::Succeeded);
|
|
3243
|
-
assert_eq!(
|
|
3244
|
-
finished.result.as_ref().map(|r| r.result_id.as_str()),
|
|
3245
|
-
Some("result-scheduled")
|
|
3246
|
-
);
|
|
3247
|
-
let _ = fs::remove_file(path);
|
|
3248
|
-
}
|
|
3249
|
-
|
|
3250
|
-
#[test]
|
|
3251
|
-
fn immediate_work_behavior_is_unchanged() {
|
|
3252
|
-
let (path, mut store) = temporal_store("immediate");
|
|
3253
|
-
// `create()` sets no `not_before`, which is what every caller did
|
|
3254
|
-
// before this field existed.
|
|
3255
|
-
let workload = store.create(create()).unwrap();
|
|
3256
|
-
assert_eq!(workload.next_attempt_at, None);
|
|
3257
|
-
assert!(workload.is_eligible_at(workload.created_at));
|
|
3258
|
-
|
|
3259
|
-
// Claimable at its own creation instant, with no waiting.
|
|
3260
|
-
store
|
|
3261
|
-
.claim(
|
|
3262
|
-
&workload.workload_id,
|
|
3263
|
-
"worker",
|
|
3264
|
-
&caps(),
|
|
3265
|
-
workload.created_at,
|
|
3266
|
-
10_000,
|
|
3267
|
-
)
|
|
3268
|
-
.expect("work with no eligibility instant is claimable immediately");
|
|
3269
|
-
let _ = fs::remove_file(path);
|
|
3270
|
-
}
|
|
3271
|
-
|
|
3272
|
-
#[test]
|
|
3273
|
-
fn multiple_workers_claim_temporally_eligible_work_once() {
|
|
3274
|
-
let (path, mut store) = temporal_store("contention");
|
|
3275
|
-
let id = store.create(scheduled(2_000)).unwrap().workload_id;
|
|
3276
|
-
|
|
3277
|
-
// Both workers are polling and both see eligibility open at the same
|
|
3278
|
-
// instant. Exactly one wins.
|
|
3279
|
-
let winner = store
|
|
3280
|
-
.claim(&id, "worker-a", &caps(), 2_000, 10_000)
|
|
3281
|
-
.unwrap();
|
|
3282
|
-
let loser = store
|
|
3283
|
-
.claim(&id, "worker-b", &caps(), 2_000, 10_000)
|
|
3284
|
-
.unwrap_err();
|
|
3285
|
-
assert_eq!(
|
|
3286
|
-
loser.code, "WORKLOAD_NOT_READY",
|
|
3287
|
-
"the second worker is refused on state, not on time: the workload is claimed"
|
|
3288
|
-
);
|
|
3289
|
-
|
|
3290
|
-
// And the loser cannot complete work it never owned.
|
|
3291
|
-
assert_eq!(
|
|
3292
|
-
store
|
|
3293
|
-
.complete(
|
|
3294
|
-
&id,
|
|
3295
|
-
&winner.claim_id,
|
|
3296
|
-
"worker-b",
|
|
3297
|
-
winner.fencing_token,
|
|
3298
|
-
"duplicate",
|
|
3299
|
-
Value::Null,
|
|
3300
|
-
vec![],
|
|
3301
|
-
1,
|
|
3302
|
-
2_001,
|
|
3303
|
-
)
|
|
3304
|
-
.unwrap_err()
|
|
3305
|
-
.code,
|
|
3306
|
-
"WORKLOAD_STALE_FENCE"
|
|
3307
|
-
);
|
|
3308
|
-
assert_eq!(store.get(&id).unwrap().attempt_count, 1);
|
|
3309
|
-
let _ = fs::remove_file(path);
|
|
3310
|
-
}
|
|
3311
|
-
|
|
3312
|
-
#[test]
|
|
3313
|
-
fn retry_sets_next_eligibility_without_duplicate_execution() {
|
|
3314
|
-
let (path, mut store) = temporal_store("retry");
|
|
3315
|
-
let id = store.create(scheduled(2_000)).unwrap().workload_id;
|
|
3316
|
-
|
|
3317
|
-
let first = store
|
|
3318
|
-
.claim(&id, "worker-a", &caps(), 2_000, 10_000)
|
|
3319
|
-
.unwrap();
|
|
3320
|
-
store
|
|
3321
|
-
.start(&id, &first.claim_id, "worker-a", first.fencing_token, 2_001)
|
|
3322
|
-
.unwrap();
|
|
3323
|
-
let state = store
|
|
3324
|
-
.fail(
|
|
3325
|
-
&id,
|
|
3326
|
-
&first.claim_id,
|
|
3327
|
-
"worker-a",
|
|
3328
|
-
first.fencing_token,
|
|
3329
|
-
"timeout",
|
|
3330
|
-
"upstream did not answer",
|
|
3331
|
-
2_002,
|
|
3332
|
-
)
|
|
3333
|
-
.unwrap();
|
|
3334
|
-
|
|
3335
|
-
// Retry re-arms the same eligibility field creation used, so there is
|
|
3336
|
-
// one temporal concept rather than two competing ones.
|
|
3337
|
-
assert_eq!(state, WorkloadState::Ready);
|
|
3338
|
-
let backing_off = store.get(&id).unwrap();
|
|
3339
|
-
let next = backing_off.next_attempt_at.expect("retry set eligibility");
|
|
3340
|
-
assert!(
|
|
3341
|
-
next > 2_002,
|
|
3342
|
-
"backoff pushed eligibility into the future: {next}"
|
|
3343
|
-
);
|
|
3344
|
-
|
|
3345
|
-
assert_eq!(
|
|
3346
|
-
store
|
|
3347
|
-
.claim(&id, "worker-b", &caps(), next - 1, 10_000)
|
|
3348
|
-
.unwrap_err()
|
|
3349
|
-
.code,
|
|
3350
|
-
"WORKLOAD_NOT_YET_ELIGIBLE"
|
|
3351
|
-
);
|
|
3352
|
-
|
|
3353
|
-
let second = store.claim(&id, "worker-b", &caps(), next, 10_000).unwrap();
|
|
3354
|
-
assert!(second.fencing_token > first.fencing_token);
|
|
3355
|
-
|
|
3356
|
-
// The first attempt cannot land a result after the retry was handed on.
|
|
3357
|
-
assert_eq!(
|
|
3358
|
-
store
|
|
3359
|
-
.complete(
|
|
3360
|
-
&id,
|
|
3361
|
-
&first.claim_id,
|
|
3362
|
-
"worker-a",
|
|
3363
|
-
first.fencing_token,
|
|
3364
|
-
"stale",
|
|
3365
|
-
Value::Null,
|
|
3366
|
-
vec![],
|
|
3367
|
-
1,
|
|
3368
|
-
next,
|
|
3369
|
-
)
|
|
3370
|
-
.unwrap_err()
|
|
3371
|
-
.code,
|
|
3372
|
-
"WORKLOAD_STALE_FENCE"
|
|
3373
|
-
);
|
|
3374
|
-
assert_eq!(store.get(&id).unwrap().attempt_count, 2);
|
|
3375
|
-
let _ = fs::remove_file(path);
|
|
3376
|
-
}
|
|
3377
|
-
|
|
3378
|
-
#[test]
|
|
3379
|
-
fn cancellation_before_not_before_prevents_execution() {
|
|
3380
|
-
let (path, mut store) = temporal_store("cancel");
|
|
3381
|
-
let id = store.create(scheduled(2_000)).unwrap().workload_id;
|
|
3382
|
-
|
|
3383
|
-
// Cancelling work that is durable but not yet executable is the whole
|
|
3384
|
-
// reason to be able to see it before it runs.
|
|
3385
|
-
assert_eq!(
|
|
3386
|
-
store.cancel(&id, "operator", 1_500).unwrap(),
|
|
3387
|
-
WorkloadState::Cancelled
|
|
3388
|
-
);
|
|
3389
|
-
|
|
3390
|
-
// Eligibility arriving does not resurrect it.
|
|
3391
|
-
assert_eq!(
|
|
3392
|
-
store
|
|
3393
|
-
.claim(&id, "worker", &caps(), 2_001, 10_000)
|
|
3394
|
-
.unwrap_err()
|
|
3395
|
-
.code,
|
|
3396
|
-
"WORKLOAD_NOT_READY"
|
|
3397
|
-
);
|
|
3398
|
-
|
|
3399
|
-
let reopened = WorkloadStore::load(&path).unwrap();
|
|
3400
|
-
let cancelled = reopened.get(&id).unwrap();
|
|
3401
|
-
assert_eq!(cancelled.state, WorkloadState::Cancelled);
|
|
3402
|
-
assert!(cancelled.attempts.is_empty(), "nothing ever executed");
|
|
3403
|
-
let _ = fs::remove_file(path);
|
|
3404
|
-
}
|
|
3405
|
-
|
|
3406
|
-
#[test]
|
|
3407
|
-
fn a_worker_dying_after_a_claim_returns_the_work_and_it_executes_once() {
|
|
3408
|
-
let (path, mut store) = temporal_store("worker-death");
|
|
3409
|
-
let id = store.create(create()).unwrap().workload_id;
|
|
3410
|
-
|
|
3411
|
-
// Worker A claims a short lease, starts, and dies without completing.
|
|
3412
|
-
let dead = store.claim(&id, "worker-a", &caps(), 10, 1_000).unwrap();
|
|
3413
|
-
store
|
|
3414
|
-
.start(&id, &dead.claim_id, "worker-a", dead.fencing_token, 10)
|
|
3415
|
-
.unwrap();
|
|
3416
|
-
assert_eq!(
|
|
3417
|
-
store.get(&id).unwrap().state,
|
|
3418
|
-
WorkloadState::Running,
|
|
3419
|
-
"the work was in flight when its worker died"
|
|
3420
|
-
);
|
|
3421
|
-
|
|
3422
|
-
// The lease expiring is the only thing that can notice.
|
|
3423
|
-
let recovered = store.recover_expired(12).unwrap();
|
|
3424
|
-
assert_eq!(recovered, vec![id.clone()]);
|
|
3425
|
-
let back = store.get(&id).unwrap();
|
|
3426
|
-
assert_eq!(back.state, WorkloadState::Ready, "returned to the pool");
|
|
3427
|
-
assert_eq!(
|
|
3428
|
-
back.attempts.last().map(|attempt| attempt.status),
|
|
3429
|
-
Some(AttemptStatus::WorkerLost),
|
|
3430
|
-
"and the dead attempt is recorded as lost rather than erased"
|
|
3431
|
-
);
|
|
3432
|
-
|
|
3433
|
-
// Worker B picks it up and finishes it.
|
|
3434
|
-
let live = store.claim(&id, "worker-b", &caps(), 12, 10_000).unwrap();
|
|
3435
|
-
assert!(live.fencing_token > dead.fencing_token);
|
|
3436
|
-
store
|
|
3437
|
-
.start(&id, &live.claim_id, "worker-b", live.fencing_token, 13)
|
|
3438
|
-
.unwrap();
|
|
3439
|
-
store
|
|
3440
|
-
.complete(
|
|
3441
|
-
&id,
|
|
3442
|
-
&live.claim_id,
|
|
3443
|
-
"worker-b",
|
|
3444
|
-
live.fencing_token,
|
|
3445
|
-
"result-recovered",
|
|
3446
|
-
serde_json::json!({"ok": true}),
|
|
3447
|
-
vec![],
|
|
3448
|
-
1,
|
|
3449
|
-
14,
|
|
3450
|
-
)
|
|
3451
|
-
.unwrap();
|
|
3452
|
-
|
|
3453
|
-
// Exactly one execution succeeded, and the dead worker cannot add another.
|
|
3454
|
-
let finished = store.get(&id).unwrap();
|
|
3455
|
-
assert_eq!(finished.state, WorkloadState::Succeeded);
|
|
3456
|
-
assert_eq!(
|
|
3457
|
-
finished
|
|
3458
|
-
.attempts
|
|
3459
|
-
.iter()
|
|
3460
|
-
.filter(|attempt| attempt.status == AttemptStatus::Succeeded)
|
|
3461
|
-
.count(),
|
|
3462
|
-
1
|
|
3463
|
-
);
|
|
3464
|
-
// The dead worker cannot add a second result. Its claim was released by
|
|
3465
|
-
// the recovery, and completion released the successor's, so it is
|
|
3466
|
-
// refused for having no claim at all rather than a stale one.
|
|
3467
|
-
assert_eq!(
|
|
3468
|
-
store
|
|
3469
|
-
.complete(
|
|
3470
|
-
&id,
|
|
3471
|
-
&dead.claim_id,
|
|
3472
|
-
"worker-a",
|
|
3473
|
-
dead.fencing_token,
|
|
3474
|
-
"resurrected",
|
|
3475
|
-
Value::Null,
|
|
3476
|
-
vec![],
|
|
3477
|
-
1,
|
|
3478
|
-
15,
|
|
3479
|
-
)
|
|
3480
|
-
.unwrap_err()
|
|
3481
|
-
.code,
|
|
3482
|
-
"WORKLOAD_CLAIM_INVALID"
|
|
3483
|
-
);
|
|
3484
|
-
let _ = fs::remove_file(path);
|
|
3485
|
-
}
|
|
3486
|
-
|
|
3487
|
-
#[test]
|
|
3488
|
-
fn a_recovered_workload_refuses_its_dead_worker_while_the_successor_still_holds_it() {
|
|
3489
|
-
let (path, mut store) = temporal_store("worker-death-inflight");
|
|
3490
|
-
let id = store.create(create()).unwrap().workload_id;
|
|
3491
|
-
|
|
3492
|
-
let dead = store.claim(&id, "worker-a", &caps(), 10, 1_000).unwrap();
|
|
3493
|
-
store
|
|
3494
|
-
.start(&id, &dead.claim_id, "worker-a", dead.fencing_token, 10)
|
|
3495
|
-
.unwrap();
|
|
3496
|
-
store.recover_expired(12).unwrap();
|
|
3497
|
-
let live = store.claim(&id, "worker-b", &caps(), 12, 10_000).unwrap();
|
|
3498
|
-
|
|
3499
|
-
// While the successor is mid-flight, the resurrected worker is refused
|
|
3500
|
-
// on the fence itself — the case a lease exists to decide.
|
|
3501
|
-
assert_eq!(
|
|
3502
|
-
store
|
|
3503
|
-
.complete(
|
|
3504
|
-
&id,
|
|
3505
|
-
&dead.claim_id,
|
|
3506
|
-
"worker-a",
|
|
3507
|
-
dead.fencing_token,
|
|
3508
|
-
"resurrected",
|
|
3509
|
-
Value::Null,
|
|
3510
|
-
vec![],
|
|
3511
|
-
1,
|
|
3512
|
-
13,
|
|
3513
|
-
)
|
|
3514
|
-
.unwrap_err()
|
|
3515
|
-
.code,
|
|
3516
|
-
"WORKLOAD_STALE_FENCE"
|
|
3517
|
-
);
|
|
3518
|
-
assert_eq!(store.get(&id).unwrap().state, WorkloadState::Claimed);
|
|
3519
|
-
assert!(live.fencing_token > dead.fencing_token);
|
|
3520
|
-
let _ = fs::remove_file(path);
|
|
3521
|
-
}
|
|
3522
|
-
|
|
3523
|
-
#[test]
|
|
3524
|
-
fn the_scheduler_skips_ineligible_work_rather_than_failing_on_it() {
|
|
3525
|
-
let (path, mut store) = temporal_store("scheduler");
|
|
3526
|
-
let mut early = create();
|
|
3527
|
-
early.idempotency_key = "early".into();
|
|
3528
|
-
early.not_before = Some(9_000);
|
|
3529
|
-
let early = store.create(early).unwrap();
|
|
3530
|
-
|
|
3531
|
-
let mut ready = create();
|
|
3532
|
-
ready.idempotency_key = "ready".into();
|
|
3533
|
-
let ready = store.create(ready).unwrap();
|
|
3534
|
-
|
|
3535
|
-
// The ineligible workload sorts first by id in some orderings and not
|
|
3536
|
-
// others, so what matters is that the scheduler never returns it and
|
|
3537
|
-
// never fails because of it.
|
|
3538
|
-
let mut scheduler = LocalDurableScheduler::default();
|
|
3539
|
-
let claim = scheduler
|
|
3540
|
-
.claim(&mut store, "t", "app", "worker", &caps(), 1_000, 10_000)
|
|
3541
|
-
.expect("an ineligible workload must not fail the whole pass")
|
|
3542
|
-
.expect("the eligible workload is still handed out");
|
|
3543
|
-
assert_eq!(claim.workload_id, ready.workload_id);
|
|
3544
|
-
assert_eq!(
|
|
3545
|
-
store.get(&early.workload_id).unwrap().state,
|
|
3546
|
-
WorkloadState::Ready
|
|
3547
|
-
);
|
|
3548
|
-
let _ = fs::remove_file(path);
|
|
3549
|
-
}
|
|
3550
|
-
|
|
3551
|
-
// ---- PR 2: authoritative reads, canonical admission, failure reports, deadlines ----
|
|
3552
|
-
|
|
3553
|
-
fn version_of(store: &WorkloadStore, id: &str) -> u64 {
|
|
3554
|
-
let db = store.db_for_tests();
|
|
3555
|
-
let w = store.get(id).unwrap();
|
|
3556
|
-
db.live_row(DURABLE_WORK_COLLECTION, &work_key(w))
|
|
3557
|
-
.and_then(|row| row.value.get("__version").and_then(Value::as_u64))
|
|
3558
|
-
.unwrap()
|
|
3559
|
-
}
|
|
3560
|
-
|
|
3561
|
-
#[test]
|
|
3562
|
-
fn another_writers_commits_are_visible_and_acted_on_without_restart() {
|
|
3563
|
-
let (path, mut a) = store("two-writers");
|
|
3564
|
-
let mut b = WorkloadStore::from_db(a.db_for_tests()).unwrap();
|
|
3565
|
-
let id = a.create(create()).unwrap().workload_id;
|
|
3566
|
-
// B never saw the admission, yet reads it from the store.
|
|
3567
|
-
assert!(b.get(&id).is_err(), "the projection alone is stale");
|
|
3568
|
-
assert_eq!(b.fresh(&id).unwrap().state, WorkloadState::Ready);
|
|
3569
|
-
assert_eq!(b.fresh_list("t", "app").unwrap().len(), 1);
|
|
3570
|
-
// A claims; B's claim decides on the committed claim, not its own view.
|
|
3571
|
-
let claim = a
|
|
3572
|
-
.claim(&id, "worker-a", &capabilities(), 11, 10_000)
|
|
3573
|
-
.unwrap();
|
|
3574
|
-
assert_eq!(
|
|
3575
|
-
b.claim(&id, "worker-b", &capabilities(), 11, 10_000)
|
|
3576
|
-
.unwrap_err()
|
|
3577
|
-
.code,
|
|
3578
|
-
"WORKLOAD_NOT_READY"
|
|
3579
|
-
);
|
|
3580
|
-
// A transition committed through A is what B fences against.
|
|
3581
|
-
b.start(&id, &claim.claim_id, "worker-a", claim.fencing_token, 12)
|
|
3582
|
-
.unwrap();
|
|
3583
|
-
assert_eq!(a.fresh(&id).unwrap().state, WorkloadState::Running);
|
|
3584
|
-
// B replays an admission A committed, by key, from the store.
|
|
3585
|
-
let mut c = WorkloadStore::from_db(a.db_for_tests()).unwrap();
|
|
3586
|
-
c.workloads.clear();
|
|
3587
|
-
c.idempotency.clear();
|
|
3588
|
-
c.document_versions.clear();
|
|
3589
|
-
let (replayed, created) = c.admit(create()).unwrap();
|
|
3590
|
-
assert!(!created);
|
|
3591
|
-
assert_eq!(replayed.workload_id, id);
|
|
3592
|
-
let _ = fs::remove_file(path);
|
|
3593
|
-
}
|
|
3594
|
-
|
|
3595
|
-
#[test]
|
|
3596
|
-
fn a_replay_matches_only_an_equivalent_request_and_names_every_conflicting_field() {
|
|
3597
|
-
let (path, mut store) = store("canonical-admission");
|
|
3598
|
-
let mut original = create();
|
|
3599
|
-
original.input = serde_json::json!({"a": 1, "b": 2});
|
|
3600
|
-
original.dependencies = vec![
|
|
3601
|
-
WorkloadDependency {
|
|
3602
|
-
workload_id: "w1".into(),
|
|
3603
|
-
required_state: WorkloadState::Succeeded,
|
|
3604
|
-
},
|
|
3605
|
-
WorkloadDependency {
|
|
3606
|
-
workload_id: "w2".into(),
|
|
3607
|
-
required_state: WorkloadState::Succeeded,
|
|
3608
|
-
},
|
|
3609
|
-
];
|
|
3610
|
-
let first = store.create(original.clone()).unwrap();
|
|
3611
|
-
|
|
3612
|
-
// Equivalent after normalization: member order, dependency order, the
|
|
3613
|
-
// defaults written out, and provenance that does not change what runs.
|
|
3614
|
-
let mut equivalent = original.clone();
|
|
3615
|
-
equivalent.input = serde_json::json!({"b": 2, "a": 1});
|
|
3616
|
-
equivalent.dependencies.reverse();
|
|
3617
|
-
equivalent.retry_policy = WorkloadRetryPolicy::default();
|
|
3618
|
-
equivalent.timeouts = WorkloadTimeouts::default();
|
|
3619
|
-
equivalent.priority = 0;
|
|
3620
|
-
equivalent.trigger = "retried-by-client".into();
|
|
3621
|
-
equivalent.session_id = Some("another-session".into());
|
|
3622
|
-
equivalent.created_at = 99;
|
|
3623
|
-
let (replayed, created) = store.admit(equivalent).unwrap();
|
|
3624
|
-
assert!(!created);
|
|
3625
|
-
assert_eq!(replayed.workload_id, first.workload_id);
|
|
3626
|
-
|
|
3627
|
-
let mutations: Vec<(&str, Box<dyn Fn(&mut CreateWorkload)>)> = vec![
|
|
3628
|
-
(
|
|
3629
|
-
"definition_id",
|
|
3630
|
-
Box::new(|r| r.definition_id = "other".into()),
|
|
3631
|
-
),
|
|
3632
|
-
(
|
|
3633
|
-
"definition_revision",
|
|
3634
|
-
Box::new(|r| r.definition_revision = "r2".into()),
|
|
3635
|
-
),
|
|
3636
|
-
(
|
|
3637
|
-
"environment",
|
|
3638
|
-
Box::new(|r| r.environment = "staging".into()),
|
|
3639
|
-
),
|
|
3640
|
-
(
|
|
3641
|
-
"input",
|
|
3642
|
-
Box::new(|r| r.input = serde_json::json!({"a": 1, "b": 3})),
|
|
3643
|
-
),
|
|
3644
|
-
(
|
|
3645
|
-
"capability_snapshot",
|
|
3646
|
-
Box::new(|r| {
|
|
3647
|
-
r.capability_snapshot.insert("sms:send".into());
|
|
3648
|
-
}),
|
|
3649
|
-
),
|
|
3650
|
-
(
|
|
3651
|
-
"connection_scopes",
|
|
3652
|
-
Box::new(|r| {
|
|
3653
|
-
r.connection_scopes.insert("crm".into());
|
|
3654
|
-
}),
|
|
3655
|
-
),
|
|
3656
|
-
("priority", Box::new(|r| r.priority = 5)),
|
|
3657
|
-
(
|
|
3658
|
-
"retry_policy",
|
|
3659
|
-
Box::new(|r| r.retry_policy.max_attempts = 9),
|
|
3660
|
-
),
|
|
3661
|
-
(
|
|
3662
|
-
"retry_policy",
|
|
3663
|
-
Box::new(|r| {
|
|
3664
|
-
r.retry_policy.retry_on.remove("worker_failure");
|
|
3665
|
-
}),
|
|
3666
|
-
),
|
|
3667
|
-
(
|
|
3668
|
-
"timeouts",
|
|
3669
|
-
Box::new(|r| r.timeouts.execution_ms = Some(60_000)),
|
|
3670
|
-
),
|
|
3671
|
-
(
|
|
3672
|
-
"dependencies",
|
|
3673
|
-
Box::new(|r| r.dependencies.pop().map(|_| ()).unwrap()),
|
|
3674
|
-
),
|
|
3675
|
-
("not_before", Box::new(|r| r.not_before = Some(5_000))),
|
|
3676
|
-
];
|
|
3677
|
-
for (field, mutate) in mutations {
|
|
3678
|
-
let mut request = original.clone();
|
|
3679
|
-
mutate(&mut request);
|
|
3680
|
-
let error = store.admit(request).unwrap_err();
|
|
3681
|
-
assert_eq!(error.code, "IDEMPOTENCY_KEY_REUSED", "{field}");
|
|
3682
|
-
assert_eq!(
|
|
3683
|
-
error.details.unwrap()["fields"],
|
|
3684
|
-
serde_json::json!([field]),
|
|
3685
|
-
"{field}"
|
|
3686
|
-
);
|
|
3687
|
-
}
|
|
3688
|
-
// Several at once are all named.
|
|
3689
|
-
let mut request = original.clone();
|
|
3690
|
-
request.priority = 1;
|
|
3691
|
-
request.environment = "staging".into();
|
|
3692
|
-
let fields = store.admit(request).unwrap_err().details.unwrap()["fields"].clone();
|
|
3693
|
-
assert_eq!(fields, serde_json::json!(["environment", "priority"]));
|
|
3694
|
-
|
|
3695
|
-
// After a restart: the same answers from the committed record.
|
|
3696
|
-
drop(store);
|
|
3697
|
-
let mut reopened = WorkloadStore::load(&path).unwrap();
|
|
3698
|
-
assert!(!reopened.admit(original.clone()).unwrap().1);
|
|
3699
|
-
let mut changed = original.clone();
|
|
3700
|
-
changed.retry_policy.initial_delay_ms = 5;
|
|
3701
|
-
assert_eq!(
|
|
3702
|
-
reopened.admit(changed).unwrap_err().code,
|
|
3703
|
-
"IDEMPOTENCY_KEY_REUSED"
|
|
3704
|
-
);
|
|
3705
|
-
assert_eq!(reopened.workloads.len(), 1, "nothing was created");
|
|
3706
|
-
let _ = fs::remove_file(path);
|
|
3707
|
-
}
|
|
3708
|
-
|
|
3709
|
-
#[test]
|
|
3710
|
-
fn concurrent_conflicting_admissions_admit_one_and_refuse_the_other() {
|
|
3711
|
-
for round in 0..10 {
|
|
3712
|
-
let path = std::env::temp_dir().join(format!(
|
|
3713
|
-
"workload-admission-race-{round}-{}.log",
|
|
3714
|
-
std::process::id()
|
|
3715
|
-
));
|
|
3716
|
-
let _ = fs::remove_file(&path);
|
|
3717
|
-
let db = FeltDb::open(&path).unwrap();
|
|
3718
|
-
let stores = [
|
|
3719
|
-
WorkloadStore::from_db(db.clone()).unwrap(),
|
|
3720
|
-
WorkloadStore::from_db(db.clone()).unwrap(),
|
|
3721
|
-
];
|
|
3722
|
-
let barrier = std::sync::Barrier::new(2);
|
|
3723
|
-
let outcomes = std::thread::scope(|scope| {
|
|
3724
|
-
stores
|
|
3725
|
-
.into_iter()
|
|
3726
|
-
.enumerate()
|
|
3727
|
-
.map(|(i, mut store)| {
|
|
3728
|
-
let barrier = &barrier;
|
|
3729
|
-
scope.spawn(move || {
|
|
3730
|
-
let mut request = create();
|
|
3731
|
-
request.priority = i as i32;
|
|
3732
|
-
barrier.wait();
|
|
3733
|
-
store
|
|
3734
|
-
.admit(request)
|
|
3735
|
-
.map(|(w, created)| (w.priority, created))
|
|
3736
|
-
})
|
|
3737
|
-
})
|
|
3738
|
-
.collect::<Vec<_>>()
|
|
3739
|
-
.into_iter()
|
|
3740
|
-
.map(|handle| handle.join().unwrap())
|
|
3741
|
-
.collect::<Vec<_>>()
|
|
3742
|
-
});
|
|
3743
|
-
let admitted = outcomes
|
|
3744
|
-
.iter()
|
|
3745
|
-
.filter(|o| matches!(o, Ok((_, true))))
|
|
3746
|
-
.count();
|
|
3747
|
-
let refused = outcomes
|
|
3748
|
-
.iter()
|
|
3749
|
-
.filter(|o| {
|
|
3750
|
-
o.as_ref()
|
|
3751
|
-
.is_err_and(|e| e.code == "IDEMPOTENCY_KEY_REUSED")
|
|
3752
|
-
})
|
|
3753
|
-
.count();
|
|
3754
|
-
assert_eq!((admitted, refused), (1, 1), "round {round}: {outcomes:?}");
|
|
3755
|
-
let _ = fs::remove_file(path);
|
|
3756
|
-
}
|
|
3757
|
-
}
|
|
3758
|
-
|
|
3759
|
-
fn claimed(
|
|
3760
|
-
name: &str,
|
|
3761
|
-
policy: impl FnOnce(&mut CreateWorkload),
|
|
3762
|
-
) -> (PathBuf, WorkloadStore, String, WorkloadClaim) {
|
|
3763
|
-
let (path, mut store) = store(name);
|
|
3764
|
-
let mut request = create();
|
|
3765
|
-
policy(&mut request);
|
|
3766
|
-
let id = store.create(request).unwrap().workload_id;
|
|
3767
|
-
let claim = store.claim(&id, "a", &capabilities(), 11, 10_000).unwrap();
|
|
3768
|
-
(path, store, id, claim)
|
|
3769
|
-
}
|
|
3770
|
-
|
|
3771
|
-
#[test]
|
|
3772
|
-
fn a_worker_reports_failure_from_claimed_fenced_and_by_policy() {
|
|
3773
|
-
// Could not start: a known failure before any effect, retried by kind.
|
|
3774
|
-
let (path, mut store, id, claim) = claimed("fail-claimed", |_| {});
|
|
3775
|
-
let state = store
|
|
3776
|
-
.fail_with_effect(
|
|
3777
|
-
&id,
|
|
3778
|
-
&claim.claim_id,
|
|
3779
|
-
"a",
|
|
3780
|
-
claim.fencing_token,
|
|
3781
|
-
"transient_connection",
|
|
3782
|
-
"no route",
|
|
3783
|
-
None,
|
|
3784
|
-
12,
|
|
3785
|
-
)
|
|
3786
|
-
.unwrap();
|
|
3787
|
-
assert_eq!(state, WorkloadState::Ready);
|
|
3788
|
-
let w = store.get(&id).unwrap();
|
|
3789
|
-
let failed = w
|
|
3790
|
-
.execution_history
|
|
3791
|
-
.iter()
|
|
3792
|
-
.find(|e| e.to == WorkloadState::Failed)
|
|
3793
|
-
.unwrap();
|
|
3794
|
-
assert_eq!(
|
|
3795
|
-
failed.evidence["effect"], "none",
|
|
3796
|
-
"never started, so no effect"
|
|
3797
|
-
);
|
|
3798
|
-
assert_eq!(w.attempts[0].status, AttemptStatus::Failed);
|
|
3799
|
-
// The identical report again (lost response) replays and commits nothing.
|
|
3800
|
-
let version = version_of(&store, &id);
|
|
3801
|
-
assert_eq!(
|
|
3802
|
-
store
|
|
3803
|
-
.fail_with_effect(
|
|
3804
|
-
&id,
|
|
3805
|
-
&claim.claim_id,
|
|
3806
|
-
"a",
|
|
3807
|
-
claim.fencing_token,
|
|
3808
|
-
"transient_connection",
|
|
3809
|
-
"no route",
|
|
3810
|
-
None,
|
|
3811
|
-
13
|
|
3812
|
-
)
|
|
3813
|
-
.unwrap(),
|
|
3814
|
-
WorkloadState::Ready
|
|
3815
|
-
);
|
|
3816
|
-
assert_eq!(version_of(&store, &id), version);
|
|
3817
|
-
// A different report from the same attempt is answered by its fence.
|
|
3818
|
-
assert_eq!(
|
|
3819
|
-
store
|
|
3820
|
-
.fail_with_effect(
|
|
3821
|
-
&id,
|
|
3822
|
-
&claim.claim_id,
|
|
3823
|
-
"a",
|
|
3824
|
-
claim.fencing_token,
|
|
3825
|
-
"timeout",
|
|
3826
|
-
"other",
|
|
3827
|
-
None,
|
|
3828
|
-
13
|
|
3829
|
-
)
|
|
3830
|
-
.unwrap_err()
|
|
3831
|
-
.code,
|
|
3832
|
-
"WORKLOAD_CLAIM_INVALID"
|
|
3833
|
-
);
|
|
3834
|
-
// So is a completion: the failure stands.
|
|
3835
|
-
assert_eq!(
|
|
3836
|
-
store
|
|
3837
|
-
.complete(
|
|
3838
|
-
&id,
|
|
3839
|
-
&claim.claim_id,
|
|
3840
|
-
"a",
|
|
3841
|
-
claim.fencing_token,
|
|
3842
|
-
"r",
|
|
3843
|
-
Value::Null,
|
|
3844
|
-
vec![],
|
|
3845
|
-
1,
|
|
3846
|
-
13
|
|
3847
|
-
)
|
|
3848
|
-
.unwrap_err()
|
|
3849
|
-
.code,
|
|
3850
|
-
"WORKLOAD_CLAIM_INVALID"
|
|
3851
|
-
);
|
|
3852
|
-
// Another worker's claim fences the old attempt out.
|
|
3853
|
-
let next = store.claim(&id, "b", &capabilities(), 14, 10_000).unwrap();
|
|
3854
|
-
assert_eq!(
|
|
3855
|
-
store
|
|
3856
|
-
.fail_with_effect(
|
|
3857
|
-
&id,
|
|
3858
|
-
&claim.claim_id,
|
|
3859
|
-
"a",
|
|
3860
|
-
claim.fencing_token,
|
|
3861
|
-
"timeout",
|
|
3862
|
-
"late",
|
|
3863
|
-
None,
|
|
3864
|
-
15
|
|
3865
|
-
)
|
|
3866
|
-
.unwrap_err()
|
|
3867
|
-
.code,
|
|
3868
|
-
"WORKLOAD_STALE_FENCE"
|
|
3869
|
-
);
|
|
3870
|
-
assert_eq!(
|
|
3871
|
-
store
|
|
3872
|
-
.get(&id)
|
|
3873
|
-
.unwrap()
|
|
3874
|
-
.current_claim
|
|
3875
|
-
.as_ref()
|
|
3876
|
-
.unwrap()
|
|
3877
|
-
.claim_id,
|
|
3878
|
-
next.claim_id
|
|
3879
|
-
);
|
|
3880
|
-
let _ = fs::remove_file(path);
|
|
3881
|
-
}
|
|
3882
|
-
|
|
3883
|
-
#[test]
|
|
3884
|
-
fn failure_policy_decides_retry_and_an_unknown_effect_is_decided_like_a_lost_worker() {
|
|
3885
|
-
// A kind the policy does not retry dead-letters.
|
|
3886
|
-
let (path, mut store, id, claim) = claimed("fail-kind", |_| {});
|
|
3887
|
-
store
|
|
3888
|
-
.start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
|
|
3889
|
-
.unwrap();
|
|
3890
|
-
assert_eq!(
|
|
3891
|
-
store
|
|
3892
|
-
.fail_with_effect(
|
|
3893
|
-
&id,
|
|
3894
|
-
&claim.claim_id,
|
|
3895
|
-
"a",
|
|
3896
|
-
claim.fencing_token,
|
|
3897
|
-
"validation",
|
|
3898
|
-
"bad input",
|
|
3899
|
-
Some("none"),
|
|
3900
|
-
13
|
|
3901
|
-
)
|
|
3902
|
-
.unwrap(),
|
|
3903
|
-
WorkloadState::DeadLetter
|
|
3904
|
-
);
|
|
3905
|
-
assert_eq!(
|
|
3906
|
-
store
|
|
3907
|
-
.get(&id)
|
|
3908
|
-
.unwrap()
|
|
3909
|
-
.execution_history
|
|
3910
|
-
.last()
|
|
3911
|
-
.unwrap()
|
|
3912
|
-
.evidence["decision"],
|
|
3913
|
-
"failure_kind_not_retryable"
|
|
3914
|
-
);
|
|
3915
|
-
let _ = fs::remove_file(path);
|
|
3916
|
-
|
|
3917
|
-
// A retryable kind with an unknown effect, under a policy that does not
|
|
3918
|
-
// re-run lost work: not retried automatically.
|
|
3919
|
-
let (path, mut store, id, claim) = claimed("fail-unknown", |r| {
|
|
3920
|
-
r.retry_policy.retry_on = ["timeout".into(), "transient_connection".into()].into();
|
|
3921
|
-
});
|
|
3922
|
-
store
|
|
3923
|
-
.start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
|
|
3924
|
-
.unwrap();
|
|
3925
|
-
assert_eq!(
|
|
3926
|
-
store
|
|
3927
|
-
.fail_with_effect(
|
|
3928
|
-
&id,
|
|
3929
|
-
&claim.claim_id,
|
|
3930
|
-
"a",
|
|
3931
|
-
claim.fencing_token,
|
|
3932
|
-
"timeout",
|
|
3933
|
-
"charge sent, no answer",
|
|
3934
|
-
Some("unknown"),
|
|
3935
|
-
13
|
|
3936
|
-
)
|
|
3937
|
-
.unwrap(),
|
|
3938
|
-
WorkloadState::DeadLetter
|
|
3939
|
-
);
|
|
3940
|
-
let w = store.get(&id).unwrap();
|
|
3941
|
-
assert_eq!(
|
|
3942
|
-
w.execution_history.last().unwrap().evidence["decision"],
|
|
3943
|
-
"effect_unknown_not_retryable"
|
|
3944
|
-
);
|
|
3945
|
-
// An operator can still re-arm it, with a reason.
|
|
3946
|
-
store
|
|
3947
|
-
.retry_with_reason(&id, "operator", 14, Some("verified: not charged"))
|
|
3948
|
-
.unwrap();
|
|
3949
|
-
assert_eq!(store.get(&id).unwrap().state, WorkloadState::Ready);
|
|
3950
|
-
let _ = fs::remove_file(path);
|
|
3951
|
-
|
|
3952
|
-
// The same report under a policy that re-runs lost work is retried.
|
|
3953
|
-
let (path, mut store, id, claim) = claimed("fail-unknown-retry", |_| {});
|
|
3954
|
-
store
|
|
3955
|
-
.start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
|
|
3956
|
-
.unwrap();
|
|
3957
|
-
assert_eq!(
|
|
3958
|
-
store
|
|
3959
|
-
.fail_with_effect(
|
|
3960
|
-
&id,
|
|
3961
|
-
&claim.claim_id,
|
|
3962
|
-
"a",
|
|
3963
|
-
claim.fencing_token,
|
|
3964
|
-
"validation",
|
|
3965
|
-
"x",
|
|
3966
|
-
Some("unknown"),
|
|
3967
|
-
13
|
|
3968
|
-
)
|
|
3969
|
-
.unwrap(),
|
|
3970
|
-
WorkloadState::Ready
|
|
3971
|
-
);
|
|
3972
|
-
// An invalid effect is refused before anything changes.
|
|
3973
|
-
assert_eq!(
|
|
3974
|
-
store
|
|
3975
|
-
.fail_with_effect(&id, "c", "a", 9, "x", "y", Some("maybe"), 14)
|
|
3976
|
-
.unwrap_err()
|
|
3977
|
-
.code,
|
|
3978
|
-
"WORKLOAD_INVALID"
|
|
3979
|
-
);
|
|
3980
|
-
let _ = fs::remove_file(path);
|
|
3981
|
-
|
|
3982
|
-
// The attempt budget bounds retries.
|
|
3983
|
-
let (path, mut store, id, mut claim) =
|
|
3984
|
-
claimed("fail-budget", |r| r.retry_policy.max_attempts = 2);
|
|
3985
|
-
assert_eq!(
|
|
3986
|
-
store
|
|
3987
|
-
.fail(
|
|
3988
|
-
&id,
|
|
3989
|
-
&claim.claim_id,
|
|
3990
|
-
"a",
|
|
3991
|
-
claim.fencing_token,
|
|
3992
|
-
"timeout",
|
|
3993
|
-
"x",
|
|
3994
|
-
12
|
|
3995
|
-
)
|
|
3996
|
-
.unwrap(),
|
|
3997
|
-
WorkloadState::Ready
|
|
3998
|
-
);
|
|
3999
|
-
let next = store.get(&id).unwrap().next_attempt_at.unwrap();
|
|
4000
|
-
claim = store
|
|
4001
|
-
.claim(&id, "a", &capabilities(), next, 10_000)
|
|
4002
|
-
.unwrap();
|
|
4003
|
-
assert_eq!(
|
|
4004
|
-
store
|
|
4005
|
-
.fail(
|
|
4006
|
-
&id,
|
|
4007
|
-
&claim.claim_id,
|
|
4008
|
-
"a",
|
|
4009
|
-
claim.fencing_token,
|
|
4010
|
-
"timeout",
|
|
4011
|
-
"x",
|
|
4012
|
-
next
|
|
4013
|
-
)
|
|
4014
|
-
.unwrap(),
|
|
4015
|
-
WorkloadState::DeadLetter
|
|
4016
|
-
);
|
|
4017
|
-
assert_eq!(
|
|
4018
|
-
store
|
|
4019
|
-
.get(&id)
|
|
4020
|
-
.unwrap()
|
|
4021
|
-
.execution_history
|
|
4022
|
-
.last()
|
|
4023
|
-
.unwrap()
|
|
4024
|
-
.evidence["decision"],
|
|
4025
|
-
"attempt_budget_exhausted"
|
|
4026
|
-
);
|
|
4027
|
-
let _ = fs::remove_file(path);
|
|
4028
|
-
}
|
|
4029
|
-
|
|
4030
|
-
#[test]
|
|
4031
|
-
fn failure_racing_completion_commits_exactly_one_outcome() {
|
|
4032
|
-
for completion_first in [true, false] {
|
|
4033
|
-
let (path, mut store, id, claim) = claimed(&format!("race-{completion_first}"), |_| {});
|
|
4034
|
-
store
|
|
4035
|
-
.start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
|
|
4036
|
-
.unwrap();
|
|
4037
|
-
let complete = |s: &mut WorkloadStore| {
|
|
4038
|
-
s.complete(
|
|
4039
|
-
&id,
|
|
4040
|
-
&claim.claim_id,
|
|
4041
|
-
"a",
|
|
4042
|
-
claim.fencing_token,
|
|
4043
|
-
"r",
|
|
4044
|
-
Value::Null,
|
|
4045
|
-
vec![],
|
|
4046
|
-
1,
|
|
4047
|
-
13,
|
|
4048
|
-
)
|
|
4049
|
-
.map(|_| ())
|
|
4050
|
-
};
|
|
4051
|
-
let fail = |s: &mut WorkloadStore| {
|
|
4052
|
-
s.fail(
|
|
4053
|
-
&id,
|
|
4054
|
-
&claim.claim_id,
|
|
4055
|
-
"a",
|
|
4056
|
-
claim.fencing_token,
|
|
4057
|
-
"timeout",
|
|
4058
|
-
"x",
|
|
4059
|
-
13,
|
|
4060
|
-
)
|
|
4061
|
-
.map(|_| ())
|
|
4062
|
-
};
|
|
4063
|
-
let (first, second) = if completion_first {
|
|
4064
|
-
(complete(&mut store), fail(&mut store))
|
|
4065
|
-
} else {
|
|
4066
|
-
(fail(&mut store), complete(&mut store))
|
|
4067
|
-
};
|
|
4068
|
-
assert!(first.is_ok());
|
|
4069
|
-
assert_eq!(second.unwrap_err().code, "WORKLOAD_CLAIM_INVALID");
|
|
4070
|
-
let w = store.get(&id).unwrap();
|
|
4071
|
-
assert_eq!(
|
|
4072
|
-
w.state,
|
|
4073
|
-
if completion_first {
|
|
4074
|
-
WorkloadState::Succeeded
|
|
4075
|
-
} else {
|
|
4076
|
-
WorkloadState::Ready
|
|
4077
|
-
}
|
|
4078
|
-
);
|
|
4079
|
-
assert_eq!(w.attempts.len(), 1);
|
|
4080
|
-
let _ = fs::remove_file(path);
|
|
4081
|
-
}
|
|
4082
|
-
}
|
|
4083
|
-
|
|
4084
|
-
#[test]
|
|
4085
|
-
fn deadlines_are_enforced_from_committed_timestamps_and_race_safely() {
|
|
4086
|
-
// execution: a heartbeat inside the deadline keeps the attempt; after it,
|
|
4087
|
-
// every worker call is refused and the attempt is timed out, once.
|
|
4088
|
-
let (path, mut store, id, claim) =
|
|
4089
|
-
claimed("deadline-exec", |r| r.timeouts.execution_ms = Some(5_000));
|
|
4090
|
-
store
|
|
4091
|
-
.start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
|
|
4092
|
-
.unwrap();
|
|
4093
|
-
store
|
|
4094
|
-
.heartbeat(&id, &claim.claim_id, "a", claim.fencing_token, 16, 10_000)
|
|
4095
|
-
.unwrap();
|
|
4096
|
-
assert!(store.enforce_deadlines(16).unwrap().is_empty());
|
|
4097
|
-
let refused = store
|
|
4098
|
-
.complete(
|
|
4099
|
-
&id,
|
|
4100
|
-
&claim.claim_id,
|
|
4101
|
-
"a",
|
|
4102
|
-
claim.fencing_token,
|
|
4103
|
-
"late",
|
|
4104
|
-
Value::Null,
|
|
4105
|
-
vec![],
|
|
4106
|
-
1,
|
|
4107
|
-
17,
|
|
4108
|
-
)
|
|
4109
|
-
.unwrap_err();
|
|
4110
|
-
assert_eq!(refused.code, "WORKLOAD_TIMED_OUT");
|
|
4111
|
-
assert_eq!(refused.details.unwrap()["deadline"], "execution");
|
|
4112
|
-
let w = store.get(&id).unwrap().clone();
|
|
4113
|
-
assert_eq!(
|
|
4114
|
-
w.state,
|
|
4115
|
-
WorkloadState::Ready,
|
|
4116
|
-
"timeout is retryable by default"
|
|
4117
|
-
);
|
|
4118
|
-
assert_eq!(w.attempts[0].status, AttemptStatus::TimedOut);
|
|
4119
|
-
assert!(w.result.is_none(), "a late completion never lands");
|
|
4120
|
-
assert!(
|
|
4121
|
-
store.enforce_deadlines(30).unwrap().is_empty(),
|
|
4122
|
-
"timed out once"
|
|
4123
|
-
);
|
|
4124
|
-
let _ = fs::remove_file(path);
|
|
4125
|
-
|
|
4126
|
-
// heartbeat: the gap since the last heartbeat; a heartbeat that commits
|
|
4127
|
-
// first moves the deadline, the recovery pass after it changes nothing.
|
|
4128
|
-
let (path, mut store, id, claim) = claimed("deadline-heartbeat", |r| {
|
|
4129
|
-
r.timeouts.heartbeat_ms = Some(3_000)
|
|
4130
|
-
});
|
|
4131
|
-
store
|
|
4132
|
-
.start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
|
|
4133
|
-
.unwrap();
|
|
4134
|
-
store
|
|
4135
|
-
.heartbeat(&id, &claim.claim_id, "a", claim.fencing_token, 13, 10_000)
|
|
4136
|
-
.unwrap();
|
|
4137
|
-
assert!(store.enforce_deadlines(15).unwrap().is_empty());
|
|
4138
|
-
assert_eq!(store.enforce_deadlines(16).unwrap(), vec![id.clone()]);
|
|
4139
|
-
assert_eq!(
|
|
4140
|
-
store
|
|
4141
|
-
.heartbeat(&id, &claim.claim_id, "a", claim.fencing_token, 16, 10_000)
|
|
4142
|
-
.unwrap_err()
|
|
4143
|
-
.code,
|
|
4144
|
-
"WORKLOAD_CLAIM_INVALID",
|
|
4145
|
-
"the timed-out claim is gone"
|
|
4146
|
-
);
|
|
4147
|
-
let _ = fs::remove_file(path);
|
|
4148
|
-
|
|
4149
|
-
// claim: claimed but never started; not retried when the policy says so.
|
|
4150
|
-
let (path, mut store, id, claim) = claimed("deadline-claim", |r| {
|
|
4151
|
-
r.timeouts.claim_ms = Some(2_000);
|
|
4152
|
-
r.retry_policy.retry_on = ["worker_failure".into()].into();
|
|
4153
|
-
});
|
|
4154
|
-
let refused = store
|
|
4155
|
-
.start(&id, &claim.claim_id, "a", claim.fencing_token, 13)
|
|
4156
|
-
.unwrap_err();
|
|
4157
|
-
assert_eq!(refused.code, "WORKLOAD_TIMED_OUT");
|
|
4158
|
-
let w = store.get(&id).unwrap();
|
|
4159
|
-
assert_eq!(w.state, WorkloadState::DeadLetter);
|
|
4160
|
-
assert_eq!(
|
|
4161
|
-
w.execution_history.last().unwrap().evidence["decision"],
|
|
4162
|
-
"timeout_not_retryable"
|
|
4163
|
-
);
|
|
4164
|
-
let _ = fs::remove_file(path);
|
|
4165
|
-
|
|
4166
|
-
// queue and overall bound the work: dead-lettered, never retried, and a
|
|
4167
|
-
// queued workload's previous attempt is not rewritten.
|
|
4168
|
-
let (path, mut store) = super::tests::store("deadline-queue");
|
|
4169
|
-
let mut request = create();
|
|
4170
|
-
request.timeouts.queue_ms = Some(4_000);
|
|
4171
|
-
let id = store.create(request).unwrap().workload_id;
|
|
4172
|
-
assert_eq!(
|
|
4173
|
-
store
|
|
4174
|
-
.claim(&id, "a", &capabilities(), 14, 10_000)
|
|
4175
|
-
.unwrap_err()
|
|
4176
|
-
.code,
|
|
4177
|
-
"WORKLOAD_TIMED_OUT"
|
|
4178
|
-
);
|
|
4179
|
-
let w = store.get(&id).unwrap();
|
|
4180
|
-
assert_eq!(w.state, WorkloadState::DeadLetter);
|
|
4181
|
-
assert_eq!(
|
|
4182
|
-
w.execution_history.last().unwrap().evidence["decision"],
|
|
4183
|
-
"deadline_not_retryable"
|
|
4184
|
-
);
|
|
4185
|
-
assert!(w.attempts.is_empty());
|
|
4186
|
-
let _ = fs::remove_file(path);
|
|
4187
|
-
|
|
4188
|
-
let (path, mut store, id, claim) =
|
|
4189
|
-
claimed("deadline-overall", |r| r.timeouts.overall_ms = Some(10_000));
|
|
4190
|
-
store
|
|
4191
|
-
.start(&id, &claim.claim_id, "a", claim.fencing_token, 12)
|
|
4192
|
-
.unwrap();
|
|
4193
|
-
// Restart: the deadline comes from the committed record.
|
|
4194
|
-
drop(store);
|
|
4195
|
-
let mut store = WorkloadStore::load(&path).unwrap();
|
|
4196
|
-
assert!(store.enforce_deadlines(19).unwrap().is_empty());
|
|
4197
|
-
assert_eq!(store.enforce_deadlines(20).unwrap(), vec![id.clone()]);
|
|
4198
|
-
assert_eq!(store.get(&id).unwrap().state, WorkloadState::DeadLetter);
|
|
4199
|
-
let _ = fs::remove_file(path);
|
|
4200
|
-
}
|
|
4201
|
-
|
|
4202
|
-
#[test]
|
|
4203
|
-
fn deadlines_are_versioned_and_zero_is_refused() {
|
|
4204
|
-
let (path, mut store) = store("deadline-version");
|
|
4205
|
-
let mut request = create();
|
|
4206
|
-
request.timeouts.queue_ms = Some(0);
|
|
4207
|
-
assert_eq!(store.create(request).unwrap_err().code, "WORKLOAD_INVALID");
|
|
4208
|
-
// A record admitted before deadlines were enforced keeps its contract.
|
|
4209
|
-
let mut request = create();
|
|
4210
|
-
request.timeouts.queue_ms = Some(1_000);
|
|
4211
|
-
let id = store.create(request).unwrap().workload_id;
|
|
4212
|
-
assert_eq!(
|
|
4213
|
-
store.get(&id).unwrap().contract_version,
|
|
4214
|
-
DEADLINE_CONTRACT_VERSION
|
|
4215
|
-
);
|
|
4216
|
-
let mut legacy = store.get(&id).unwrap().clone();
|
|
4217
|
-
legacy.contract_version = 1;
|
|
4218
|
-
assert_eq!(deadline_breach(&legacy, 1_000_000), None);
|
|
4219
|
-
assert!(deadline_breach(store.get(&id).unwrap(), 1_000_000).is_some());
|
|
4220
|
-
let _ = fs::remove_file(path);
|
|
4221
|
-
}
|
|
4222
|
-
}
|