@feltdb/core 0.8.4 → 0.8.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/create/package-versions.js +1 -1
- package/dist/create/server-source/Cargo.lock +165 -0
- package/dist/create/server-source/Cargo.toml +9 -0
- package/dist/create/server-source/crates/feltdb/Cargo.toml +3 -0
- package/dist/create/server-source/crates/feltdb/benches/gate13_baseline.rs +44 -44
- package/dist/create/server-source/crates/feltdb/benches/gate13_phase_7_1_release_economics.rs +12 -24
- package/dist/create/server-source/crates/feltdb/benches/gate_13_redux.rs +7 -13
- package/dist/create/server-source/crates/feltdb/benches/gate_13_regression_runner.rs +13 -10
- package/dist/create/server-source/crates/feltdb/benches/gate_14a_concurrent_writer_scaling.rs +12 -9
- package/dist/create/server-source/crates/feltdb/benches/gate_14a_production_admission_revalidation.rs +78 -25
- package/dist/create/server-source/crates/feltdb/benches/gate_14a_rc2_admission_contract.rs +16 -13
- package/dist/create/server-source/crates/feltdb/benches/gate_14a_rc_root_cause.rs +13 -5
- package/dist/create/server-source/crates/feltdb/benches/gate_14a_sync1_queued_prototype.rs +41 -22
- package/dist/create/server-source/crates/feltdb/benches/gate_14a_sync_economics.rs +33 -15
- package/dist/create/server-source/crates/feltdb/benches/gate_14b_causal_backlog_scaling.rs +100 -33
- package/dist/create/server-source/crates/feltdb/benches/gate_14c_replication_contract_test.rs +56 -20
- package/dist/create/server-source/crates/feltdb/benches/gate_14c_replication_scaling.rs +116 -41
- package/dist/create/server-source/crates/feltdb/benches/gate_14d_combined_dimension_scaling.rs +186 -55
- package/dist/create/server-source/crates/feltdb/benches/phase_7_1_2_optimization_benchmark.rs +64 -26
- package/dist/create/server-source/crates/feltdb/benches/phase_7_1_3_crossover_analysis.rs +46 -15
- package/dist/create/server-source/crates/feltdb/src/admission.rs +8 -15
- package/dist/create/server-source/crates/feltdb/src/admission_contract_tests.rs +43 -13
- package/dist/create/server-source/crates/feltdb/src/adversarial_transport.rs +15 -42
- package/dist/create/server-source/crates/feltdb/src/analytics.rs +65 -19
- package/dist/create/server-source/crates/feltdb/src/application.rs +113 -30
- package/dist/create/server-source/crates/feltdb/src/authorization_security_tests.rs +475 -140
- package/dist/create/server-source/crates/feltdb/src/cardinality_diagnostics.rs +17 -15
- package/dist/create/server-source/crates/feltdb/src/cardinality_endpoint.rs +0 -1
- package/dist/create/server-source/crates/feltdb/src/causal_backlog_bound.rs +59 -15
- package/dist/create/server-source/crates/feltdb/src/causal_dependency_barrier.rs +266 -114
- package/dist/create/server-source/crates/feltdb/src/causal_dependency_barrier_phase_7_1.rs +25 -7
- package/dist/create/server-source/crates/feltdb/src/concurrency_fuzzing.rs +10 -15
- package/dist/create/server-source/crates/feltdb/src/consistency_contract.rs +3 -11
- package/dist/create/server-source/crates/feltdb/src/crash_atomic_boundary.rs +14 -5
- package/dist/create/server-source/crates/feltdb/src/crash_injection.rs +21 -25
- package/dist/create/server-source/crates/feltdb/src/crash_recovery_tests.rs +14 -11
- package/dist/create/server-source/crates/feltdb/src/dedup_bound_investigation.rs +103 -22
- package/dist/create/server-source/crates/feltdb/src/distributed_indexing.rs +18 -15
- package/dist/create/server-source/crates/feltdb/src/durability_guarantees.rs +12 -8
- package/dist/create/server-source/crates/feltdb/src/durable_dedup_set.rs +1 -5
- package/dist/create/server-source/crates/feltdb/src/durable_operation_identity.rs +87 -23
- package/dist/create/server-source/crates/feltdb/src/durable_operation_log.rs +3 -7
- package/dist/create/server-source/crates/feltdb/src/durable_sync.rs +10 -9
- package/dist/create/server-source/crates/feltdb/src/in_process_transport.rs +1 -6
- package/dist/create/server-source/crates/feltdb/src/indexing.rs +35 -38
- package/dist/create/server-source/crates/feltdb/src/lib.rs +1648 -46
- package/dist/create/server-source/crates/feltdb/src/managed_cas_tests.rs +4 -1
- package/dist/create/server-source/crates/feltdb/src/metrics.rs +0 -1
- package/dist/create/server-source/crates/feltdb/src/multi_node_convergence.rs +1 -2
- package/dist/create/server-source/crates/feltdb/src/multi_operation_transaction.rs +107 -30
- package/dist/create/server-source/crates/feltdb/src/observability.rs +19 -6
- package/dist/create/server-source/crates/feltdb/src/operation.rs +39 -0
- package/dist/create/server-source/crates/feltdb/src/operation_algebra.rs +12 -11
- package/dist/create/server-source/crates/feltdb/src/operation_log.rs +9 -4
- package/dist/create/server-source/crates/feltdb/src/p1_application_atomicity.rs +65 -18
- package/dist/create/server-source/crates/feltdb/src/p1_atomicity_acceptance.rs +193 -57
- package/dist/create/server-source/crates/feltdb/src/partition_reconciliation.rs +37 -27
- package/dist/create/server-source/crates/feltdb/src/permutation_scheduler.rs +38 -10
- package/dist/create/server-source/crates/feltdb/src/persistence_reality.rs +20 -14
- package/dist/create/server-source/crates/feltdb/src/phase1b_acceptance.rs +394 -229
- package/dist/create/server-source/crates/feltdb/src/phase1c1_acceptance.rs +8 -6
- package/dist/create/server-source/crates/feltdb/src/phase1c2_acceptance.rs +11 -13
- package/dist/create/server-source/crates/feltdb/src/phase1c3_acceptance.rs +79 -70
- package/dist/create/server-source/crates/feltdb/src/phase1c_atomicity_proof.rs +3 -3
- package/dist/create/server-source/crates/feltdb/src/phase5_integration.rs +33 -11
- package/dist/create/server-source/crates/feltdb/src/phase5_scenarios.rs +6 -6
- package/dist/create/server-source/crates/feltdb/src/phase6_adversarial_scenarios.rs +14 -56
- package/dist/create/server-source/crates/feltdb/src/phase6_convergence_validator.rs +29 -27
- package/dist/create/server-source/crates/feltdb/src/phase6_persistence.rs +35 -17
- package/dist/create/server-source/crates/feltdb/src/phase_1c_real_tcp.rs +8 -2
- package/dist/create/server-source/crates/feltdb/src/phase_2a_failures.rs +59 -15
- package/dist/create/server-source/crates/feltdb/src/phase_2b_network.rs +70 -17
- package/dist/create/server-source/crates/feltdb/src/phase_2c_cascading.rs +23 -6
- package/dist/create/server-source/crates/feltdb/src/phase_3_durability.rs +12 -3
- package/dist/create/server-source/crates/feltdb/src/phase_4_baseline.rs +41 -11
- package/dist/create/server-source/crates/feltdb/src/phase_5_soak.rs +56 -25
- package/dist/create/server-source/crates/feltdb/src/policy_evaluation.rs +701 -245
- package/dist/create/server-source/crates/feltdb/src/production_api.rs +31 -13
- package/dist/create/server-source/crates/feltdb/src/query_performance.rs +6 -8
- package/dist/create/server-source/crates/feltdb/src/replay_fuzzing.rs +5 -5
- package/dist/create/server-source/crates/feltdb/src/replica_acknowledgements.rs +48 -18
- package/dist/create/server-source/crates/feltdb/src/replica_membership.rs +30 -11
- package/dist/create/server-source/crates/feltdb/src/replication_manager.rs +6 -3
- package/dist/create/server-source/crates/feltdb/src/replication_protocol.rs +4 -3
- package/dist/create/server-source/crates/feltdb/src/sharding.rs +36 -10
- package/dist/create/server-source/crates/feltdb/src/state_conflict_contract.rs +516 -0
- package/dist/create/server-source/crates/feltdb/src/state_contract.rs +13 -4
- package/dist/create/server-source/crates/feltdb/src/state_diff_contract.rs +222 -0
- package/dist/create/server-source/crates/feltdb/src/state_facade.rs +82 -54
- package/dist/create/server-source/crates/feltdb/src/state_hash.rs +2 -2
- package/dist/create/server-source/crates/feltdb/src/state_model.rs +1565 -536
- package/dist/create/server-source/crates/feltdb/src/state_transition_store.rs +6 -3
- package/dist/create/server-source/crates/feltdb/src/state_trigger.rs +672 -0
- package/dist/create/server-source/crates/feltdb/src/storage.rs +9 -3
- package/dist/create/server-source/crates/feltdb/src/submission.rs +5 -11
- package/dist/create/server-source/crates/feltdb/src/tcp_transport.rs +6 -8
- package/dist/create/server-source/crates/feltdb/src/transaction_api.rs +24 -35
- package/dist/create/server-source/crates/feltdb/src/transaction_invariants.rs +24 -8
- package/dist/create/server-source/crates/feltdb/src/transaction_preconditions.rs +248 -59
- package/dist/create/server-source/crates/feltdb/src/transactions.rs +17 -20
- package/dist/create/server-source/crates/feltdb/src/trigger_contract.rs +749 -0
- package/dist/create/server-source/crates/feltdb/src/worker_mesh.rs +1 -0
- package/dist/create/server-source/crates/feltdb/src/workload.rs +512 -4
- package/dist/create/server-source/crates/feltdb/src/workload_diagnostics.rs +3 -4
- package/dist/create/server-source/crates/feltdb/tests/bounded_read_contract.rs +132 -0
- package/dist/create/server-source/crates/feltdb/tests/branching_evidence.rs +299 -0
- package/dist/create/server-source/crates/feltdb/tests/compaction_stall_contract.rs +272 -0
- package/dist/create/server-source/crates/feltdb/tests/crash_durability_contract.rs +467 -0
- package/dist/create/server-source/crates/feltdb/tests/current_revision_authority_evidence.rs +309 -0
- package/dist/create/server-source/crates/feltdb/tests/durable_backup_contract.rs +445 -0
- package/dist/create/server-source/crates/feltdb/tests/durable_corruption_contract.rs +518 -0
- package/dist/create/server-source/crates/feltdb/tests/durable_format_compatibility.rs +392 -0
- package/dist/create/server-source/crates/feltdb/tests/feltdb_state_boundary_tests.rs +436 -220
- package/dist/create/server-source/crates/feltdb/tests/fixtures/state_conflict_contract_corpus.json +1916 -0
- package/dist/create/server-source/crates/feltdb/tests/fixtures/state_diff_contract_corpus.json +1878 -0
- package/dist/create/server-source/crates/feltdb/tests/fixtures/trigger_contract_corpus.json +1862 -0
- package/dist/create/server-source/crates/feltdb/tests/operational_health_contract.rs +278 -0
- package/dist/create/server-source/crates/feltdb/tests/pr34_query_collection.rs +2 -1
- package/dist/create/server-source/crates/feltdb/tests/pr35_equality_index.rs +80 -25
- package/dist/create/server-source/crates/feltdb/tests/pr7_self_authorization_proof.rs +5 -8
- package/dist/create/server-source/crates/feltdb/tests/pr8_vocabulary_assessment.rs +52 -44
- package/dist/create/server-source/crates/feltdb/tests/pr9_phase2_boundary_tests.rs +33 -16
- package/dist/create/server-source/crates/feltdb/tests/pr9_phase3a_path_a_tests.rs +22 -7
- package/dist/create/server-source/crates/feltdb/tests/pr9_phase3c_authorized_mutations.rs +41 -22
- package/dist/create/server-source/crates/feltdb/tests/pr9_phase3c_role_based_authorization.rs +25 -8
- package/dist/create/server-source/crates/feltdb/tests/pr9_phase3c_simple_auth_delete.rs +9 -6
- package/dist/create/server-source/crates/feltdb/tests/pr9_phase3c_team_delete_role_authorization.rs +120 -69
- package/dist/create/server-source/crates/feltdb/tests/pr9_teams_role_based_access.rs +21 -10
- package/dist/create/server-source/crates/feltdb/tests/production_readiness_contract.rs +1365 -0
- package/dist/create/server-source/crates/feltdb/tests/reconciliation_application.rs +868 -0
- package/dist/create/server-source/crates/feltdb/tests/reconciliation_wire_format_evidence.rs +221 -0
- package/dist/create/server-source/crates/feltdb/tests/replicated_history_contract.rs +417 -0
- package/dist/create/server-source/crates/feltdb/tests/resource_scoped_revisions.rs +338 -0
- package/dist/create/server-source/crates/feltdb/tests/revision_identity_contract.rs +1039 -0
- package/dist/create/server-source/crates/feltdb/tests/revision_model_decision.rs +739 -0
- package/dist/create/server-source/crates/feltdb/tests/revision_retention_boundary_evidence.rs +427 -0
- package/dist/create/server-source/crates/feltdb/tests/saas_authorization_integration.rs +3 -3
- package/dist/create/server-source/crates/feltdb/tests/saas_invitation_lifecycle.rs +25 -22
- package/dist/create/server-source/crates/feltdb/tests/state_conflict_contract_conformance.rs +1799 -0
- package/dist/create/server-source/crates/feltdb/tests/state_diff_contract_conformance.rs +1316 -0
- package/dist/create/server-source/crates/feltdb/tests/state_model_integration.rs +53 -61
- package/dist/create/server-source/crates/feltdb/tests/state_persistence_integration.rs +156 -61
- package/dist/create/server-source/crates/feltdb/tests/state_store_boundary_evidence.rs +299 -0
- package/dist/create/server-source/crates/feltdb/tests/sync_divergence_evidence.rs +255 -0
- package/dist/create/server-source/crates/feltdb/tests/three_way_input_boundary_evidence.rs +249 -0
- package/dist/create/server-source/crates/feltdb/tests/trigger_contract_conformance.rs +994 -0
- package/dist/create/server-source/crates/feltdb/tests/workload_envelope_contract.rs +442 -0
- package/dist/create/server-source/crates/feltdb-server/src/app_state.rs +16 -1
- package/dist/create/server-source/crates/feltdb-server/src/auth.rs +164 -13
- package/dist/create/server-source/crates/feltdb-server/src/main.rs +695 -47
- package/dist/create/server-source/crates/feltdb-server/src/metrics.rs +21 -0
- package/dist/studio-app/assets/{feltdb_wasm-CVQWgXO-.js → feltdb_wasm-C1VhI-U5.js} +1 -1
- package/dist/studio-app/assets/feltdb_wasm_bg-C8HXbAXb.wasm +0 -0
- package/dist/studio-app/assets/{index-DwgNAIIX.js → index-Bbos1m2U.js} +1 -1
- package/dist/studio-app/index.html +1 -1
- package/dist/wasm/feltdb_wasm_bg.wasm +0 -0
- package/dist/workload.d.ts +2 -1
- package/dist/workload.d.ts.map +1 -1
- package/package.json +1 -1
- package/dist/studio-app/assets/feltdb_wasm_bg-CNVpvaZV.wasm +0 -0
|
@@ -109,11 +109,14 @@ pub mod replication_manager;
|
|
|
109
109
|
pub mod replication_protocol;
|
|
110
110
|
mod routing;
|
|
111
111
|
pub mod sharding;
|
|
112
|
+
pub mod state_conflict_contract;
|
|
112
113
|
pub mod state_contract;
|
|
114
|
+
pub mod state_diff_contract;
|
|
113
115
|
pub mod state_facade;
|
|
114
116
|
pub mod state_hash;
|
|
115
117
|
pub mod state_model;
|
|
116
118
|
pub mod state_transition_store;
|
|
119
|
+
pub mod state_trigger;
|
|
117
120
|
mod storage;
|
|
118
121
|
pub mod submission;
|
|
119
122
|
mod sync;
|
|
@@ -126,6 +129,7 @@ mod transaction_invariants;
|
|
|
126
129
|
pub mod transaction_preconditions;
|
|
127
130
|
pub mod transactions;
|
|
128
131
|
mod trigger;
|
|
132
|
+
pub mod trigger_contract;
|
|
129
133
|
pub mod worker_mesh;
|
|
130
134
|
mod workflow;
|
|
131
135
|
#[cfg(test)]
|
|
@@ -222,14 +226,27 @@ pub use sharding::{
|
|
|
222
226
|
HotspotAlert, RebalanceOperation, ShardDistributionSummary, ShardId, ShardKey, ShardManager,
|
|
223
227
|
ShardMetrics, ShardRange, ShardingStrategy,
|
|
224
228
|
};
|
|
229
|
+
pub use state_conflict_contract::{
|
|
230
|
+
classify_states, StateConflictInput, StateConflictOutput, STATE_CONFLICT_CONTRACT_ID,
|
|
231
|
+
STATE_CONFLICT_CONTRACT_VERSION,
|
|
232
|
+
};
|
|
233
|
+
pub use state_diff_contract::{
|
|
234
|
+
diff_states, StateDiffInput, StateDiffOutput, STATE_DIFF_CONTRACT_ID,
|
|
235
|
+
STATE_DIFF_CONTRACT_VERSION,
|
|
236
|
+
};
|
|
225
237
|
pub use state_facade::FeltDBStateSystem;
|
|
226
238
|
pub use state_hash::{CanonicalState, StateHash};
|
|
227
239
|
pub use state_model::{
|
|
228
|
-
|
|
240
|
+
apply_reconciliation_plan, path_relation, paths_overlap, reconcile, resolve_path, ChangeKind,
|
|
241
|
+
ConflictClass, ConflictClassification, PathComponent, PathConflict, PathRelation,
|
|
229
242
|
ReconciliationPlan, Relationship, SemanticChange, SemanticDiff, StateId,
|
|
230
243
|
StateReconciliationResult, StateRevision, StateStore, StateTopology, STATE_MODEL_VERSION,
|
|
231
244
|
};
|
|
232
245
|
pub use state_transition_store::{MemoryStateTransitionStore, StateTransitionRecord};
|
|
246
|
+
pub use state_trigger::{
|
|
247
|
+
StateTrigger, StateTriggerChangeKind, StateTriggerError, StateTriggerPredicate,
|
|
248
|
+
StateTriggerStore, TriggeredWorkload,
|
|
249
|
+
};
|
|
233
250
|
pub use storage::{CheckpointData, FileStorage, MemoryStorage, Storage};
|
|
234
251
|
pub use submission::{SubmissionManager, SubmissionMetrics};
|
|
235
252
|
pub use sync::{ChangeLog, Conflict, ConflictDetector, PeerState, SyncMessage, SyncState};
|
|
@@ -239,6 +256,12 @@ pub use transactions::{
|
|
|
239
256
|
StateTransition, TransactionExecutor, TransitionResult,
|
|
240
257
|
};
|
|
241
258
|
pub use trigger::{Trigger, TriggerFilter, TriggerRegistry};
|
|
259
|
+
pub use trigger_contract::{
|
|
260
|
+
canonical_input_bytes, canonical_output_bytes, evaluate_canonical, evaluate_trigger,
|
|
261
|
+
ContractChange, ContractChangeKind, ContractErrorBody, ContractErrorCode, ContractPredicate,
|
|
262
|
+
ContractRuntime, ContractTrigger, MatchOutcome, MatchReason, TriggerMatchInput,
|
|
263
|
+
TriggerMatchOutput, TRIGGER_MATCH_CONTRACT_ID, TRIGGER_MATCH_CONTRACT_VERSION,
|
|
264
|
+
};
|
|
242
265
|
pub use workflow::{
|
|
243
266
|
BlockedReason, WorkflowGraph, WorkflowInstance, WorkflowNode, WorkflowOperation, WorkflowRef,
|
|
244
267
|
WorkflowState, WorkflowStepRef, WorkflowStepState,
|
|
@@ -272,7 +295,7 @@ pub type Result<T> = std::result::Result<T, FlowError>;
|
|
|
272
295
|
pub enum FlowError {
|
|
273
296
|
Io(std::io::Error),
|
|
274
297
|
Serde(serde_json::Error),
|
|
275
|
-
CorruptLogLine(
|
|
298
|
+
CorruptLogLine(Box<LogCorruption>),
|
|
276
299
|
CapabilityError(String),
|
|
277
300
|
/// A transaction precondition did not hold. Nothing was written.
|
|
278
301
|
///
|
|
@@ -285,6 +308,14 @@ pub enum FlowError {
|
|
|
285
308
|
expected: u64,
|
|
286
309
|
actual: u64,
|
|
287
310
|
},
|
|
311
|
+
/// A backup artifact was rejected. Nothing was restored.
|
|
312
|
+
BackupRejected(Box<BackupProblem>),
|
|
313
|
+
/// The durable database is in a format this build does not understand.
|
|
314
|
+
///
|
|
315
|
+
/// Returned by [`FeltDb::open`] **before any record is interpreted**, and
|
|
316
|
+
/// before anything is written. A database that returns this error is
|
|
317
|
+
/// untouched.
|
|
318
|
+
IncompatibleFormat(Box<FormatIncompatibility>),
|
|
288
319
|
}
|
|
289
320
|
|
|
290
321
|
impl Display for FlowError {
|
|
@@ -292,13 +323,17 @@ impl Display for FlowError {
|
|
|
292
323
|
match self {
|
|
293
324
|
FlowError::Io(e) => write!(f, "io error: {e}"),
|
|
294
325
|
FlowError::Serde(e) => write!(f, "serde error: {e}"),
|
|
295
|
-
FlowError::CorruptLogLine(
|
|
326
|
+
FlowError::CorruptLogLine(corruption) => write!(f, "CORRUPT_DURABLE_LOG: {corruption}"),
|
|
296
327
|
FlowError::CapabilityError(msg) => write!(f, "capability error: {msg}"),
|
|
297
328
|
FlowError::PreconditionFailed(failure) => write!(f, "PRECONDITION_FAILED: {failure}"),
|
|
298
329
|
FlowError::RevisionConflict { expected, actual } => write!(
|
|
299
330
|
f,
|
|
300
331
|
"REVISION_CONFLICT: expected authority revision {expected}, current revision is {actual}"
|
|
301
332
|
),
|
|
333
|
+
FlowError::IncompatibleFormat(incompatibility) => {
|
|
334
|
+
write!(f, "INCOMPATIBLE_DURABLE_FORMAT: {incompatibility}")
|
|
335
|
+
}
|
|
336
|
+
FlowError::BackupRejected(problem) => write!(f, "BACKUP_REJECTED: {problem}"),
|
|
302
337
|
}
|
|
303
338
|
}
|
|
304
339
|
}
|
|
@@ -361,6 +396,29 @@ struct Inner {
|
|
|
361
396
|
/// inside the same state mutation boundary as `rows` and rebuilt from
|
|
362
397
|
/// `rows` after recovery.
|
|
363
398
|
equality_index: EqualityIndex,
|
|
399
|
+
/// What replaying the durable log had to do to succeed.
|
|
400
|
+
///
|
|
401
|
+
/// Recorded rather than logged, so an operator can ask after the fact
|
|
402
|
+
/// whether this database opened cleanly.
|
|
403
|
+
log_recovery: LogRecovery,
|
|
404
|
+
/// The durable format this database was accepted as, recorded at the open
|
|
405
|
+
/// that accepted it.
|
|
406
|
+
durable_format: DurableFormat,
|
|
407
|
+
/// How far a single-record write is pushed before it returns.
|
|
408
|
+
durability_mode: DurabilityMode,
|
|
409
|
+
/// Writes since the last stable-storage barrier, for grouped durability.
|
|
410
|
+
writes_since_barrier: u32,
|
|
411
|
+
/// When a policy-driven compaction should rewrite the durable log.
|
|
412
|
+
compaction_policy: CompactionPolicy,
|
|
413
|
+
/// Operations pruned since the last log rewrite.
|
|
414
|
+
pruned_since_rewrite: usize,
|
|
415
|
+
/// Each resource's newest revision, as `(id, sequence)`.
|
|
416
|
+
///
|
|
417
|
+
/// Derived from the `state` rows exactly like `equality_index` is derived
|
|
418
|
+
/// from `rows`: never durable, rebuilt after recovery. It exists so that
|
|
419
|
+
/// minting a revision does not have to scan a resource's history to find
|
|
420
|
+
/// the parent it descends from.
|
|
421
|
+
revision_heads: HashMap<String, (state_model::StateId, u64)>,
|
|
364
422
|
}
|
|
365
423
|
|
|
366
424
|
/// The state lock, held with its wait and hold time attributable.
|
|
@@ -732,11 +790,627 @@ pub struct StoredRow {
|
|
|
732
790
|
pub operation: Option<Operation>,
|
|
733
791
|
}
|
|
734
792
|
|
|
793
|
+
/// When a compaction should actually rewrite the durable log.
|
|
794
|
+
///
|
|
795
|
+
/// Pruning acknowledged operations from memory is cheap. **Rewriting the log is
|
|
796
|
+
/// not**: it serializes every row in the database and fsyncs it while holding
|
|
797
|
+
/// the one lock that serializes every read and every write. Doing that on a
|
|
798
|
+
/// short timer stops the world on that timer.
|
|
799
|
+
///
|
|
800
|
+
/// The policy separates the two. Pruning still happens whenever it can; the
|
|
801
|
+
/// rewrite waits until enough has accumulated to be worth the stall.
|
|
802
|
+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
|
803
|
+
pub struct CompactionPolicy {
|
|
804
|
+
/// Operations that must have been pruned since the last rewrite before the
|
|
805
|
+
/// next one is worth its cost.
|
|
806
|
+
pub rewrite_after_pruned: usize,
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
impl Default for CompactionPolicy {
|
|
810
|
+
fn default() -> Self {
|
|
811
|
+
// Chosen so a busy database rewrites occasionally rather than
|
|
812
|
+
// constantly. At one operation per write this is one rewrite per
|
|
813
|
+
// thousand acknowledged writes instead of one every timer tick.
|
|
814
|
+
CompactionPolicy {
|
|
815
|
+
rewrite_after_pruned: 1000,
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
/// What a policy-driven compaction did.
|
|
821
|
+
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
822
|
+
pub enum CompactionOutcome {
|
|
823
|
+
/// Nothing was acknowledged by every active peer; there was no work.
|
|
824
|
+
Idle,
|
|
825
|
+
/// Operations were pruned from memory and the log rewrite was **deferred**,
|
|
826
|
+
/// because the accumulated gain does not yet justify the stall.
|
|
827
|
+
Deferred {
|
|
828
|
+
pruned: usize,
|
|
829
|
+
pending_since_rewrite: usize,
|
|
830
|
+
},
|
|
831
|
+
/// The durable log was rewritten.
|
|
832
|
+
Rewritten { pruned: usize },
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
impl CompactionOutcome {
|
|
836
|
+
/// Operations pruned from the in-memory log by this call.
|
|
837
|
+
pub fn pruned(&self) -> usize {
|
|
838
|
+
match self {
|
|
839
|
+
CompactionOutcome::Idle => 0,
|
|
840
|
+
CompactionOutcome::Deferred { pruned, .. }
|
|
841
|
+
| CompactionOutcome::Rewritten { pruned } => *pruned,
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
|
|
845
|
+
/// Whether this call rewrote the durable log.
|
|
846
|
+
pub fn rewrote_log(&self) -> bool {
|
|
847
|
+
matches!(self, CompactionOutcome::Rewritten { .. })
|
|
848
|
+
}
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
/// How far a single-record write is pushed before it returns.
|
|
852
|
+
///
|
|
853
|
+
/// This is a **contract**, not a tuning knob: it decides what a successful
|
|
854
|
+
/// mutation means. Nothing here makes a claim about surviving power loss —
|
|
855
|
+
/// see [`DurabilityMode::guarantee`] for what each mode does and does not
|
|
856
|
+
/// establish.
|
|
857
|
+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
|
858
|
+
pub enum DurabilityMode {
|
|
859
|
+
/// Write and flush to the operating system. **The default.**
|
|
860
|
+
///
|
|
861
|
+
/// The bytes leave the process before the call returns, so a committed
|
|
862
|
+
/// write survives the process dying. They are not forced to the device.
|
|
863
|
+
Flushed,
|
|
864
|
+
/// Write, flush, and `fsync` before returning.
|
|
865
|
+
///
|
|
866
|
+
/// Issues a stable-storage barrier per mutation. Whether that barrier
|
|
867
|
+
/// actually reaches the platter depends on the filesystem, the mount
|
|
868
|
+
/// options and the drive's write cache — none of which this database can
|
|
869
|
+
/// observe, so this mode buys a barrier, not a proof.
|
|
870
|
+
Synced,
|
|
871
|
+
/// Flush every write, and `fsync` once every `every` writes.
|
|
872
|
+
///
|
|
873
|
+
/// Amortises the barrier. The cost is an explicit window: up to `every - 1`
|
|
874
|
+
/// acknowledged writes may not yet have been through one.
|
|
875
|
+
Grouped { every: u32 },
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
impl Default for DurabilityMode {
|
|
879
|
+
fn default() -> Self {
|
|
880
|
+
DurabilityMode::Flushed
|
|
881
|
+
}
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
impl DurabilityMode {
|
|
885
|
+
/// What a successful mutation guarantees under this mode, stated so that a
|
|
886
|
+
/// caller can read the contract rather than infer it from an API name.
|
|
887
|
+
pub fn guarantee(&self) -> &'static str {
|
|
888
|
+
match self {
|
|
889
|
+
DurabilityMode::Flushed => {
|
|
890
|
+
"the record has left the process; it survives a process crash. Stable-storage durability is not established."
|
|
891
|
+
}
|
|
892
|
+
DurabilityMode::Synced => {
|
|
893
|
+
"a stable-storage barrier was issued for this record. Whether it reached the device depends on the storage stack, which this database cannot observe."
|
|
894
|
+
}
|
|
895
|
+
DurabilityMode::Grouped { .. } => {
|
|
896
|
+
"the record has left the process. A stable-storage barrier is issued periodically, so an acknowledged write may not yet have been through one."
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
/// How many acknowledged writes may not yet have been through a barrier.
|
|
902
|
+
///
|
|
903
|
+
/// `None` where no barrier is issued at all.
|
|
904
|
+
pub fn unbarriered_window(&self) -> Option<u32> {
|
|
905
|
+
match self {
|
|
906
|
+
DurabilityMode::Flushed => None,
|
|
907
|
+
DurabilityMode::Synced => Some(0),
|
|
908
|
+
DurabilityMode::Grouped { every } => Some(every.saturating_sub(1)),
|
|
909
|
+
}
|
|
910
|
+
}
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
/// What is actually known about this database's durable storage.
|
|
914
|
+
///
|
|
915
|
+
/// Every variant corresponds to a fact the open established. There is no
|
|
916
|
+
/// variant meaning "probably fine", and none that says anything about power
|
|
917
|
+
/// loss — that is unproven, and health may not manufacture a guarantee the
|
|
918
|
+
/// database does not have.
|
|
919
|
+
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
920
|
+
pub enum StorageHealth {
|
|
921
|
+
/// Every durable byte replayed as a complete, valid record.
|
|
922
|
+
Clean,
|
|
923
|
+
/// Operational, **and the last write did not land**.
|
|
924
|
+
///
|
|
925
|
+
/// The log ended in an incomplete append, which was discarded. The database
|
|
926
|
+
/// is usable and is not in the same condition as one that replayed cleanly,
|
|
927
|
+
/// and reporting them identically is what makes an operator unable to tell
|
|
928
|
+
/// a healthy database from one that lost its most recent write.
|
|
929
|
+
RecoveredIncompleteWrite {
|
|
930
|
+
byte_offset: u64,
|
|
931
|
+
discarded_bytes: usize,
|
|
932
|
+
},
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
impl StorageHealth {
|
|
936
|
+
/// Whether the durable log replayed without discarding anything.
|
|
937
|
+
pub fn is_clean(&self) -> bool {
|
|
938
|
+
matches!(self, StorageHealth::Clean)
|
|
939
|
+
}
|
|
940
|
+
|
|
941
|
+
/// A short stable label for an operational surface.
|
|
942
|
+
pub fn label(&self) -> &'static str {
|
|
943
|
+
match self {
|
|
944
|
+
StorageHealth::Clean => "clean",
|
|
945
|
+
StorageHealth::RecoveredIncompleteWrite { .. } => "recovered-incomplete-write",
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
}
|
|
949
|
+
|
|
950
|
+
impl Display for StorageHealth {
|
|
951
|
+
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
|
952
|
+
match self {
|
|
953
|
+
StorageHealth::Clean => write!(f, "durable log replayed cleanly"),
|
|
954
|
+
StorageHealth::RecoveredIncompleteWrite {
|
|
955
|
+
byte_offset,
|
|
956
|
+
discarded_bytes,
|
|
957
|
+
} => write!(
|
|
958
|
+
f,
|
|
959
|
+
"durable log ended in an incomplete append: {discarded_bytes} bytes at offset \
|
|
960
|
+
{byte_offset} were discarded, so the last write did not land"
|
|
961
|
+
),
|
|
962
|
+
}
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
/// What this database knows about itself, as facts rather than labels.
|
|
967
|
+
///
|
|
968
|
+
/// Deliberately small. It reports only conditions the open actually
|
|
969
|
+
/// established, and it is not an operations dashboard: replication health,
|
|
970
|
+
/// backup freshness and capacity are not here, because none of them has a
|
|
971
|
+
/// defined operational contract yet and health must not invent one.
|
|
972
|
+
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
973
|
+
pub struct DatabaseHealth {
|
|
974
|
+
/// The durable format this database was accepted as.
|
|
975
|
+
///
|
|
976
|
+
/// A database in a format this build does not understand never opens, so
|
|
977
|
+
/// reaching this at all is itself the compatibility fact.
|
|
978
|
+
pub durable_format: DurableFormat,
|
|
979
|
+
/// What replaying the durable log had to do.
|
|
980
|
+
pub recovery: LogRecovery,
|
|
981
|
+
/// The storage condition that follows from it.
|
|
982
|
+
pub storage: StorageHealth,
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
impl DatabaseHealth {
|
|
986
|
+
/// Whether this database is in the condition a clean open produces.
|
|
987
|
+
///
|
|
988
|
+
/// **Not** a claim that the database is correct, replicated, backed up or
|
|
989
|
+
/// durable against power loss. It says the durable log replayed with
|
|
990
|
+
/// nothing discarded, and nothing more.
|
|
991
|
+
pub fn is_nominal(&self) -> bool {
|
|
992
|
+
self.storage.is_clean()
|
|
993
|
+
}
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
/// The backup artifact format this build writes and understands.
|
|
997
|
+
///
|
|
998
|
+
/// Versioned separately from the on-disk database: a backup is an artifact an
|
|
999
|
+
/// operator keeps, moves and restores long after the database that produced it,
|
|
1000
|
+
/// so its compatibility boundary is its own.
|
|
1001
|
+
pub const BACKUP_VERSION: u32 = 1;
|
|
1002
|
+
|
|
1003
|
+
const BACKUP_RECORD_TYPE: &str = "feltdb.backup.v1";
|
|
1004
|
+
|
|
1005
|
+
/// The first line of a backup artifact. Everything needed to verify the rest.
|
|
1006
|
+
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
1007
|
+
struct BackupHeader {
|
|
1008
|
+
record_type: String,
|
|
1009
|
+
backup_version: u32,
|
|
1010
|
+
/// The durable format the rows are in.
|
|
1011
|
+
format_version: u32,
|
|
1012
|
+
created_unix_ms: u64,
|
|
1013
|
+
row_count: usize,
|
|
1014
|
+
observed_versions: HashMap<String, u64>,
|
|
1015
|
+
/// Digest of the rows exactly as written. Detects any alteration.
|
|
1016
|
+
content_digest: String,
|
|
1017
|
+
/// Digest of what the database *means* — see [`BackupVerification`].
|
|
1018
|
+
meaning_digest: String,
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
/// Why a backup artifact was rejected.
|
|
1022
|
+
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
1023
|
+
pub struct BackupProblem {
|
|
1024
|
+
/// What is wrong, in terms of the artifact rather than the parser.
|
|
1025
|
+
pub reason: String,
|
|
1026
|
+
/// Where, when the problem has a location.
|
|
1027
|
+
pub line_number: Option<usize>,
|
|
1028
|
+
/// What the operator can do.
|
|
1029
|
+
pub action: String,
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
impl Display for BackupProblem {
|
|
1033
|
+
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
|
1034
|
+
match self.line_number {
|
|
1035
|
+
Some(line) => write!(f, "record {line}: {}. {}", self.reason, self.action),
|
|
1036
|
+
None => write!(f, "{}. {}", self.reason, self.action),
|
|
1037
|
+
}
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
|
|
1041
|
+
/// What a backup artifact contains, established by reading it.
|
|
1042
|
+
///
|
|
1043
|
+
/// `meaning_digest` is the part that matters. A content digest proves the file
|
|
1044
|
+
/// is intact; it does not prove that restoring it produces the same database.
|
|
1045
|
+
/// The meaning digest covers what the database *is*, independent of how it was
|
|
1046
|
+
/// written down:
|
|
1047
|
+
///
|
|
1048
|
+
/// - every live record: capability, key and canonical value;
|
|
1049
|
+
/// - every revision: its resource, identity, parent, sequence and content
|
|
1050
|
+
/// identity, and the authority that produced it;
|
|
1051
|
+
/// - every resource's retention policy and horizon.
|
|
1052
|
+
///
|
|
1053
|
+
/// It deliberately excludes wall-clock timestamps and sync envelopes, which
|
|
1054
|
+
/// differ between two databases that mean the same thing. Restore recomputes it
|
|
1055
|
+
/// from the **restored database** and refuses if it disagrees with the
|
|
1056
|
+
/// artifact, so a restore proves equivalence rather than mere parseability.
|
|
1057
|
+
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
1058
|
+
pub struct BackupVerification {
|
|
1059
|
+
pub backup_version: u32,
|
|
1060
|
+
pub format_version: u32,
|
|
1061
|
+
pub created_unix_ms: u64,
|
|
1062
|
+
pub rows: usize,
|
|
1063
|
+
pub content_digest: String,
|
|
1064
|
+
pub meaning_digest: String,
|
|
1065
|
+
}
|
|
1066
|
+
|
|
1067
|
+
fn backup_problem(reason: &str, line_number: Option<usize>, action: &str) -> FlowError {
|
|
1068
|
+
FlowError::BackupRejected(Box::new(BackupProblem {
|
|
1069
|
+
reason: reason.to_string(),
|
|
1070
|
+
line_number,
|
|
1071
|
+
action: action.to_string(),
|
|
1072
|
+
}))
|
|
1073
|
+
}
|
|
1074
|
+
|
|
1075
|
+
/// Digest the rows exactly as they will be written.
|
|
1076
|
+
fn backup_content_digest(rows: &[StoredRow]) -> Result<String> {
|
|
1077
|
+
let mut hasher = Sha256::new();
|
|
1078
|
+
for row in rows {
|
|
1079
|
+
hasher.update(serde_json::to_vec(row)?);
|
|
1080
|
+
hasher.update(b"\n");
|
|
1081
|
+
}
|
|
1082
|
+
Ok(format!("{:x}", hasher.finalize()))
|
|
1083
|
+
}
|
|
1084
|
+
|
|
1085
|
+
/// Digest what a set of rows *means*, independent of how it was written.
|
|
1086
|
+
///
|
|
1087
|
+
/// Canonical by construction: rows are ordered by capability and key, and only
|
|
1088
|
+
/// the fields that carry meaning take part. Two databases with the same digest
|
|
1089
|
+
/// hold the same records, the same revision graph and the same retention state.
|
|
1090
|
+
fn backup_meaning_digest(rows: &[StoredRow]) -> Result<String> {
|
|
1091
|
+
let mut meaningful: Vec<(&str, &str, String)> = rows
|
|
1092
|
+
.iter()
|
|
1093
|
+
.filter(|row| !row.deleted)
|
|
1094
|
+
.map(|row| {
|
|
1095
|
+
(
|
|
1096
|
+
row.capability.as_str(),
|
|
1097
|
+
row.key.as_str(),
|
|
1098
|
+
row.value.to_string(),
|
|
1099
|
+
)
|
|
1100
|
+
})
|
|
1101
|
+
.collect();
|
|
1102
|
+
meaningful.sort();
|
|
1103
|
+
|
|
1104
|
+
let mut hasher = Sha256::new();
|
|
1105
|
+
hasher.update(b"feltdb.backup.meaning.v1");
|
|
1106
|
+
for (capability, key, value) in meaningful {
|
|
1107
|
+
hasher.update(capability.as_bytes());
|
|
1108
|
+
hasher.update([0u8]);
|
|
1109
|
+
hasher.update(key.as_bytes());
|
|
1110
|
+
hasher.update([0u8]);
|
|
1111
|
+
hasher.update(value.as_bytes());
|
|
1112
|
+
hasher.update([0u8]);
|
|
1113
|
+
}
|
|
1114
|
+
Ok(format!("{:x}", hasher.finalize()))
|
|
1115
|
+
}
|
|
1116
|
+
|
|
1117
|
+
/// Read and check a backup artifact **without opening it as a database**.
|
|
1118
|
+
///
|
|
1119
|
+
/// Structural and semantic checks only, and it never writes: verifying a backup
|
|
1120
|
+
/// must be something an operator can do to an artifact they are not yet willing
|
|
1121
|
+
/// to restore.
|
|
1122
|
+
pub fn verify_backup(path: &Path) -> Result<BackupVerification> {
|
|
1123
|
+
let text = std::fs::read_to_string(path)?;
|
|
1124
|
+
let mut lines = text.lines();
|
|
1125
|
+
|
|
1126
|
+
let Some(header_line) = lines.next() else {
|
|
1127
|
+
return Err(backup_problem(
|
|
1128
|
+
"the artifact is empty",
|
|
1129
|
+
None,
|
|
1130
|
+
"Restore from a different backup.",
|
|
1131
|
+
));
|
|
1132
|
+
};
|
|
1133
|
+
let header: BackupHeader = serde_json::from_str(header_line).map_err(|error| {
|
|
1134
|
+
backup_problem(
|
|
1135
|
+
&format!("the header is not a FeltDB backup header: {error}"),
|
|
1136
|
+
Some(1),
|
|
1137
|
+
"Check that this file is a FeltDB backup and was not truncated at the start.",
|
|
1138
|
+
)
|
|
1139
|
+
})?;
|
|
1140
|
+
if header.record_type != BACKUP_RECORD_TYPE {
|
|
1141
|
+
return Err(backup_problem(
|
|
1142
|
+
&format!("unexpected record type {}", header.record_type),
|
|
1143
|
+
Some(1),
|
|
1144
|
+
"Check that this file is a FeltDB backup.",
|
|
1145
|
+
));
|
|
1146
|
+
}
|
|
1147
|
+
if header.backup_version != BACKUP_VERSION {
|
|
1148
|
+
return Err(backup_problem(
|
|
1149
|
+
&format!(
|
|
1150
|
+
"backup format version {} is not the version this build reads ({BACKUP_VERSION})",
|
|
1151
|
+
header.backup_version
|
|
1152
|
+
),
|
|
1153
|
+
Some(1),
|
|
1154
|
+
"Restore with a FeltDB build that reads this backup version.",
|
|
1155
|
+
));
|
|
1156
|
+
}
|
|
1157
|
+
if header.format_version != DURABLE_FORMAT_VERSION {
|
|
1158
|
+
return Err(backup_problem(
|
|
1159
|
+
&format!(
|
|
1160
|
+
"the rows are in durable format version {} and this build requires version {DURABLE_FORMAT_VERSION}",
|
|
1161
|
+
header.format_version
|
|
1162
|
+
),
|
|
1163
|
+
Some(1),
|
|
1164
|
+
"Restore with a FeltDB build matching the format the backup was taken in.",
|
|
1165
|
+
));
|
|
1166
|
+
}
|
|
1167
|
+
|
|
1168
|
+
let mut rows = Vec::with_capacity(header.row_count);
|
|
1169
|
+
for (index, line) in lines.enumerate() {
|
|
1170
|
+
if line.trim().is_empty() {
|
|
1171
|
+
continue;
|
|
1172
|
+
}
|
|
1173
|
+
let row: StoredRow = serde_json::from_str(line).map_err(|error| {
|
|
1174
|
+
backup_problem(
|
|
1175
|
+
&format!("not a durable record: {error}"),
|
|
1176
|
+
Some(index + 2),
|
|
1177
|
+
"The backup is damaged. Restore from another copy.",
|
|
1178
|
+
)
|
|
1179
|
+
})?;
|
|
1180
|
+
rows.push(row);
|
|
1181
|
+
}
|
|
1182
|
+
|
|
1183
|
+
if rows.len() != header.row_count {
|
|
1184
|
+
return Err(backup_problem(
|
|
1185
|
+
&format!(
|
|
1186
|
+
"the header declares {} records and the artifact holds {}",
|
|
1187
|
+
header.row_count,
|
|
1188
|
+
rows.len()
|
|
1189
|
+
),
|
|
1190
|
+
None,
|
|
1191
|
+
"The backup is truncated or was appended to. Restore from another copy.",
|
|
1192
|
+
));
|
|
1193
|
+
}
|
|
1194
|
+
|
|
1195
|
+
let content_digest = backup_content_digest(&rows)?;
|
|
1196
|
+
if content_digest != header.content_digest {
|
|
1197
|
+
return Err(backup_problem(
|
|
1198
|
+
"the records do not match the digest in the header",
|
|
1199
|
+
None,
|
|
1200
|
+
"The backup has been altered or corrupted. Restore from another copy.",
|
|
1201
|
+
));
|
|
1202
|
+
}
|
|
1203
|
+
|
|
1204
|
+
let meaning_digest = backup_meaning_digest(&rows)?;
|
|
1205
|
+
if meaning_digest != header.meaning_digest {
|
|
1206
|
+
return Err(backup_problem(
|
|
1207
|
+
"the records do not mean what the header says they mean",
|
|
1208
|
+
None,
|
|
1209
|
+
"The backup has been altered. Restore from another copy.",
|
|
1210
|
+
));
|
|
1211
|
+
}
|
|
1212
|
+
|
|
1213
|
+
Ok(BackupVerification {
|
|
1214
|
+
backup_version: header.backup_version,
|
|
1215
|
+
format_version: header.format_version,
|
|
1216
|
+
created_unix_ms: header.created_unix_ms,
|
|
1217
|
+
rows: rows.len(),
|
|
1218
|
+
content_digest,
|
|
1219
|
+
meaning_digest,
|
|
1220
|
+
})
|
|
1221
|
+
}
|
|
1222
|
+
|
|
1223
|
+
/// Where durable corruption was found, and what is wrong with it.
|
|
1224
|
+
///
|
|
1225
|
+
/// Carries a location so an operator can find the damage, and an excerpt so
|
|
1226
|
+
/// they can recognise it, without the error having to quote a whole record.
|
|
1227
|
+
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
1228
|
+
pub struct LogCorruption {
|
|
1229
|
+
/// 1-based line number in the durable log.
|
|
1230
|
+
pub line_number: usize,
|
|
1231
|
+
/// Byte offset of the start of the record.
|
|
1232
|
+
pub byte_offset: u64,
|
|
1233
|
+
/// What is wrong, in terms of the record rather than the parser.
|
|
1234
|
+
pub reason: String,
|
|
1235
|
+
/// The beginning of the record, truncated.
|
|
1236
|
+
pub excerpt: String,
|
|
1237
|
+
}
|
|
1238
|
+
|
|
1239
|
+
impl Display for LogCorruption {
|
|
1240
|
+
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
|
1241
|
+
write!(
|
|
1242
|
+
f,
|
|
1243
|
+
"durable log record {} at byte {} is corrupt: {}. Excerpt: {}. \
|
|
1244
|
+
The database was not opened and was not modified.",
|
|
1245
|
+
self.line_number, self.byte_offset, self.reason, self.excerpt
|
|
1246
|
+
)
|
|
1247
|
+
}
|
|
1248
|
+
}
|
|
1249
|
+
|
|
1250
|
+
/// What replaying the durable log had to do to succeed.
|
|
1251
|
+
///
|
|
1252
|
+
/// A clean open and an open that discarded an incomplete final append are
|
|
1253
|
+
/// different facts, and the difference must survive past an internal debug
|
|
1254
|
+
/// message: an operator needs to know their database opened *and* that its last
|
|
1255
|
+
/// write did not land.
|
|
1256
|
+
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
1257
|
+
pub enum LogRecovery {
|
|
1258
|
+
/// Every durable byte was a complete, valid record.
|
|
1259
|
+
Clean,
|
|
1260
|
+
/// The log ended in an incomplete append.
|
|
1261
|
+
///
|
|
1262
|
+
/// The final bytes carried no record terminator, so they cannot be a record
|
|
1263
|
+
/// this database finished writing. They were not replayed, and the log was
|
|
1264
|
+
/// truncated back to the last complete record so that the next append does
|
|
1265
|
+
/// not splice onto a partial line.
|
|
1266
|
+
///
|
|
1267
|
+
/// **This is the only condition under which durable bytes are discarded.**
|
|
1268
|
+
/// A *complete* record that is invalid is corruption, not a torn write, and
|
|
1269
|
+
/// refuses the open however close to the end of the file it sits.
|
|
1270
|
+
RecoveredTornTail {
|
|
1271
|
+
/// Byte offset where the incomplete append began.
|
|
1272
|
+
byte_offset: u64,
|
|
1273
|
+
/// How many bytes were discarded.
|
|
1274
|
+
discarded_bytes: usize,
|
|
1275
|
+
},
|
|
1276
|
+
}
|
|
1277
|
+
|
|
1278
|
+
impl Default for LogRecovery {
|
|
1279
|
+
fn default() -> Self {
|
|
1280
|
+
LogRecovery::Clean
|
|
1281
|
+
}
|
|
1282
|
+
}
|
|
1283
|
+
|
|
1284
|
+
impl Default for DurableFormat {
|
|
1285
|
+
fn default() -> Self {
|
|
1286
|
+
DurableFormat::Versioned(DURABLE_FORMAT_VERSION)
|
|
1287
|
+
}
|
|
1288
|
+
}
|
|
1289
|
+
|
|
1290
|
+
impl LogRecovery {
|
|
1291
|
+
/// Whether the open replayed the log without discarding anything.
|
|
1292
|
+
pub fn is_clean(&self) -> bool {
|
|
1293
|
+
matches!(self, LogRecovery::Clean)
|
|
1294
|
+
}
|
|
1295
|
+
}
|
|
1296
|
+
|
|
1297
|
+
impl Display for LogRecovery {
|
|
1298
|
+
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
|
1299
|
+
match self {
|
|
1300
|
+
LogRecovery::Clean => write!(f, "clean"),
|
|
1301
|
+
LogRecovery::RecoveredTornTail {
|
|
1302
|
+
byte_offset,
|
|
1303
|
+
discarded_bytes,
|
|
1304
|
+
} => write!(
|
|
1305
|
+
f,
|
|
1306
|
+
"recovered from an incomplete final append: {discarded_bytes} bytes at offset \
|
|
1307
|
+
{byte_offset} were not a complete record and were discarded"
|
|
1308
|
+
),
|
|
1309
|
+
}
|
|
1310
|
+
}
|
|
1311
|
+
}
|
|
1312
|
+
|
|
1313
|
+
/// The durable format this build writes and understands.
|
|
1314
|
+
///
|
|
1315
|
+
/// Independent of the package version on purpose: an application release that
|
|
1316
|
+
/// changes nothing about persistence must not appear to change the database.
|
|
1317
|
+
///
|
|
1318
|
+
/// - **1** — everything before revisions became resource-scoped. Never written
|
|
1319
|
+
/// by any build; it names the era that predates format versioning.
|
|
1320
|
+
/// - **2** — resource-scoped revisions: a `StateRevision` carries `resource`,
|
|
1321
|
+
/// `content_id` and `sequence`.
|
|
1322
|
+
pub const DURABLE_FORMAT_VERSION: u32 = 2;
|
|
1323
|
+
|
|
1324
|
+
/// The log record that carries the format version.
|
|
1325
|
+
const FORMAT_RECORD_TYPE: &str = "feltdb.format.v1";
|
|
1326
|
+
|
|
1327
|
+
#[derive(Debug, Serialize, Deserialize)]
|
|
1328
|
+
struct FormatRecord {
|
|
1329
|
+
record_type: String,
|
|
1330
|
+
format_version: u32,
|
|
1331
|
+
}
|
|
1332
|
+
|
|
1333
|
+
/// What format a durable database is in.
|
|
1334
|
+
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
1335
|
+
pub enum DurableFormat {
|
|
1336
|
+
/// A format record states the version.
|
|
1337
|
+
Versioned(u32),
|
|
1338
|
+
/// No format record: the database predates format versioning.
|
|
1339
|
+
///
|
|
1340
|
+
/// This is a version, not an absence of one. It is the only case where the
|
|
1341
|
+
/// records themselves are examined to decide compatibility, and that
|
|
1342
|
+
/// examination is a deliberate one-time probe rather than a deserialization
|
|
1343
|
+
/// attempt whose failure is discarded.
|
|
1344
|
+
Unversioned,
|
|
1345
|
+
}
|
|
1346
|
+
|
|
1347
|
+
impl Display for DurableFormat {
|
|
1348
|
+
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
|
1349
|
+
match self {
|
|
1350
|
+
DurableFormat::Versioned(version) => write!(f, "version {version}"),
|
|
1351
|
+
DurableFormat::Unversioned => write!(f, "unversioned (predates format versioning)"),
|
|
1352
|
+
}
|
|
1353
|
+
}
|
|
1354
|
+
}
|
|
1355
|
+
|
|
1356
|
+
/// Why a durable database cannot be opened, and what to do about it.
|
|
1357
|
+
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
1358
|
+
pub struct FormatIncompatibility {
|
|
1359
|
+
/// What the database is.
|
|
1360
|
+
pub found: DurableFormat,
|
|
1361
|
+
/// What this build requires.
|
|
1362
|
+
pub required: u32,
|
|
1363
|
+
/// What is wrong, in terms of the data rather than the code.
|
|
1364
|
+
pub reason: String,
|
|
1365
|
+
/// What the operator can do.
|
|
1366
|
+
pub action: String,
|
|
1367
|
+
}
|
|
1368
|
+
|
|
1369
|
+
impl Display for FormatIncompatibility {
|
|
1370
|
+
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
|
1371
|
+
write!(
|
|
1372
|
+
f,
|
|
1373
|
+
"durable format {} is incompatible with this FeltDB, which requires \
|
|
1374
|
+
format version {}; {}. No data was read or modified. {}",
|
|
1375
|
+
self.found, self.required, self.reason, self.action
|
|
1376
|
+
)
|
|
1377
|
+
}
|
|
1378
|
+
}
|
|
1379
|
+
|
|
1380
|
+
/// The result of examining a durable database without opening it.
|
|
1381
|
+
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
1382
|
+
pub enum FormatCompatibility {
|
|
1383
|
+
/// Safe to open.
|
|
1384
|
+
Compatible(DurableFormat),
|
|
1385
|
+
/// Must not be opened.
|
|
1386
|
+
Incompatible(FormatIncompatibility),
|
|
1387
|
+
}
|
|
1388
|
+
|
|
1389
|
+
impl FormatCompatibility {
|
|
1390
|
+
/// Whether opening this database is safe.
|
|
1391
|
+
pub fn is_compatible(&self) -> bool {
|
|
1392
|
+
matches!(self, FormatCompatibility::Compatible(_))
|
|
1393
|
+
}
|
|
1394
|
+
}
|
|
1395
|
+
|
|
735
1396
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
736
1397
|
pub struct DatabaseSnapshot {
|
|
737
1398
|
pub rows: Vec<StoredRow>,
|
|
738
1399
|
pub versions: HashMap<String, u64>,
|
|
739
1400
|
pub content_hash: String,
|
|
1401
|
+
/// The durable format the rows are in.
|
|
1402
|
+
///
|
|
1403
|
+
/// A snapshot carries whole `StoredRow`s, revisions included, so it is a
|
|
1404
|
+
/// durable interchange artifact and not merely a protocol value: installing
|
|
1405
|
+
/// one from a different format would reintroduce exactly the silent
|
|
1406
|
+
/// misinterpretation the on-disk version exists to prevent.
|
|
1407
|
+
///
|
|
1408
|
+
/// Defaults to 0 so a snapshot serialized before versioning still
|
|
1409
|
+
/// deserializes; 0 is treated as unversioned and refused rather than
|
|
1410
|
+
/// assumed current. `content_hash` deliberately does not cover this field,
|
|
1411
|
+
/// so hashes computed before it existed remain valid.
|
|
1412
|
+
#[serde(default)]
|
|
1413
|
+
pub format_version: u32,
|
|
740
1414
|
}
|
|
741
1415
|
|
|
742
1416
|
impl DatabaseSnapshot {
|
|
@@ -764,6 +1438,10 @@ struct LogHeader {
|
|
|
764
1438
|
record_type: String,
|
|
765
1439
|
local_sequence: u64,
|
|
766
1440
|
observed_versions: HashMap<String, u64>,
|
|
1441
|
+
/// Present since format versioning. Absent in headers written before it,
|
|
1442
|
+
/// which deserialize as 0 and are classified as unversioned.
|
|
1443
|
+
#[serde(default)]
|
|
1444
|
+
format_version: u32,
|
|
767
1445
|
}
|
|
768
1446
|
|
|
769
1447
|
#[derive(Debug, Default, Serialize, Deserialize)]
|
|
@@ -777,14 +1455,303 @@ pub fn open<P: AsRef<Path>>(path: P) -> Result<FeltDb> {
|
|
|
777
1455
|
FeltDb::open(path)
|
|
778
1456
|
}
|
|
779
1457
|
|
|
1458
|
+
/// Capabilities whose rows are not themselves versioned.
|
|
1459
|
+
///
|
|
1460
|
+
/// `state` holds the revisions, and `_retention` holds the watermark that says
|
|
1461
|
+
/// how much of each resource's history survives. Minting a revision of either
|
|
1462
|
+
/// would mint a revision of the mint.
|
|
1463
|
+
const UNVERSIONED_CAPABILITIES: &[&str] = &["state", "_retention"];
|
|
1464
|
+
|
|
1465
|
+
/// Whether a write to this capability creates a revision.
|
|
1466
|
+
fn is_versioned(capability: &str) -> bool {
|
|
1467
|
+
!UNVERSIONED_CAPABILITIES.contains(&capability)
|
|
1468
|
+
}
|
|
1469
|
+
|
|
1470
|
+
/// Rebuild the derived head index from the durable revisions.
|
|
1471
|
+
///
|
|
1472
|
+
/// Called after recovery, for the same reason `equality_index` is: the index is
|
|
1473
|
+
/// a convenience over `rows` and must never be the authority for what `rows`
|
|
1474
|
+
/// contains.
|
|
1475
|
+
fn rebuild_revision_heads(inner: &mut Inner) {
|
|
1476
|
+
let mut heads: HashMap<String, (state_model::StateId, u64)> = HashMap::new();
|
|
1477
|
+
if let Some(bucket) = inner.rows.get("state") {
|
|
1478
|
+
for row in bucket.values() {
|
|
1479
|
+
if row.deleted || !row.key.starts_with("state:revision:") {
|
|
1480
|
+
continue;
|
|
1481
|
+
}
|
|
1482
|
+
let Ok(revision) =
|
|
1483
|
+
serde_json::from_value::<state_model::StateRevision>(row.value.clone())
|
|
1484
|
+
else {
|
|
1485
|
+
continue;
|
|
1486
|
+
};
|
|
1487
|
+
let entry = heads
|
|
1488
|
+
.entry(revision.resource.clone())
|
|
1489
|
+
.or_insert_with(|| (revision.id.clone(), revision.sequence));
|
|
1490
|
+
if revision.sequence >= entry.1 {
|
|
1491
|
+
*entry = (revision.id.clone(), revision.sequence);
|
|
1492
|
+
}
|
|
1493
|
+
}
|
|
1494
|
+
}
|
|
1495
|
+
inner.revision_heads = heads;
|
|
1496
|
+
}
|
|
1497
|
+
|
|
1498
|
+
/// Write one derived row inside an already-held state lock.
|
|
1499
|
+
///
|
|
1500
|
+
/// Deliberately **not** a mutation: no sequence number, no vector clock tick,
|
|
1501
|
+
/// no entry in the change log. A revision is a *derived* record of a mutation
|
|
1502
|
+
/// that already has all three, and giving it its own would make every write
|
|
1503
|
+
/// advance replication twice and double the operation stream.
|
|
1504
|
+
///
|
|
1505
|
+
/// It is still durable — the row is appended to the log and enters `rows`
|
|
1506
|
+
/// inside the same lock as the state it records, so a reader never observes a
|
|
1507
|
+
/// mutation whose revision is missing, and a reopened database has both.
|
|
1508
|
+
///
|
|
1509
|
+
/// The consequence is that revisions are **local and derived**, not replicated.
|
|
1510
|
+
/// That is sound in principle, because a revision's identity is a function of
|
|
1511
|
+
/// the resource, the content, the parent and the sequence — all of which a peer
|
|
1512
|
+
/// that applies the same operation stream has. Whether the sync-apply path
|
|
1513
|
+
/// actually mints is a separate question this change does not answer.
|
|
1514
|
+
fn append_derived_row_locked(
|
|
1515
|
+
inner: &mut Inner,
|
|
1516
|
+
capability: &str,
|
|
1517
|
+
key: &str,
|
|
1518
|
+
value: Value,
|
|
1519
|
+
) -> Result<()> {
|
|
1520
|
+
let row = StoredRow {
|
|
1521
|
+
capability: capability.to_string(),
|
|
1522
|
+
key: key.to_string(),
|
|
1523
|
+
rust_type: "feltdb::state_model::StateRevision".to_string(),
|
|
1524
|
+
value,
|
|
1525
|
+
unix_ms: now_ms(),
|
|
1526
|
+
content_hash: None,
|
|
1527
|
+
flow_ref: None,
|
|
1528
|
+
deleted: false,
|
|
1529
|
+
operation: None,
|
|
1530
|
+
};
|
|
1531
|
+
append_event_locked(inner, &row)?;
|
|
1532
|
+
inner.put_row(row);
|
|
1533
|
+
Ok(())
|
|
1534
|
+
}
|
|
1535
|
+
|
|
1536
|
+
/// Work out the revision an authoritative write will produce, without writing
|
|
1537
|
+
/// anything.
|
|
1538
|
+
///
|
|
1539
|
+
/// Separated from persisting it so the operation that carries this mutation can
|
|
1540
|
+
/// name the revision it produced. A peer then **reconstructs that revision**
|
|
1541
|
+
/// rather than minting a local substitute for the resulting state — which is
|
|
1542
|
+
/// the difference between replicating history and replicating the present.
|
|
1543
|
+
fn plan_revision(inner: &Inner, resource: &str, content: String) -> state_model::StateRevision {
|
|
1544
|
+
let parent = inner.revision_heads.get(resource).cloned();
|
|
1545
|
+
let (parent_id, sequence) = match &parent {
|
|
1546
|
+
Some((id, sequence)) => (Some(id.clone()), sequence + 1),
|
|
1547
|
+
None => (None, 0),
|
|
1548
|
+
};
|
|
1549
|
+
state_model::StateRevision::at(
|
|
1550
|
+
resource.to_string(),
|
|
1551
|
+
content,
|
|
1552
|
+
parent_id,
|
|
1553
|
+
sequence,
|
|
1554
|
+
inner.instance_id.clone(),
|
|
1555
|
+
)
|
|
1556
|
+
}
|
|
1557
|
+
|
|
1558
|
+
/// What a peer needs to rebuild this revision exactly.
|
|
1559
|
+
fn provenance_of(revision: &state_model::StateRevision) -> operation::RevisionProvenance {
|
|
1560
|
+
operation::RevisionProvenance {
|
|
1561
|
+
resource: revision.resource.clone(),
|
|
1562
|
+
id: revision.id.as_hex().to_string(),
|
|
1563
|
+
parent_id: revision
|
|
1564
|
+
.parent_id
|
|
1565
|
+
.as_ref()
|
|
1566
|
+
.map(|id| id.as_hex().to_string()),
|
|
1567
|
+
sequence: revision.sequence,
|
|
1568
|
+
content_id: revision.content_id.as_hex().to_string(),
|
|
1569
|
+
authority: revision.authority.clone(),
|
|
1570
|
+
}
|
|
1571
|
+
}
|
|
1572
|
+
|
|
1573
|
+
/// Persist a revision under the caller's lock, and apply the resource's
|
|
1574
|
+
/// retention policy.
|
|
1575
|
+
///
|
|
1576
|
+
/// Idempotent: a revision already stored is left exactly as it is. Committing
|
|
1577
|
+
/// the same historical fact twice — which replication does routinely — must not
|
|
1578
|
+
/// rewrite it.
|
|
1579
|
+
fn persist_revision_locked(inner: &mut Inner, revision: state_model::StateRevision) -> Result<()> {
|
|
1580
|
+
let key = format!("state:revision:{}", revision.id.as_hex());
|
|
1581
|
+
let already = inner
|
|
1582
|
+
.rows
|
|
1583
|
+
.get("state")
|
|
1584
|
+
.and_then(|bucket| bucket.get(&key))
|
|
1585
|
+
.is_some_and(|row| !row.deleted);
|
|
1586
|
+
|
|
1587
|
+
let resource = revision.resource.clone();
|
|
1588
|
+
let id = revision.id.clone();
|
|
1589
|
+
let sequence = revision.sequence;
|
|
1590
|
+
|
|
1591
|
+
if !already {
|
|
1592
|
+
let value = serde_json::to_value(&revision)?;
|
|
1593
|
+
append_derived_row_locked(inner, "state", &key, value)?;
|
|
1594
|
+
}
|
|
1595
|
+
|
|
1596
|
+
// The head advances only forward. A fork leaves two revisions at one
|
|
1597
|
+
// sequence; whichever is seen last does not become "the" head by accident.
|
|
1598
|
+
let advance = inner
|
|
1599
|
+
.revision_heads
|
|
1600
|
+
.get(&resource)
|
|
1601
|
+
.is_none_or(|(_, current)| sequence > *current);
|
|
1602
|
+
if advance {
|
|
1603
|
+
inner
|
|
1604
|
+
.revision_heads
|
|
1605
|
+
.insert(resource.clone(), (id, sequence));
|
|
1606
|
+
}
|
|
1607
|
+
|
|
1608
|
+
apply_retention_locked(inner, &resource)
|
|
1609
|
+
}
|
|
1610
|
+
|
|
1611
|
+
/// Rebuild the revision an operation carries, and check it is the one the
|
|
1612
|
+
/// originating authority actually committed.
|
|
1613
|
+
///
|
|
1614
|
+
/// The identity is recomputed from the parts rather than trusted: if the value
|
|
1615
|
+
/// that arrived is not the value the revision was computed from, or the stated
|
|
1616
|
+
/// identity does not follow from the stated resource, parent and sequence, the
|
|
1617
|
+
/// operation is refused rather than turned into a plausible local history.
|
|
1618
|
+
fn reconstruct_revision(
|
|
1619
|
+
provenance: &operation::RevisionProvenance,
|
|
1620
|
+
value: &Value,
|
|
1621
|
+
) -> Result<state_model::StateRevision> {
|
|
1622
|
+
let content = value.to_string();
|
|
1623
|
+
let revision = state_model::StateRevision::at(
|
|
1624
|
+
provenance.resource.clone(),
|
|
1625
|
+
content,
|
|
1626
|
+
provenance
|
|
1627
|
+
.parent_id
|
|
1628
|
+
.as_ref()
|
|
1629
|
+
.map(|id| state_model::StateId::from_hex(id.clone())),
|
|
1630
|
+
provenance.sequence,
|
|
1631
|
+
provenance.authority.clone(),
|
|
1632
|
+
);
|
|
1633
|
+
if revision.content_id.as_hex() != provenance.content_id {
|
|
1634
|
+
return Err(FlowError::CapabilityError(format!(
|
|
1635
|
+
"replicated revision {} does not match the value it arrived with",
|
|
1636
|
+
provenance.id
|
|
1637
|
+
)));
|
|
1638
|
+
}
|
|
1639
|
+
if revision.id.as_hex() != provenance.id {
|
|
1640
|
+
return Err(FlowError::CapabilityError(format!(
|
|
1641
|
+
"replicated revision identity {} does not follow from its own resource, parent and sequence",
|
|
1642
|
+
provenance.id
|
|
1643
|
+
)));
|
|
1644
|
+
}
|
|
1645
|
+
Ok(revision)
|
|
1646
|
+
}
|
|
1647
|
+
|
|
1648
|
+
/// Expire whatever the resource's configured policy no longer retains.
|
|
1649
|
+
/// Expire whatever the resource's configured policy no longer retains.
|
|
1650
|
+
///
|
|
1651
|
+
/// The common path is one map lookup: with no policy configured there is
|
|
1652
|
+
/// nothing to scan. A resource's history is only walked once a policy exists to
|
|
1653
|
+
/// bound it.
|
|
1654
|
+
fn apply_retention_locked(inner: &mut Inner, resource: &str) -> Result<()> {
|
|
1655
|
+
let retention_key = format!("_retention:{resource}");
|
|
1656
|
+
let Some(state_row) = inner
|
|
1657
|
+
.rows
|
|
1658
|
+
.get("_retention")
|
|
1659
|
+
.and_then(|bucket| bucket.get(&retention_key))
|
|
1660
|
+
else {
|
|
1661
|
+
return Ok(());
|
|
1662
|
+
};
|
|
1663
|
+
let Ok(mut state) = serde_json::from_value::<RetentionRecord>(state_row.value.clone()) else {
|
|
1664
|
+
return Ok(());
|
|
1665
|
+
};
|
|
1666
|
+
if state.policy.keep_last.is_none() {
|
|
1667
|
+
return Ok(());
|
|
1668
|
+
}
|
|
1669
|
+
|
|
1670
|
+
let mut history: Vec<(state_model::StateId, u64)> = Vec::new();
|
|
1671
|
+
if let Some(bucket) = inner.rows.get("state") {
|
|
1672
|
+
for row in bucket.values() {
|
|
1673
|
+
if row.deleted || !row.key.starts_with("state:revision:") {
|
|
1674
|
+
continue;
|
|
1675
|
+
}
|
|
1676
|
+
if let Ok(revision) =
|
|
1677
|
+
serde_json::from_value::<state_model::StateRevision>(row.value.clone())
|
|
1678
|
+
{
|
|
1679
|
+
if revision.resource == resource {
|
|
1680
|
+
history.push((revision.id, revision.sequence));
|
|
1681
|
+
}
|
|
1682
|
+
}
|
|
1683
|
+
}
|
|
1684
|
+
}
|
|
1685
|
+
history.sort_by_key(|(_, sequence)| *sequence);
|
|
1686
|
+
|
|
1687
|
+
let (expire, horizon) = state_model::revisions_to_expire(&history, state.policy.keep_last);
|
|
1688
|
+
if expire.is_empty() {
|
|
1689
|
+
return Ok(());
|
|
1690
|
+
}
|
|
1691
|
+
for id in &expire {
|
|
1692
|
+
let key = format!("state:revision:{}", id.as_hex());
|
|
1693
|
+
if let Some(bucket) = inner.rows.get_mut("state") {
|
|
1694
|
+
bucket.remove(&key);
|
|
1695
|
+
}
|
|
1696
|
+
let tombstone = StoredRow {
|
|
1697
|
+
capability: "state".to_string(),
|
|
1698
|
+
key,
|
|
1699
|
+
rust_type: "feltdb::state_model::StateRevision".to_string(),
|
|
1700
|
+
value: Value::Null,
|
|
1701
|
+
unix_ms: now_ms(),
|
|
1702
|
+
content_hash: None,
|
|
1703
|
+
flow_ref: None,
|
|
1704
|
+
deleted: true,
|
|
1705
|
+
operation: None,
|
|
1706
|
+
};
|
|
1707
|
+
append_event(&inner.path, &tombstone)?;
|
|
1708
|
+
}
|
|
1709
|
+
if let Some(horizon) = horizon {
|
|
1710
|
+
state.horizon = horizon;
|
|
1711
|
+
}
|
|
1712
|
+
let value = serde_json::to_value(&state)?;
|
|
1713
|
+
append_derived_row_locked(inner, "_retention", &retention_key, value)
|
|
1714
|
+
}
|
|
1715
|
+
|
|
1716
|
+
/// The durable shape of a resource's retention state.
|
|
1717
|
+
///
|
|
1718
|
+
/// Mirrors `state_model`'s private record so the write boundary can read and
|
|
1719
|
+
/// advance the horizon without the store type.
|
|
1720
|
+
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
|
1721
|
+
struct RetentionRecord {
|
|
1722
|
+
policy: state_model::RetentionPolicy,
|
|
1723
|
+
horizon: u64,
|
|
1724
|
+
}
|
|
1725
|
+
|
|
780
1726
|
impl FeltDb {
|
|
781
1727
|
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
|
|
782
1728
|
let path = path.as_ref().to_path_buf();
|
|
1729
|
+
|
|
1730
|
+
// Fail closed. The format is decided before any record is interpreted
|
|
1731
|
+
// and before anything is written, so a database this build does not
|
|
1732
|
+
// understand is refused with its bytes untouched rather than opened
|
|
1733
|
+
// with whatever happened to deserialize.
|
|
1734
|
+
let accepted_format = match inspect_durable_format(&path)? {
|
|
1735
|
+
FormatCompatibility::Incompatible(incompatibility) => {
|
|
1736
|
+
return Err(FlowError::IncompatibleFormat(Box::new(incompatibility)));
|
|
1737
|
+
}
|
|
1738
|
+
FormatCompatibility::Compatible(format) => format,
|
|
1739
|
+
};
|
|
1740
|
+
|
|
783
1741
|
if let Some(parent) = path.parent() {
|
|
784
1742
|
fs::create_dir_all(parent)?;
|
|
785
1743
|
}
|
|
786
1744
|
if !path.exists() {
|
|
787
|
-
OpenOptions::new().create(true).append(true).open(&path)?;
|
|
1745
|
+
let mut file = OpenOptions::new().create(true).append(true).open(&path)?;
|
|
1746
|
+
// A new database states its format in its first record, so it never
|
|
1747
|
+
// has to be inferred again.
|
|
1748
|
+
write_json_line(
|
|
1749
|
+
&mut file,
|
|
1750
|
+
&FormatRecord {
|
|
1751
|
+
record_type: FORMAT_RECORD_TYPE.to_string(),
|
|
1752
|
+
format_version: DURABLE_FORMAT_VERSION,
|
|
1753
|
+
},
|
|
1754
|
+
)?;
|
|
788
1755
|
}
|
|
789
1756
|
|
|
790
1757
|
// Generate instance ID from path hash
|
|
@@ -797,6 +1764,7 @@ impl FeltDb {
|
|
|
797
1764
|
let mut inner = Inner {
|
|
798
1765
|
path: path.clone(),
|
|
799
1766
|
instance_id: instance_id.clone(),
|
|
1767
|
+
durable_format: accepted_format,
|
|
800
1768
|
..Inner::default()
|
|
801
1769
|
};
|
|
802
1770
|
|
|
@@ -804,6 +1772,7 @@ impl FeltDb {
|
|
|
804
1772
|
inner.sync_state = SyncState::new(instance_id);
|
|
805
1773
|
replay_log(&mut inner)?;
|
|
806
1774
|
load_sync_metadata(&mut inner)?;
|
|
1775
|
+
rebuild_revision_heads(&mut inner);
|
|
807
1776
|
|
|
808
1777
|
let (event_tx, _) = broadcast::channel(1024);
|
|
809
1778
|
|
|
@@ -813,6 +1782,239 @@ impl FeltDb {
|
|
|
813
1782
|
})
|
|
814
1783
|
}
|
|
815
1784
|
|
|
1785
|
+
/// Write a self-contained durable backup artifact.
|
|
1786
|
+
///
|
|
1787
|
+
/// The artifact does not depend on this database remaining present: it
|
|
1788
|
+
/// carries every live record, the revision graph, retention state and the
|
|
1789
|
+
/// observed operation versions, plus the digests needed to check all of it
|
|
1790
|
+
/// without a live database.
|
|
1791
|
+
pub fn write_backup(&self, path: &Path) -> Result<BackupVerification> {
|
|
1792
|
+
let (rows, observed_versions) = {
|
|
1793
|
+
let inner = self.state();
|
|
1794
|
+
let mut rows: Vec<StoredRow> = inner
|
|
1795
|
+
.rows
|
|
1796
|
+
.values()
|
|
1797
|
+
.flat_map(|bucket| bucket.values().cloned())
|
|
1798
|
+
.filter(|row| !row.deleted)
|
|
1799
|
+
.collect();
|
|
1800
|
+
rows.sort_by(|left, right| {
|
|
1801
|
+
left.capability
|
|
1802
|
+
.cmp(&right.capability)
|
|
1803
|
+
.then_with(|| left.key.cmp(&right.key))
|
|
1804
|
+
});
|
|
1805
|
+
(rows, inner.change_log.versions())
|
|
1806
|
+
};
|
|
1807
|
+
|
|
1808
|
+
let content_digest = backup_content_digest(&rows)?;
|
|
1809
|
+
let meaning_digest = backup_meaning_digest(&rows)?;
|
|
1810
|
+
let header = BackupHeader {
|
|
1811
|
+
record_type: BACKUP_RECORD_TYPE.to_string(),
|
|
1812
|
+
backup_version: BACKUP_VERSION,
|
|
1813
|
+
format_version: DURABLE_FORMAT_VERSION,
|
|
1814
|
+
created_unix_ms: now_ms() as u64,
|
|
1815
|
+
row_count: rows.len(),
|
|
1816
|
+
observed_versions,
|
|
1817
|
+
content_digest: content_digest.clone(),
|
|
1818
|
+
meaning_digest: meaning_digest.clone(),
|
|
1819
|
+
};
|
|
1820
|
+
|
|
1821
|
+
if let Some(parent) = path.parent() {
|
|
1822
|
+
fs::create_dir_all(parent)?;
|
|
1823
|
+
}
|
|
1824
|
+
let mut file = OpenOptions::new()
|
|
1825
|
+
.create(true)
|
|
1826
|
+
.write(true)
|
|
1827
|
+
.truncate(true)
|
|
1828
|
+
.open(path)?;
|
|
1829
|
+
write_json_line(&mut file, &header)?;
|
|
1830
|
+
for row in &rows {
|
|
1831
|
+
write_json_line(&mut file, row)?;
|
|
1832
|
+
}
|
|
1833
|
+
file.flush()?;
|
|
1834
|
+
file.sync_all()?;
|
|
1835
|
+
|
|
1836
|
+
Ok(BackupVerification {
|
|
1837
|
+
backup_version: BACKUP_VERSION,
|
|
1838
|
+
format_version: DURABLE_FORMAT_VERSION,
|
|
1839
|
+
created_unix_ms: header.created_unix_ms,
|
|
1840
|
+
rows: rows.len(),
|
|
1841
|
+
content_digest,
|
|
1842
|
+
meaning_digest,
|
|
1843
|
+
})
|
|
1844
|
+
}
|
|
1845
|
+
|
|
1846
|
+
/// Restore a backup artifact into a **new** database.
|
|
1847
|
+
///
|
|
1848
|
+
/// Refuses to write over an existing database: a restore that could
|
|
1849
|
+
/// overwrite the thing an operator is trying to recover is not a recovery
|
|
1850
|
+
/// tool. The artifact is verified first, and the restored database is
|
|
1851
|
+
/// verified again afterwards — its meaning is recomputed from the live
|
|
1852
|
+
/// database and compared with the artifact's. A restore therefore proves
|
|
1853
|
+
/// that the result *means the same thing*, not merely that the file parsed.
|
|
1854
|
+
pub fn restore_backup(backup: &Path, target: &Path) -> Result<(Self, BackupVerification)> {
|
|
1855
|
+
let verification = verify_backup(backup)?;
|
|
1856
|
+
if target.exists() {
|
|
1857
|
+
return Err(backup_problem(
|
|
1858
|
+
"the restore target already exists",
|
|
1859
|
+
None,
|
|
1860
|
+
"Restore into a new path. FeltDB will not write over an existing database.",
|
|
1861
|
+
));
|
|
1862
|
+
}
|
|
1863
|
+
|
|
1864
|
+
let text = std::fs::read_to_string(backup)?;
|
|
1865
|
+
let mut lines = text.lines();
|
|
1866
|
+
let header: BackupHeader = serde_json::from_str(lines.next().unwrap_or_default())?;
|
|
1867
|
+
|
|
1868
|
+
if let Some(parent) = target.parent() {
|
|
1869
|
+
fs::create_dir_all(parent)?;
|
|
1870
|
+
}
|
|
1871
|
+
{
|
|
1872
|
+
let mut file = OpenOptions::new()
|
|
1873
|
+
.create_new(true)
|
|
1874
|
+
.write(true)
|
|
1875
|
+
.open(target)?;
|
|
1876
|
+
write_json_line(
|
|
1877
|
+
&mut file,
|
|
1878
|
+
&FormatRecord {
|
|
1879
|
+
record_type: FORMAT_RECORD_TYPE.to_string(),
|
|
1880
|
+
format_version: header.format_version,
|
|
1881
|
+
},
|
|
1882
|
+
)?;
|
|
1883
|
+
for line in lines {
|
|
1884
|
+
if line.trim().is_empty() {
|
|
1885
|
+
continue;
|
|
1886
|
+
}
|
|
1887
|
+
writeln!(file, "{line}")?;
|
|
1888
|
+
}
|
|
1889
|
+
file.flush()?;
|
|
1890
|
+
file.sync_all()?;
|
|
1891
|
+
}
|
|
1892
|
+
|
|
1893
|
+
let restored = FeltDb::open(target)?;
|
|
1894
|
+
|
|
1895
|
+
// The equivalence check. Anything that changed the database's meaning
|
|
1896
|
+
// between the artifact and the live result fails here.
|
|
1897
|
+
let live_rows = {
|
|
1898
|
+
let inner = restored.state();
|
|
1899
|
+
let mut rows: Vec<StoredRow> = inner
|
|
1900
|
+
.rows
|
|
1901
|
+
.values()
|
|
1902
|
+
.flat_map(|bucket| bucket.values().cloned())
|
|
1903
|
+
.filter(|row| !row.deleted)
|
|
1904
|
+
.collect();
|
|
1905
|
+
rows.sort_by(|left, right| {
|
|
1906
|
+
left.capability
|
|
1907
|
+
.cmp(&right.capability)
|
|
1908
|
+
.then_with(|| left.key.cmp(&right.key))
|
|
1909
|
+
});
|
|
1910
|
+
rows
|
|
1911
|
+
};
|
|
1912
|
+
let restored_meaning = backup_meaning_digest(&live_rows)?;
|
|
1913
|
+
if restored_meaning != verification.meaning_digest {
|
|
1914
|
+
return Err(backup_problem(
|
|
1915
|
+
"the restored database does not mean what the backup says it means",
|
|
1916
|
+
None,
|
|
1917
|
+
"The backup or the restore path is damaged. Do not use this database.",
|
|
1918
|
+
));
|
|
1919
|
+
}
|
|
1920
|
+
|
|
1921
|
+
Ok((restored, verification))
|
|
1922
|
+
}
|
|
1923
|
+
|
|
1924
|
+
/// A digest of this database's **application state**: the records an
|
|
1925
|
+
/// application reads, and nothing else.
|
|
1926
|
+
///
|
|
1927
|
+
/// Excludes revisions and retention state, so it answers only "do these two
|
|
1928
|
+
/// databases hold the same values?". Pair it with
|
|
1929
|
+
/// `StateStore::history_digest` to ask the separate and stronger question
|
|
1930
|
+
/// of whether they hold the same history — two databases can match on this
|
|
1931
|
+
/// and differ completely on that, which is exactly the failure replication
|
|
1932
|
+
/// used to have.
|
|
1933
|
+
pub fn state_digest(&self) -> Result<String> {
|
|
1934
|
+
let inner = self.state();
|
|
1935
|
+
let mut entries: Vec<(String, String, String)> = inner
|
|
1936
|
+
.rows
|
|
1937
|
+
.iter()
|
|
1938
|
+
.filter(|(capability, _)| is_versioned(capability))
|
|
1939
|
+
.flat_map(|(capability, bucket)| {
|
|
1940
|
+
bucket
|
|
1941
|
+
.values()
|
|
1942
|
+
.filter(|row| !row.deleted)
|
|
1943
|
+
.map(move |row| (capability.clone(), row.key.clone(), row.value.to_string()))
|
|
1944
|
+
})
|
|
1945
|
+
.collect();
|
|
1946
|
+
entries.sort();
|
|
1947
|
+
|
|
1948
|
+
let mut hasher = Sha256::new();
|
|
1949
|
+
hasher.update(b"feltdb.state.v1");
|
|
1950
|
+
for (capability, key, value) in entries {
|
|
1951
|
+
hasher.update(capability.as_bytes());
|
|
1952
|
+
hasher.update([0u8]);
|
|
1953
|
+
hasher.update(key.as_bytes());
|
|
1954
|
+
hasher.update([0u8]);
|
|
1955
|
+
hasher.update(value.as_bytes());
|
|
1956
|
+
hasher.update([0u8]);
|
|
1957
|
+
}
|
|
1958
|
+
Ok(format!("{:x}", hasher.finalize()))
|
|
1959
|
+
}
|
|
1960
|
+
|
|
1961
|
+
/// How far a single-record write is pushed before it returns.
|
|
1962
|
+
pub fn durability_mode(&self) -> DurabilityMode {
|
|
1963
|
+
self.state().durability_mode
|
|
1964
|
+
}
|
|
1965
|
+
|
|
1966
|
+
/// Choose what a successful single-record write guarantees.
|
|
1967
|
+
///
|
|
1968
|
+
/// This changes the database's contract, not merely its speed. Transaction
|
|
1969
|
+
/// commits always issue a stable-storage barrier and are unaffected.
|
|
1970
|
+
pub fn set_durability_mode(&self, mode: DurabilityMode) {
|
|
1971
|
+
let mut inner = self.state();
|
|
1972
|
+
inner.durability_mode = mode;
|
|
1973
|
+
inner.writes_since_barrier = 0;
|
|
1974
|
+
}
|
|
1975
|
+
|
|
1976
|
+
/// What this database knows about itself.
|
|
1977
|
+
///
|
|
1978
|
+
/// Reports facts the open established — the durable format it was accepted
|
|
1979
|
+
/// as, and whether replay discarded an incomplete final append. It reports
|
|
1980
|
+
/// nothing it has not observed: **health can expose a proven fact, it
|
|
1981
|
+
/// cannot create a stronger guarantee.**
|
|
1982
|
+
///
|
|
1983
|
+
/// In particular this says nothing about power-loss durability, which is
|
|
1984
|
+
/// unproven, nor about replication or backup state, which have no
|
|
1985
|
+
/// operational contract yet.
|
|
1986
|
+
pub fn health(&self) -> DatabaseHealth {
|
|
1987
|
+
let recovery = self.log_recovery();
|
|
1988
|
+
let storage = match &recovery {
|
|
1989
|
+
LogRecovery::Clean => StorageHealth::Clean,
|
|
1990
|
+
LogRecovery::RecoveredTornTail {
|
|
1991
|
+
byte_offset,
|
|
1992
|
+
discarded_bytes,
|
|
1993
|
+
} => StorageHealth::RecoveredIncompleteWrite {
|
|
1994
|
+
byte_offset: *byte_offset,
|
|
1995
|
+
discarded_bytes: *discarded_bytes,
|
|
1996
|
+
},
|
|
1997
|
+
};
|
|
1998
|
+
DatabaseHealth {
|
|
1999
|
+
durable_format: self.state().durable_format.clone(),
|
|
2000
|
+
recovery,
|
|
2001
|
+
storage,
|
|
2002
|
+
}
|
|
2003
|
+
}
|
|
2004
|
+
|
|
2005
|
+
/// What replaying the durable log had to do for this database to open.
|
|
2006
|
+
///
|
|
2007
|
+
/// [`LogRecovery::Clean`] means every durable byte was a complete, valid
|
|
2008
|
+
/// record. [`LogRecovery::RecoveredTornTail`] means the log ended in an
|
|
2009
|
+
/// incomplete append that was discarded — the database is usable and its
|
|
2010
|
+
/// **last write did not land**, which is a different fact from healthy and
|
|
2011
|
+
/// one an operator has to be able to learn.
|
|
2012
|
+
///
|
|
2013
|
+
/// A corrupt log does not reach here at all: it refuses the open.
|
|
2014
|
+
pub fn log_recovery(&self) -> LogRecovery {
|
|
2015
|
+
self.state().log_recovery.clone()
|
|
2016
|
+
}
|
|
2017
|
+
|
|
816
2018
|
pub fn insert<T: Serialize>(&self, key: &str, value: T) -> Result<()> {
|
|
817
2019
|
let capability = key
|
|
818
2020
|
.split_once(':')
|
|
@@ -929,7 +2131,9 @@ impl FeltDb {
|
|
|
929
2131
|
inner.sequence += 1;
|
|
930
2132
|
inner.sync_state.increment_vector_clock();
|
|
931
2133
|
let vector_clock = inner.sync_state.get_or_init_vector_clock().clone();
|
|
932
|
-
let
|
|
2134
|
+
let planned =
|
|
2135
|
+
is_versioned(&capability).then(|| plan_revision(&inner, key, value.to_string()));
|
|
2136
|
+
let mut operation = Operation::update(
|
|
933
2137
|
inner.sequence,
|
|
934
2138
|
inner.instance_id.clone(),
|
|
935
2139
|
inner.sequence,
|
|
@@ -939,6 +2143,7 @@ impl FeltDb {
|
|
|
939
2143
|
capability.clone(),
|
|
940
2144
|
)
|
|
941
2145
|
.with_vector_clock(vector_clock);
|
|
2146
|
+
operation.revision = planned.as_ref().map(provenance_of);
|
|
942
2147
|
let row = StoredRow {
|
|
943
2148
|
capability: capability.clone(),
|
|
944
2149
|
key: key.to_string(),
|
|
@@ -950,10 +2155,13 @@ impl FeltDb {
|
|
|
950
2155
|
deleted: false,
|
|
951
2156
|
operation: Some(operation.clone()),
|
|
952
2157
|
};
|
|
953
|
-
|
|
2158
|
+
append_event_locked(&mut inner, &row)?;
|
|
954
2159
|
inner.authority_revision += 1;
|
|
955
2160
|
inner.change_log.add_operation(operation);
|
|
956
2161
|
inner.put_row(row.clone());
|
|
2162
|
+
if let Some(revision) = planned {
|
|
2163
|
+
persist_revision_locked(&mut inner, revision)?;
|
|
2164
|
+
}
|
|
957
2165
|
let event = ChangeEvent {
|
|
958
2166
|
capability,
|
|
959
2167
|
key: key.to_string(),
|
|
@@ -1740,6 +2948,9 @@ impl FeltDb {
|
|
|
1740
2948
|
) -> Result<()> {
|
|
1741
2949
|
let rust_type = type_name::<T>().to_string();
|
|
1742
2950
|
let value_json = serde_json::to_value(value)?;
|
|
2951
|
+
// Canonical because `serde_json` is built without `preserve_order`, so
|
|
2952
|
+
// `Map` is a `BTreeMap` and `to_string` emits keys in sorted order.
|
|
2953
|
+
let canonical = value_json.to_string();
|
|
1743
2954
|
let mut row = StoredRow {
|
|
1744
2955
|
capability: capability.clone(),
|
|
1745
2956
|
key: key.clone(),
|
|
@@ -1764,8 +2975,13 @@ impl FeltDb {
|
|
|
1764
2975
|
// Get current vector clock to attach to operation
|
|
1765
2976
|
let vector_clock = inner.sync_state.get_or_init_vector_clock().clone();
|
|
1766
2977
|
|
|
2978
|
+
// The revision this write produces is worked out first, so the
|
|
2979
|
+
// operation can name it and a peer can rebuild the same one.
|
|
2980
|
+
let planned =
|
|
2981
|
+
is_versioned(&capability).then(|| plan_revision(&inner, &key, canonical.clone()));
|
|
2982
|
+
|
|
1767
2983
|
// Create an operation with vector clock for convergence
|
|
1768
|
-
let op = Operation::insert(
|
|
2984
|
+
let mut op = Operation::insert(
|
|
1769
2985
|
inner.sequence,
|
|
1770
2986
|
inner.instance_id.clone(),
|
|
1771
2987
|
inner.sequence,
|
|
@@ -1775,14 +2991,18 @@ impl FeltDb {
|
|
|
1775
2991
|
capability.clone(),
|
|
1776
2992
|
)
|
|
1777
2993
|
.with_vector_clock(vector_clock);
|
|
2994
|
+
op.revision = planned.as_ref().map(provenance_of);
|
|
1778
2995
|
|
|
1779
2996
|
// Add operation to change log for sync
|
|
1780
2997
|
row.operation = Some(op.clone());
|
|
1781
2998
|
inner.change_log.add_operation(op.clone());
|
|
1782
2999
|
|
|
1783
|
-
|
|
3000
|
+
append_event_locked(&mut inner, &row)?;
|
|
1784
3001
|
inner.authority_revision += 1;
|
|
1785
3002
|
inner.put_row(row);
|
|
3003
|
+
if let Some(revision) = planned {
|
|
3004
|
+
persist_revision_locked(&mut inner, revision)?;
|
|
3005
|
+
}
|
|
1786
3006
|
let suffix = key.rsplit_once(':').map(|(_, right)| right);
|
|
1787
3007
|
if let Some(id) = suffix.and_then(|s| s.parse::<u64>().ok()) {
|
|
1788
3008
|
let entry = inner.key_counters.entry(capability.clone()).or_default();
|
|
@@ -1809,6 +3029,9 @@ impl FeltDb {
|
|
|
1809
3029
|
) -> Result<()> {
|
|
1810
3030
|
let rust_type = type_name::<T>().to_string();
|
|
1811
3031
|
let value_json = serde_json::to_value(value)?;
|
|
3032
|
+
// Canonical because `serde_json` is built without `preserve_order`, so
|
|
3033
|
+
// `Map` is a `BTreeMap` and `to_string` emits keys in sorted order.
|
|
3034
|
+
let canonical = value_json.to_string();
|
|
1812
3035
|
let mut row = StoredRow {
|
|
1813
3036
|
capability: capability.clone(),
|
|
1814
3037
|
key: key.clone(),
|
|
@@ -1833,8 +3056,11 @@ impl FeltDb {
|
|
|
1833
3056
|
// Get current vector clock to attach to operation
|
|
1834
3057
|
let vector_clock = inner.sync_state.get_or_init_vector_clock().clone();
|
|
1835
3058
|
|
|
3059
|
+
let planned =
|
|
3060
|
+
is_versioned(&capability).then(|| plan_revision(&inner, &key, canonical.clone()));
|
|
3061
|
+
|
|
1836
3062
|
// Create an update operation with vector clock for convergence
|
|
1837
|
-
let op = Operation::update(
|
|
3063
|
+
let mut op = Operation::update(
|
|
1838
3064
|
inner.sequence,
|
|
1839
3065
|
inner.instance_id.clone(),
|
|
1840
3066
|
inner.sequence,
|
|
@@ -1844,14 +3070,18 @@ impl FeltDb {
|
|
|
1844
3070
|
capability.clone(),
|
|
1845
3071
|
)
|
|
1846
3072
|
.with_vector_clock(vector_clock);
|
|
3073
|
+
op.revision = planned.as_ref().map(provenance_of);
|
|
1847
3074
|
|
|
1848
3075
|
// Add operation to change log for sync
|
|
1849
3076
|
row.operation = Some(op.clone());
|
|
1850
3077
|
inner.change_log.add_operation(op.clone());
|
|
1851
3078
|
|
|
1852
|
-
|
|
3079
|
+
append_event_locked(&mut inner, &row)?;
|
|
1853
3080
|
inner.authority_revision += 1;
|
|
1854
3081
|
inner.put_row(row);
|
|
3082
|
+
if let Some(revision) = planned {
|
|
3083
|
+
persist_revision_locked(&mut inner, revision)?;
|
|
3084
|
+
}
|
|
1855
3085
|
}
|
|
1856
3086
|
|
|
1857
3087
|
let event = ChangeEvent {
|
|
@@ -2062,8 +3292,9 @@ impl FeltDb {
|
|
|
2062
3292
|
// The same predicate phase the indexed execution reports to,
|
|
2063
3293
|
// so the two executions' predicate cost is directly
|
|
2064
3294
|
// comparable rather than being two different measurements.
|
|
2065
|
-
let _predicate =
|
|
2066
|
-
workload_diagnostics::
|
|
3295
|
+
let _predicate = workload_diagnostics::span(
|
|
3296
|
+
workload_diagnostics::Phase::PredicateEvaluation,
|
|
3297
|
+
);
|
|
2067
3298
|
if predicate(row) {
|
|
2068
3299
|
drop(_predicate);
|
|
2069
3300
|
matched.push(row.clone());
|
|
@@ -2102,7 +3333,10 @@ impl FeltDb {
|
|
|
2102
3333
|
let acquired = std::time::Instant::now();
|
|
2103
3334
|
workload_diagnostics::record(
|
|
2104
3335
|
workload_diagnostics::Phase::StateLockWait,
|
|
2105
|
-
acquired
|
|
3336
|
+
acquired
|
|
3337
|
+
.duration_since(requested)
|
|
3338
|
+
.as_nanos()
|
|
3339
|
+
.min(u64::MAX as u128) as u64,
|
|
2106
3340
|
);
|
|
2107
3341
|
StateGuard {
|
|
2108
3342
|
inner,
|
|
@@ -2216,7 +3450,7 @@ impl FeltDb {
|
|
|
2216
3450
|
}
|
|
2217
3451
|
}
|
|
2218
3452
|
Ok(Err(
|
|
2219
|
-
"the live index holds entries no authoritative record derives".to_string()
|
|
3453
|
+
"the live index holds entries no authoritative record derives".to_string(),
|
|
2220
3454
|
))
|
|
2221
3455
|
}
|
|
2222
3456
|
|
|
@@ -2432,6 +3666,7 @@ impl FeltDb {
|
|
|
2432
3666
|
rows,
|
|
2433
3667
|
versions,
|
|
2434
3668
|
content_hash,
|
|
3669
|
+
format_version: DURABLE_FORMAT_VERSION,
|
|
2435
3670
|
})
|
|
2436
3671
|
}
|
|
2437
3672
|
/// Current materialized rows including their causal operation envelopes.
|
|
@@ -2470,6 +3705,24 @@ impl FeltDb {
|
|
|
2470
3705
|
/// Install a verified bootstrap snapshot into a pristine node. A node with
|
|
2471
3706
|
/// local history must converge through operations rather than overwrite it.
|
|
2472
3707
|
pub fn install_snapshot(&self, snapshot: DatabaseSnapshot) -> Result<()> {
|
|
3708
|
+
// A snapshot carries whole rows, revisions included, so installing one
|
|
3709
|
+
// from another format would reintroduce exactly the silent
|
|
3710
|
+
// misinterpretation the on-disk version prevents.
|
|
3711
|
+
if snapshot.format_version != DURABLE_FORMAT_VERSION {
|
|
3712
|
+
return Err(FlowError::IncompatibleFormat(Box::new(
|
|
3713
|
+
FormatIncompatibility {
|
|
3714
|
+
found: if snapshot.format_version == 0 {
|
|
3715
|
+
DurableFormat::Unversioned
|
|
3716
|
+
} else {
|
|
3717
|
+
DurableFormat::Versioned(snapshot.format_version)
|
|
3718
|
+
},
|
|
3719
|
+
required: DURABLE_FORMAT_VERSION,
|
|
3720
|
+
reason: "the snapshot was produced in a different durable format".to_string(),
|
|
3721
|
+
action: "Take a fresh snapshot from a FeltDB build matching this one."
|
|
3722
|
+
.to_string(),
|
|
3723
|
+
},
|
|
3724
|
+
)));
|
|
3725
|
+
}
|
|
2473
3726
|
snapshot.verify()?;
|
|
2474
3727
|
let mut inner = self.state();
|
|
2475
3728
|
if !inner.rows.is_empty() || !inner.change_log.pending.is_empty() {
|
|
@@ -2498,6 +3751,7 @@ impl FeltDb {
|
|
|
2498
3751
|
&mut file,
|
|
2499
3752
|
&LogHeader {
|
|
2500
3753
|
record_type: "feltdb.snapshot.v1".to_string(),
|
|
3754
|
+
format_version: DURABLE_FORMAT_VERSION,
|
|
2501
3755
|
local_sequence: inner.sequence,
|
|
2502
3756
|
observed_versions: snapshot.versions.clone(),
|
|
2503
3757
|
},
|
|
@@ -2568,12 +3822,56 @@ impl FeltDb {
|
|
|
2568
3822
|
/// Atomically replace acknowledged history with a state snapshot and any
|
|
2569
3823
|
/// operations still needed by at least one configured peer.
|
|
2570
3824
|
pub fn compact_operation_log(&self, active_peers: &[String]) -> Result<usize> {
|
|
3825
|
+
self.compact_internal(active_peers, true)
|
|
3826
|
+
.map(|outcome| outcome.pruned())
|
|
3827
|
+
}
|
|
3828
|
+
|
|
3829
|
+
/// Compact under the database's [`CompactionPolicy`].
|
|
3830
|
+
///
|
|
3831
|
+
/// Prunes acknowledged operations from memory whenever it can, and rewrites
|
|
3832
|
+
/// the durable log **only when enough has accumulated to justify the
|
|
3833
|
+
/// stall**. Use this on a timer; use
|
|
3834
|
+
/// [`compact_operation_log`](Self::compact_operation_log) when a caller is
|
|
3835
|
+
/// explicitly asking for a rewrite now.
|
|
3836
|
+
///
|
|
3837
|
+
/// The distinction is the whole point. Pruning is cheap and in memory.
|
|
3838
|
+
/// Rewriting serializes every row in the database and fsyncs it while
|
|
3839
|
+
/// holding the one lock that serializes every read and every write, so
|
|
3840
|
+
/// doing it on a short timer stops the world on that timer.
|
|
3841
|
+
pub fn maybe_compact_operation_log(
|
|
3842
|
+
&self,
|
|
3843
|
+
active_peers: &[String],
|
|
3844
|
+
) -> Result<CompactionOutcome> {
|
|
3845
|
+
self.compact_internal(active_peers, false)
|
|
3846
|
+
}
|
|
3847
|
+
|
|
3848
|
+
/// The policy deciding when a timer-driven compaction rewrites the log.
|
|
3849
|
+
pub fn compaction_policy(&self) -> CompactionPolicy {
|
|
3850
|
+
self.state().compaction_policy
|
|
3851
|
+
}
|
|
3852
|
+
|
|
3853
|
+
/// Set the policy deciding when a timer-driven compaction rewrites the log.
|
|
3854
|
+
pub fn set_compaction_policy(&self, policy: CompactionPolicy) {
|
|
3855
|
+
self.state().compaction_policy = policy;
|
|
3856
|
+
}
|
|
3857
|
+
|
|
3858
|
+
fn compact_internal(&self, active_peers: &[String], force: bool) -> Result<CompactionOutcome> {
|
|
2571
3859
|
let mut inner = self.state();
|
|
2572
3860
|
let removed = inner.change_log.prune_acknowledged(active_peers);
|
|
2573
3861
|
if removed == 0 {
|
|
2574
|
-
return Ok(
|
|
3862
|
+
return Ok(CompactionOutcome::Idle);
|
|
2575
3863
|
}
|
|
2576
3864
|
persist_sync_metadata(&inner)?;
|
|
3865
|
+
inner.pruned_since_rewrite += removed;
|
|
3866
|
+
|
|
3867
|
+
// The cheap half is done. The expensive half runs only when asked for
|
|
3868
|
+
// explicitly, or when the accumulated gain has earned it.
|
|
3869
|
+
if !force && inner.pruned_since_rewrite < inner.compaction_policy.rewrite_after_pruned {
|
|
3870
|
+
return Ok(CompactionOutcome::Deferred {
|
|
3871
|
+
pruned: removed,
|
|
3872
|
+
pending_since_rewrite: inner.pruned_since_rewrite,
|
|
3873
|
+
});
|
|
3874
|
+
}
|
|
2577
3875
|
|
|
2578
3876
|
let temporary = inner.path.with_extension("compact.tmp");
|
|
2579
3877
|
let mut file = OpenOptions::new()
|
|
@@ -2585,6 +3883,7 @@ impl FeltDb {
|
|
|
2585
3883
|
&mut file,
|
|
2586
3884
|
&LogHeader {
|
|
2587
3885
|
record_type: "feltdb.snapshot.v1".to_string(),
|
|
3886
|
+
format_version: DURABLE_FORMAT_VERSION,
|
|
2588
3887
|
local_sequence: inner.sequence,
|
|
2589
3888
|
observed_versions: inner.change_log.versions(),
|
|
2590
3889
|
},
|
|
@@ -2603,7 +3902,8 @@ impl FeltDb {
|
|
|
2603
3902
|
file.sync_all()?;
|
|
2604
3903
|
fs::rename(temporary, &inner.path)?;
|
|
2605
3904
|
inner.bootstrapped = true;
|
|
2606
|
-
|
|
3905
|
+
inner.pruned_since_rewrite = 0;
|
|
3906
|
+
Ok(CompactionOutcome::Rewritten { pruned: removed })
|
|
2607
3907
|
}
|
|
2608
3908
|
|
|
2609
3909
|
/// Register a peer in the distributed fabric
|
|
@@ -2786,6 +4086,22 @@ impl FeltDb {
|
|
|
2786
4086
|
*counter = (*counter).max(id + 1);
|
|
2787
4087
|
}
|
|
2788
4088
|
|
|
4089
|
+
// Reconstruct the historical fact the originating authority recorded.
|
|
4090
|
+
//
|
|
4091
|
+
// Not a local mint of the resulting state: the identity, parent and
|
|
4092
|
+
// sequence come from the operation, so the peer ends up holding the
|
|
4093
|
+
// *same* revision rather than an equivalent-looking one of its own.
|
|
4094
|
+
// Deliberately independent of which value won any last-writer race
|
|
4095
|
+
// above — the origin committed this revision whether or not its value
|
|
4096
|
+
// is what is current here, and dropping it would lose the divergence
|
|
4097
|
+
// that reconciliation needs to see.
|
|
4098
|
+
if let Some(provenance) = op.revision.clone() {
|
|
4099
|
+
if let Some(value) = op.value.clone() {
|
|
4100
|
+
let revision = reconstruct_revision(&provenance, &value)?;
|
|
4101
|
+
persist_revision_locked(&mut inner, revision)?;
|
|
4102
|
+
}
|
|
4103
|
+
}
|
|
4104
|
+
|
|
2789
4105
|
inner.change_log.add_operation(op);
|
|
2790
4106
|
inner.sync_state.increment_received(1);
|
|
2791
4107
|
let event = ChangeEvent {
|
|
@@ -3113,64 +4429,246 @@ where
|
|
|
3113
4429
|
///
|
|
3114
4430
|
/// Damage anywhere earlier is not an interrupted write. Recovery refuses it
|
|
3115
4431
|
/// rather than silently dropping committed history.
|
|
3116
|
-
|
|
4432
|
+
/// Examine a durable database's format **without opening or modifying it**.
|
|
4433
|
+
///
|
|
4434
|
+
/// This runs before any record is interpreted, and reads the file only. A
|
|
4435
|
+
/// database this reports as incompatible is left exactly as it was found.
|
|
4436
|
+
///
|
|
4437
|
+
/// The order matters: an explicit format record decides, and the records
|
|
4438
|
+
/// themselves are examined *only* when no format record exists — the
|
|
4439
|
+
/// unversioned era. Compatibility is never inferred from whether a
|
|
4440
|
+
/// deserialization happened to succeed.
|
|
4441
|
+
pub fn inspect_durable_format(path: &Path) -> Result<FormatCompatibility> {
|
|
4442
|
+
if !path.exists() {
|
|
4443
|
+
// A database that does not exist yet will be created in the current
|
|
4444
|
+
// format.
|
|
4445
|
+
return Ok(FormatCompatibility::Compatible(DurableFormat::Versioned(
|
|
4446
|
+
DURABLE_FORMAT_VERSION,
|
|
4447
|
+
)));
|
|
4448
|
+
}
|
|
4449
|
+
|
|
4450
|
+
// Read-only: inspecting a database it is about to reject must not touch it.
|
|
4451
|
+
let scan = scan_log(path)?;
|
|
4452
|
+
let lines: Vec<String> = scan.lines.into_iter().map(|line| line.text).collect();
|
|
4453
|
+
let mut declared: Option<u32> = None;
|
|
4454
|
+
let mut legacy_revision = false;
|
|
4455
|
+
|
|
4456
|
+
for line in &lines {
|
|
4457
|
+
if line.trim().is_empty() {
|
|
4458
|
+
continue;
|
|
4459
|
+
}
|
|
4460
|
+
let Ok(value) = serde_json::from_str::<Value>(line) else {
|
|
4461
|
+
continue;
|
|
4462
|
+
};
|
|
4463
|
+
match value.get("record_type").and_then(Value::as_str) {
|
|
4464
|
+
Some(FORMAT_RECORD_TYPE) => {
|
|
4465
|
+
if let Some(version) = value.get("format_version").and_then(Value::as_u64) {
|
|
4466
|
+
declared = Some(version as u32);
|
|
4467
|
+
}
|
|
4468
|
+
}
|
|
4469
|
+
Some("feltdb.snapshot.v1" | "flowdb.snapshot.v1") => {
|
|
4470
|
+
match value.get("format_version").and_then(Value::as_u64) {
|
|
4471
|
+
Some(version) if version > 0 => declared = Some(version as u32),
|
|
4472
|
+
_ => {}
|
|
4473
|
+
}
|
|
4474
|
+
}
|
|
4475
|
+
_ => {
|
|
4476
|
+
// The unversioned-era probe: a revision record that predates
|
|
4477
|
+
// resource-scoped identity. Deliberate and specific — it looks
|
|
4478
|
+
// for a named absence rather than trying to parse and shrugging
|
|
4479
|
+
// at failure.
|
|
4480
|
+
let is_revision = value
|
|
4481
|
+
.get("key")
|
|
4482
|
+
.and_then(Value::as_str)
|
|
4483
|
+
.is_some_and(|key| key.starts_with("state:revision:"));
|
|
4484
|
+
if is_revision
|
|
4485
|
+
&& value
|
|
4486
|
+
.get("value")
|
|
4487
|
+
.and_then(Value::as_object)
|
|
4488
|
+
.is_some_and(|revision| !revision.contains_key("resource"))
|
|
4489
|
+
{
|
|
4490
|
+
legacy_revision = true;
|
|
4491
|
+
}
|
|
4492
|
+
}
|
|
4493
|
+
}
|
|
4494
|
+
}
|
|
4495
|
+
|
|
4496
|
+
if let Some(version) = declared {
|
|
4497
|
+
return Ok(match version.cmp(&DURABLE_FORMAT_VERSION) {
|
|
4498
|
+
std::cmp::Ordering::Equal => {
|
|
4499
|
+
FormatCompatibility::Compatible(DurableFormat::Versioned(version))
|
|
4500
|
+
}
|
|
4501
|
+
std::cmp::Ordering::Greater => {
|
|
4502
|
+
FormatCompatibility::Incompatible(FormatIncompatibility {
|
|
4503
|
+
found: DurableFormat::Versioned(version),
|
|
4504
|
+
required: DURABLE_FORMAT_VERSION,
|
|
4505
|
+
reason: "the database was written by a newer FeltDB".to_string(),
|
|
4506
|
+
action: "Run a FeltDB build that understands this format, or restore \
|
|
4507
|
+
a backup taken in an older format."
|
|
4508
|
+
.to_string(),
|
|
4509
|
+
})
|
|
4510
|
+
}
|
|
4511
|
+
std::cmp::Ordering::Less => FormatCompatibility::Incompatible(FormatIncompatibility {
|
|
4512
|
+
found: DurableFormat::Versioned(version),
|
|
4513
|
+
required: DURABLE_FORMAT_VERSION,
|
|
4514
|
+
reason: "the database was written in an older durable format".to_string(),
|
|
4515
|
+
action: "Migrate the database with a FeltDB build that supports both \
|
|
4516
|
+
formats, or restore a backup."
|
|
4517
|
+
.to_string(),
|
|
4518
|
+
}),
|
|
4519
|
+
});
|
|
4520
|
+
}
|
|
4521
|
+
|
|
4522
|
+
if legacy_revision {
|
|
4523
|
+
return Ok(FormatCompatibility::Incompatible(FormatIncompatibility {
|
|
4524
|
+
found: DurableFormat::Unversioned,
|
|
4525
|
+
required: DURABLE_FORMAT_VERSION,
|
|
4526
|
+
reason: "it holds revision records written before revisions belonged to a \
|
|
4527
|
+
resource, and those records cannot be migrated without inventing \
|
|
4528
|
+
information they never contained: a revision recorded no resource, \
|
|
4529
|
+
so which history each one belongs to is not recoverable from the data"
|
|
4530
|
+
.to_string(),
|
|
4531
|
+
action: "Export the application state you need from a FeltDB build that \
|
|
4532
|
+
understands the old format, then load it into a new database. The \
|
|
4533
|
+
existing file has not been modified."
|
|
4534
|
+
.to_string(),
|
|
4535
|
+
}));
|
|
4536
|
+
}
|
|
4537
|
+
|
|
4538
|
+
// No format record and no legacy revision: nothing in this database depends
|
|
4539
|
+
// on the difference.
|
|
4540
|
+
Ok(FormatCompatibility::Compatible(DurableFormat::Unversioned))
|
|
4541
|
+
}
|
|
4542
|
+
|
|
4543
|
+
/// One complete record in the durable log, with where it came from.
|
|
4544
|
+
struct ScannedLine {
|
|
4545
|
+
text: String,
|
|
4546
|
+
line_number: usize,
|
|
4547
|
+
byte_offset: u64,
|
|
4548
|
+
}
|
|
4549
|
+
|
|
4550
|
+
/// The bytes at the end of a log that are not a complete record.
|
|
4551
|
+
struct TornTail {
|
|
4552
|
+
byte_offset: u64,
|
|
4553
|
+
bytes: usize,
|
|
4554
|
+
}
|
|
4555
|
+
|
|
4556
|
+
/// The durable log, read and classified. **This never writes.**
|
|
4557
|
+
struct LogScan {
|
|
4558
|
+
lines: Vec<ScannedLine>,
|
|
4559
|
+
torn_tail: Option<TornTail>,
|
|
4560
|
+
}
|
|
4561
|
+
|
|
4562
|
+
/// Read the durable log without modifying it.
|
|
4563
|
+
///
|
|
4564
|
+
/// The only thing separated out here is the **terminator**: a final record with
|
|
4565
|
+
/// no newline cannot be one this database finished writing, so it is reported
|
|
4566
|
+
/// as a torn tail. Everything else — including a complete final record that
|
|
4567
|
+
/// happens to be invalid — is returned as a line, to be validated by the
|
|
4568
|
+
/// caller and refused rather than quietly dropped for being near the end.
|
|
4569
|
+
///
|
|
4570
|
+
/// Being read-only is load-bearing twice over: a refused open must leave the
|
|
4571
|
+
/// database byte-identical, and format inspection must be able to run on a
|
|
4572
|
+
/// database it is about to reject.
|
|
4573
|
+
fn scan_log(path: &Path) -> Result<LogScan> {
|
|
3117
4574
|
let mut file = BufReader::new(OpenOptions::new().read(true).open(path)?);
|
|
3118
|
-
let mut lines
|
|
4575
|
+
let mut lines = Vec::new();
|
|
4576
|
+
let mut torn_tail = None;
|
|
3119
4577
|
let mut buffer = Vec::new();
|
|
4578
|
+
let mut offset = 0u64;
|
|
4579
|
+
let mut line_number = 0usize;
|
|
4580
|
+
|
|
3120
4581
|
loop {
|
|
3121
4582
|
buffer.clear();
|
|
3122
|
-
|
|
4583
|
+
let read = file.read_until(b'\n', &mut buffer)?;
|
|
4584
|
+
if read == 0 {
|
|
3123
4585
|
break;
|
|
3124
4586
|
}
|
|
3125
4587
|
let terminated = buffer.ends_with(b"\n");
|
|
3126
4588
|
let text = String::from_utf8_lossy(&buffer)
|
|
3127
4589
|
.trim_end_matches('\n')
|
|
3128
4590
|
.to_string();
|
|
3129
|
-
|
|
3130
|
-
|
|
3131
|
-
|
|
3132
|
-
|
|
3133
|
-
|
|
3134
|
-
|
|
3135
|
-
|
|
3136
|
-
|
|
3137
|
-
|
|
4591
|
+
line_number += 1;
|
|
4592
|
+
|
|
4593
|
+
if !terminated {
|
|
4594
|
+
// An unterminated final record: the write did not finish. Empty
|
|
4595
|
+
// trailing bytes are nothing at all and are simply ignored.
|
|
4596
|
+
if !text.trim().is_empty() {
|
|
4597
|
+
torn_tail = Some(TornTail {
|
|
4598
|
+
byte_offset: offset,
|
|
4599
|
+
bytes: read,
|
|
4600
|
+
});
|
|
4601
|
+
}
|
|
4602
|
+
break;
|
|
3138
4603
|
}
|
|
3139
|
-
}
|
|
3140
4604
|
|
|
3141
|
-
|
|
3142
|
-
|
|
3143
|
-
|
|
3144
|
-
|
|
3145
|
-
|
|
4605
|
+
lines.push(ScannedLine {
|
|
4606
|
+
text,
|
|
4607
|
+
line_number,
|
|
4608
|
+
byte_offset: offset,
|
|
4609
|
+
});
|
|
4610
|
+
offset += read as u64;
|
|
3146
4611
|
}
|
|
3147
4612
|
|
|
3148
|
-
Ok(lines
|
|
4613
|
+
Ok(LogScan { lines, torn_tail })
|
|
4614
|
+
}
|
|
4615
|
+
|
|
4616
|
+
/// Discard an incomplete final append, so the next append does not splice onto
|
|
4617
|
+
/// a partial record.
|
|
4618
|
+
///
|
|
4619
|
+
/// Called only after the whole log has replayed successfully. A log that is
|
|
4620
|
+
/// going to be refused is never truncated.
|
|
4621
|
+
fn truncate_torn_tail(path: &Path, tail: &TornTail) -> Result<()> {
|
|
4622
|
+
let file = OpenOptions::new().write(true).open(path)?;
|
|
4623
|
+
file.set_len(tail.byte_offset)?;
|
|
4624
|
+
file.sync_all()?;
|
|
4625
|
+
Ok(())
|
|
4626
|
+
}
|
|
4627
|
+
|
|
4628
|
+
fn corruption(line: &ScannedLine, reason: &str) -> FlowError {
|
|
4629
|
+
let excerpt: String = line.text.chars().take(120).collect();
|
|
4630
|
+
FlowError::CorruptLogLine(Box::new(LogCorruption {
|
|
4631
|
+
line_number: line.line_number,
|
|
4632
|
+
byte_offset: line.byte_offset,
|
|
4633
|
+
reason: reason.to_string(),
|
|
4634
|
+
excerpt,
|
|
4635
|
+
}))
|
|
3149
4636
|
}
|
|
3150
4637
|
|
|
3151
4638
|
fn replay_log(inner: &mut Inner) -> Result<()> {
|
|
3152
|
-
let
|
|
4639
|
+
let scan = scan_log(&inner.path)?;
|
|
4640
|
+
let torn_tail = scan.torn_tail;
|
|
4641
|
+
let lines = scan.lines;
|
|
3153
4642
|
|
|
3154
4643
|
let mut snapshot_log = false;
|
|
3155
|
-
|
|
3156
|
-
|
|
4644
|
+
let mut recoverable_revisions: Vec<(operation::RevisionProvenance, Value)> = Vec::new();
|
|
4645
|
+
for line in &lines {
|
|
4646
|
+
if line.text.trim().is_empty() {
|
|
3157
4647
|
continue;
|
|
3158
4648
|
}
|
|
3159
4649
|
let value: Value =
|
|
3160
|
-
serde_json::from_str(&line).map_err(|_|
|
|
4650
|
+
serde_json::from_str(&line.text).map_err(|_| corruption(line, "not valid JSON"))?;
|
|
3161
4651
|
if matches!(
|
|
3162
4652
|
value.get("record_type").and_then(Value::as_str),
|
|
3163
4653
|
Some("feltdb.snapshot.v1" | "flowdb.snapshot.v1")
|
|
3164
4654
|
) {
|
|
3165
|
-
let header: LogHeader = serde_json::from_value(value)
|
|
4655
|
+
let header: LogHeader = serde_json::from_value(value)
|
|
4656
|
+
.map_err(|error| corruption(line, &format!("malformed log header: {error}")))?;
|
|
3166
4657
|
inner.sequence = inner.sequence.max(header.local_sequence);
|
|
3167
4658
|
inner.change_log.observed_versions = header.observed_versions;
|
|
3168
4659
|
inner.bootstrapped = true;
|
|
3169
4660
|
snapshot_log = true;
|
|
3170
4661
|
continue;
|
|
3171
4662
|
}
|
|
4663
|
+
if value.get("record_type").and_then(Value::as_str) == Some(FORMAT_RECORD_TYPE) {
|
|
4664
|
+
// Already honoured by `inspect_durable_format` before the open.
|
|
4665
|
+
continue;
|
|
4666
|
+
}
|
|
3172
4667
|
if value.get("record_type").and_then(Value::as_str) == Some("feltdb.transaction.v1") {
|
|
3173
|
-
let transaction: TransactionLogRecord =
|
|
4668
|
+
let transaction: TransactionLogRecord =
|
|
4669
|
+
serde_json::from_value(value).map_err(|error| {
|
|
4670
|
+
corruption(line, &format!("malformed transaction record: {error}"))
|
|
4671
|
+
})?;
|
|
3174
4672
|
// Older records did not carry a commit revision. Their durable
|
|
3175
4673
|
// order in the log supplies an unambiguous upgrade path.
|
|
3176
4674
|
inner.authority_revision =
|
|
@@ -3217,9 +4715,16 @@ fn replay_log(inner: &mut Inner) -> Result<()> {
|
|
|
3217
4715
|
inner.sequence = inner.sequence.max(transaction.state_after);
|
|
3218
4716
|
continue;
|
|
3219
4717
|
}
|
|
3220
|
-
let row: StoredRow =
|
|
3221
|
-
|
|
4718
|
+
let row: StoredRow = serde_json::from_value(value)
|
|
4719
|
+
.map_err(|_| corruption(line, "not a durable record this build understands"))?;
|
|
3222
4720
|
if let Some(operation) = row.operation.clone() {
|
|
4721
|
+
// A revision is a deterministic function of the operation that
|
|
4722
|
+
// produced it, so it never has to be lost. If the crash landed
|
|
4723
|
+
// between a record and the revision recording it, the revision is
|
|
4724
|
+
// rebuilt below rather than left missing.
|
|
4725
|
+
if let (Some(provenance), Some(value)) = (&operation.revision, &operation.value) {
|
|
4726
|
+
recoverable_revisions.push((provenance.clone(), value.clone()));
|
|
4727
|
+
}
|
|
3223
4728
|
inner.sequence = inner.sequence.max(
|
|
3224
4729
|
(operation.instance_id == inner.instance_id)
|
|
3225
4730
|
.then_some(operation.sequence)
|
|
@@ -3253,6 +4758,61 @@ fn replay_log(inner: &mut Inner) -> Result<()> {
|
|
|
3253
4758
|
}
|
|
3254
4759
|
}
|
|
3255
4760
|
|
|
4761
|
+
// Rebuild any revision whose record survived but whose own row did not.
|
|
4762
|
+
//
|
|
4763
|
+
// This is the crash window between appending a record and appending the
|
|
4764
|
+
// revision that records it. The revision is derivable from the operation —
|
|
4765
|
+
// resource, content, parent, sequence and authority all travel with it —
|
|
4766
|
+
// so recovery reconstructs it instead of leaving an authority holding less
|
|
4767
|
+
// history than its own operation stream implies. Nothing is invented: the
|
|
4768
|
+
// identity is recomputed from its parts and discarded if it does not follow
|
|
4769
|
+
// from them.
|
|
4770
|
+
//
|
|
4771
|
+
// The rebuilt rows are not appended here. They are derived, and stay
|
|
4772
|
+
// derivable from the same operations on every subsequent open.
|
|
4773
|
+
for (provenance, value) in recoverable_revisions {
|
|
4774
|
+
let key = format!("state:revision:{}", provenance.id);
|
|
4775
|
+
let present = inner
|
|
4776
|
+
.rows
|
|
4777
|
+
.get("state")
|
|
4778
|
+
.and_then(|bucket| bucket.get(&key))
|
|
4779
|
+
.is_some_and(|row| !row.deleted);
|
|
4780
|
+
if present {
|
|
4781
|
+
continue;
|
|
4782
|
+
}
|
|
4783
|
+
|
|
4784
|
+
// A revision below the resource's retention horizon is absent because
|
|
4785
|
+
// it was expired on purpose. Rebuilding it would make recovery
|
|
4786
|
+
// resurrect history a policy deliberately discarded, which is a
|
|
4787
|
+
// different kind of wrong from losing it.
|
|
4788
|
+
let horizon = inner
|
|
4789
|
+
.rows
|
|
4790
|
+
.get("_retention")
|
|
4791
|
+
.and_then(|bucket| bucket.get(&format!("_retention:{}", provenance.resource)))
|
|
4792
|
+
.and_then(|row| serde_json::from_value::<RetentionRecord>(row.value.clone()).ok())
|
|
4793
|
+
.map(|state| state.horizon);
|
|
4794
|
+
if horizon.is_some_and(|horizon| provenance.sequence < horizon) {
|
|
4795
|
+
continue;
|
|
4796
|
+
}
|
|
4797
|
+
let Ok(revision) = reconstruct_revision(&provenance, &value) else {
|
|
4798
|
+
continue;
|
|
4799
|
+
};
|
|
4800
|
+
let Ok(encoded) = serde_json::to_value(&revision) else {
|
|
4801
|
+
continue;
|
|
4802
|
+
};
|
|
4803
|
+
inner.put_row(StoredRow {
|
|
4804
|
+
capability: "state".to_string(),
|
|
4805
|
+
key,
|
|
4806
|
+
rust_type: "feltdb::state_model::StateRevision".to_string(),
|
|
4807
|
+
value: encoded,
|
|
4808
|
+
unix_ms: now_ms(),
|
|
4809
|
+
content_hash: None,
|
|
4810
|
+
flow_ref: None,
|
|
4811
|
+
deleted: false,
|
|
4812
|
+
operation: None,
|
|
4813
|
+
});
|
|
4814
|
+
}
|
|
4815
|
+
|
|
3256
4816
|
// Compute collection cardinality from final row state (count non-deleted rows per capability)
|
|
3257
4817
|
for (capability, bucket) in &inner.rows {
|
|
3258
4818
|
let count = bucket.values().filter(|row| !row.deleted).count() as u64;
|
|
@@ -3261,6 +4821,17 @@ fn replay_log(inner: &mut Inner) -> Result<()> {
|
|
|
3261
4821
|
.insert(capability.clone(), count);
|
|
3262
4822
|
}
|
|
3263
4823
|
|
|
4824
|
+
// Only now, with every complete record replayed, is it safe to discard an
|
|
4825
|
+
// incomplete final append. A log that was going to be refused never reaches
|
|
4826
|
+
// this point, so a refused open leaves the file byte-identical.
|
|
4827
|
+
if let Some(tail) = torn_tail {
|
|
4828
|
+
truncate_torn_tail(&inner.path, &tail)?;
|
|
4829
|
+
inner.log_recovery = LogRecovery::RecoveredTornTail {
|
|
4830
|
+
byte_offset: tail.byte_offset,
|
|
4831
|
+
discarded_bytes: tail.bytes,
|
|
4832
|
+
};
|
|
4833
|
+
}
|
|
4834
|
+
|
|
3264
4835
|
Ok(())
|
|
3265
4836
|
}
|
|
3266
4837
|
|
|
@@ -3328,15 +4899,46 @@ fn persist_sync_metadata(inner: &Inner) -> Result<()> {
|
|
|
3328
4899
|
}
|
|
3329
4900
|
|
|
3330
4901
|
fn append_event(path: &Path, event: &StoredRow) -> Result<()> {
|
|
3331
|
-
|
|
3332
|
-
|
|
3333
|
-
|
|
4902
|
+
append_event_with(path, event, DurabilityMode::Flushed, &mut 0)
|
|
4903
|
+
}
|
|
4904
|
+
|
|
4905
|
+
/// Append one record under the database's configured durability mode.
|
|
4906
|
+
fn append_event_locked(inner: &mut Inner, event: &StoredRow) -> Result<()> {
|
|
4907
|
+
let path = inner.path.clone();
|
|
4908
|
+
let mode = inner.durability_mode;
|
|
4909
|
+
append_event_with(&path, event, mode, &mut inner.writes_since_barrier)
|
|
4910
|
+
}
|
|
4911
|
+
|
|
4912
|
+
/// Append one record, pushing it as far as the durability mode requires.
|
|
4913
|
+
///
|
|
4914
|
+
/// `since_barrier` counts writes since the last stable-storage barrier and is
|
|
4915
|
+
/// only read by [`DurabilityMode::Grouped`].
|
|
4916
|
+
fn append_event_with(
|
|
4917
|
+
path: &Path,
|
|
4918
|
+
event: &StoredRow,
|
|
4919
|
+
mode: DurabilityMode,
|
|
4920
|
+
since_barrier: &mut u32,
|
|
4921
|
+
) -> Result<()> {
|
|
3334
4922
|
let _span = workload_diagnostics::span(workload_diagnostics::Phase::Persistence);
|
|
3335
4923
|
let mut file = OpenOptions::new().create(true).append(true).open(path)?;
|
|
3336
4924
|
let mut line = serde_json::to_string(event)?;
|
|
3337
4925
|
line.push('\n');
|
|
3338
4926
|
file.write_all(line.as_bytes())?;
|
|
3339
4927
|
file.flush()?;
|
|
4928
|
+
|
|
4929
|
+
// The flush above is what makes a record survive the process dying. What
|
|
4930
|
+
// follows is the only thing that even asks the storage stack for more.
|
|
4931
|
+
match mode {
|
|
4932
|
+
DurabilityMode::Flushed => {}
|
|
4933
|
+
DurabilityMode::Synced => file.sync_all()?,
|
|
4934
|
+
DurabilityMode::Grouped { every } => {
|
|
4935
|
+
*since_barrier += 1;
|
|
4936
|
+
if *since_barrier >= every.max(1) {
|
|
4937
|
+
file.sync_all()?;
|
|
4938
|
+
*since_barrier = 0;
|
|
4939
|
+
}
|
|
4940
|
+
}
|
|
4941
|
+
}
|
|
3340
4942
|
Ok(())
|
|
3341
4943
|
}
|
|
3342
4944
|
|