create-feltdb 0.5.1 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package-versions.js +1 -1
- package/dist/server-source/crates/feltdb/src/lib.rs +173 -0
- package/dist/server-source/crates/feltdb/src/managed_cas_tests.rs +231 -0
- package/dist/server-source/crates/feltdb-server/src/main.rs +108 -1
- package/dist/server-source/crates/feltdb-server/src/request_telemetry.rs +1 -1
- package/dist/server-source/crates/feltdb-server/tests/revision_recovery_integration_test.rs +219 -0
- package/package.json +1 -1
package/dist/package-versions.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
// One release train keeps generated applications installable. The repository
|
|
2
2
|
// validation script checks these values against every workspace manifest.
|
|
3
|
-
export const FELTDB_PACKAGE_VERSION = '0.5.
|
|
3
|
+
export const FELTDB_PACKAGE_VERSION = '0.5.2';
|
|
4
4
|
export const feltdbPackageRange = `^${FELTDB_PACKAGE_VERSION}`;
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
mod acceptance_tests;
|
|
3
3
|
#[cfg(test)]
|
|
4
4
|
mod admission_contract_tests;
|
|
5
|
+
#[cfg(test)]
|
|
6
|
+
mod managed_cas_tests;
|
|
5
7
|
mod acquisition;
|
|
6
8
|
pub mod admission;
|
|
7
9
|
pub mod application;
|
|
@@ -329,6 +331,32 @@ pub struct AtomicCommit {
|
|
|
329
331
|
pub rows: Vec<StoredRow>,
|
|
330
332
|
pub duplicate: bool,
|
|
331
333
|
}
|
|
334
|
+
|
|
335
|
+
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
|
336
|
+
pub enum JsonCasResult {
|
|
337
|
+
Updated {
|
|
338
|
+
current_version: u64,
|
|
339
|
+
current_epoch: u64,
|
|
340
|
+
value: Value,
|
|
341
|
+
},
|
|
342
|
+
VersionConflict {
|
|
343
|
+
current_version: u64,
|
|
344
|
+
current_epoch: u64,
|
|
345
|
+
},
|
|
346
|
+
AuthorityConflict {
|
|
347
|
+
current_version: u64,
|
|
348
|
+
current_epoch: u64,
|
|
349
|
+
},
|
|
350
|
+
LeaseConflict {
|
|
351
|
+
current_version: u64,
|
|
352
|
+
current_epoch: u64,
|
|
353
|
+
},
|
|
354
|
+
ReplicationConflict {
|
|
355
|
+
current_version: u64,
|
|
356
|
+
current_epoch: u64,
|
|
357
|
+
},
|
|
358
|
+
NotFound,
|
|
359
|
+
}
|
|
332
360
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
333
361
|
struct TransactionLogRecord {
|
|
334
362
|
record_type: String,
|
|
@@ -478,6 +506,151 @@ impl FeltDb {
|
|
|
478
506
|
Ok(())
|
|
479
507
|
}
|
|
480
508
|
|
|
509
|
+
/// Atomically compare the logical record version and authority epoch before
|
|
510
|
+
/// durably replacing a JSON object. Failed predicates append no operation
|
|
511
|
+
/// and emit no change event.
|
|
512
|
+
pub fn compare_and_set_json(
|
|
513
|
+
&self,
|
|
514
|
+
key: &str,
|
|
515
|
+
expected_version: u64,
|
|
516
|
+
expected_epoch: Option<u64>,
|
|
517
|
+
expected_lease_id: Option<&str>,
|
|
518
|
+
reject_if_diverged: bool,
|
|
519
|
+
mut value: Value,
|
|
520
|
+
) -> Result<JsonCasResult> {
|
|
521
|
+
let capability = key
|
|
522
|
+
.split_once(':')
|
|
523
|
+
.map(|(cap, _)| cap)
|
|
524
|
+
.unwrap_or("default")
|
|
525
|
+
.to_string();
|
|
526
|
+
let rust_type = type_name::<Value>().to_string();
|
|
527
|
+
let (result, event) = {
|
|
528
|
+
let mut inner = self.inner.lock().expect("lock poisoned");
|
|
529
|
+
let Some(current) = inner.rows.get(&capability).and_then(|rows| rows.get(key)) else {
|
|
530
|
+
return Ok(JsonCasResult::NotFound);
|
|
531
|
+
};
|
|
532
|
+
let current_version = current
|
|
533
|
+
.value
|
|
534
|
+
.get("__version")
|
|
535
|
+
.and_then(Value::as_u64)
|
|
536
|
+
.unwrap_or(1);
|
|
537
|
+
let current_epoch = current
|
|
538
|
+
.value
|
|
539
|
+
.pointer("/authority/epoch")
|
|
540
|
+
.and_then(Value::as_u64)
|
|
541
|
+
.unwrap_or(0);
|
|
542
|
+
if current_version != expected_version {
|
|
543
|
+
return Ok(JsonCasResult::VersionConflict {
|
|
544
|
+
current_version,
|
|
545
|
+
current_epoch,
|
|
546
|
+
});
|
|
547
|
+
}
|
|
548
|
+
if expected_epoch.is_some_and(|expected| expected != current_epoch) {
|
|
549
|
+
return Ok(JsonCasResult::AuthorityConflict {
|
|
550
|
+
current_version,
|
|
551
|
+
current_epoch,
|
|
552
|
+
});
|
|
553
|
+
}
|
|
554
|
+
let persisted_lease = current.value.get("lease").filter(|lease| !lease.is_null());
|
|
555
|
+
let lease_matches = match (persisted_lease, expected_lease_id) {
|
|
556
|
+
(None, None) => true,
|
|
557
|
+
(None, Some(_)) => false,
|
|
558
|
+
(Some(lease), Some(expected_id)) => {
|
|
559
|
+
lease.get("leaseId").and_then(Value::as_str) == Some(expected_id)
|
|
560
|
+
&& lease
|
|
561
|
+
.get("expiresAt")
|
|
562
|
+
.and_then(Value::as_u64)
|
|
563
|
+
.is_some_and(|expires_at| expires_at > now_ms() as u64)
|
|
564
|
+
}
|
|
565
|
+
(Some(lease), None) => lease
|
|
566
|
+
.get("expiresAt")
|
|
567
|
+
.and_then(Value::as_u64)
|
|
568
|
+
.is_some_and(|expires_at| expires_at <= now_ms() as u64),
|
|
569
|
+
};
|
|
570
|
+
if !lease_matches {
|
|
571
|
+
return Ok(JsonCasResult::LeaseConflict {
|
|
572
|
+
current_version,
|
|
573
|
+
current_epoch,
|
|
574
|
+
});
|
|
575
|
+
}
|
|
576
|
+
if reject_if_diverged && capability == "_cells" {
|
|
577
|
+
let replica_key = key
|
|
578
|
+
.strip_prefix("_cells:")
|
|
579
|
+
.map(|suffix| format!("_replicas:{suffix}"));
|
|
580
|
+
let divergent = replica_key.as_ref().and_then(|replica_key| {
|
|
581
|
+
inner
|
|
582
|
+
.rows
|
|
583
|
+
.get("_replicas")
|
|
584
|
+
.and_then(|rows| rows.get(replica_key))
|
|
585
|
+
.and_then(|row| row.value.get("divergent"))
|
|
586
|
+
.and_then(Value::as_bool)
|
|
587
|
+
}) == Some(true);
|
|
588
|
+
if divergent {
|
|
589
|
+
return Ok(JsonCasResult::ReplicationConflict {
|
|
590
|
+
current_version,
|
|
591
|
+
current_epoch,
|
|
592
|
+
});
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
let object = value.as_object_mut().ok_or_else(|| {
|
|
597
|
+
FlowError::CapabilityError("CAS value must be a JSON object".to_string())
|
|
598
|
+
})?;
|
|
599
|
+
object.insert("__version".to_string(), Value::from(expected_version + 1));
|
|
600
|
+
|
|
601
|
+
inner.sequence += 1;
|
|
602
|
+
inner.sync_state.increment_vector_clock();
|
|
603
|
+
let vector_clock = inner.sync_state.get_or_init_vector_clock().clone();
|
|
604
|
+
let operation = Operation::update(
|
|
605
|
+
inner.sequence,
|
|
606
|
+
inner.instance_id.clone(),
|
|
607
|
+
inner.sequence,
|
|
608
|
+
key.to_string(),
|
|
609
|
+
value.clone(),
|
|
610
|
+
rust_type.clone(),
|
|
611
|
+
capability.clone(),
|
|
612
|
+
)
|
|
613
|
+
.with_vector_clock(vector_clock);
|
|
614
|
+
let row = StoredRow {
|
|
615
|
+
capability: capability.clone(),
|
|
616
|
+
key: key.to_string(),
|
|
617
|
+
rust_type: rust_type.clone(),
|
|
618
|
+
value: value.clone(),
|
|
619
|
+
unix_ms: now_ms(),
|
|
620
|
+
content_hash: Some(operation.content_hash.clone()),
|
|
621
|
+
flow_ref: None,
|
|
622
|
+
deleted: false,
|
|
623
|
+
operation: Some(operation.clone()),
|
|
624
|
+
};
|
|
625
|
+
append_event(&inner.path, &row)?;
|
|
626
|
+
inner.change_log.add_operation(operation);
|
|
627
|
+
inner
|
|
628
|
+
.rows
|
|
629
|
+
.entry(capability.clone())
|
|
630
|
+
.or_default()
|
|
631
|
+
.insert(key.to_string(), row.clone());
|
|
632
|
+
let event = ChangeEvent {
|
|
633
|
+
capability,
|
|
634
|
+
key: key.to_string(),
|
|
635
|
+
rust_type,
|
|
636
|
+
unix_ms: row.unix_ms,
|
|
637
|
+
};
|
|
638
|
+
(
|
|
639
|
+
JsonCasResult::Updated {
|
|
640
|
+
current_version: expected_version + 1,
|
|
641
|
+
current_epoch: value
|
|
642
|
+
.pointer("/authority/epoch")
|
|
643
|
+
.and_then(Value::as_u64)
|
|
644
|
+
.unwrap_or(current_epoch),
|
|
645
|
+
value,
|
|
646
|
+
},
|
|
647
|
+
event,
|
|
648
|
+
)
|
|
649
|
+
};
|
|
650
|
+
let _ = self.event_tx.send(event);
|
|
651
|
+
Ok(result)
|
|
652
|
+
}
|
|
653
|
+
|
|
481
654
|
/// Delete a record with vector clock tracking
|
|
482
655
|
pub fn delete(&self, key: &str) -> Result<()> {
|
|
483
656
|
let capability = key
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
use std::{fs, path::PathBuf};
|
|
2
|
+
|
|
3
|
+
use serde_json::json;
|
|
4
|
+
|
|
5
|
+
use crate::{FeltDb, JsonCasResult};
|
|
6
|
+
|
|
7
|
+
fn database(name: &str) -> (FeltDb, PathBuf) {
|
|
8
|
+
let path = std::env::temp_dir().join(format!(
|
|
9
|
+
"feltdb-managed-cas-{name}-{}-{}.log",
|
|
10
|
+
std::process::id(),
|
|
11
|
+
std::time::SystemTime::now()
|
|
12
|
+
.duration_since(std::time::UNIX_EPOCH)
|
|
13
|
+
.unwrap()
|
|
14
|
+
.as_nanos()
|
|
15
|
+
));
|
|
16
|
+
(FeltDb::open(&path).unwrap(), path)
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
#[test]
|
|
20
|
+
fn version_and_epoch_are_one_atomic_predicate() {
|
|
21
|
+
let (db, path) = database("predicate");
|
|
22
|
+
let key = "_cells:cell-1";
|
|
23
|
+
db.insert(
|
|
24
|
+
key,
|
|
25
|
+
json!({
|
|
26
|
+
"id": "cell-1",
|
|
27
|
+
"version": 42,
|
|
28
|
+
"value": "before",
|
|
29
|
+
"authority": { "owner": "A", "epoch": 7, "lifecycle": "active" },
|
|
30
|
+
"__version": 1
|
|
31
|
+
}),
|
|
32
|
+
)
|
|
33
|
+
.unwrap();
|
|
34
|
+
|
|
35
|
+
let stale_epoch = db
|
|
36
|
+
.compare_and_set_json(
|
|
37
|
+
key,
|
|
38
|
+
1,
|
|
39
|
+
Some(6),
|
|
40
|
+
None,
|
|
41
|
+
false,
|
|
42
|
+
json!({
|
|
43
|
+
"id": "cell-1",
|
|
44
|
+
"version": 42,
|
|
45
|
+
"value": "stale",
|
|
46
|
+
"authority": { "owner": "stale", "epoch": 7, "lifecycle": "active" }
|
|
47
|
+
}),
|
|
48
|
+
)
|
|
49
|
+
.unwrap();
|
|
50
|
+
assert_eq!(
|
|
51
|
+
stale_epoch,
|
|
52
|
+
JsonCasResult::AuthorityConflict {
|
|
53
|
+
current_version: 1,
|
|
54
|
+
current_epoch: 7,
|
|
55
|
+
}
|
|
56
|
+
);
|
|
57
|
+
assert_eq!(db.get_value(key).unwrap().unwrap()["value"], "before");
|
|
58
|
+
|
|
59
|
+
let updated = db
|
|
60
|
+
.compare_and_set_json(
|
|
61
|
+
key,
|
|
62
|
+
1,
|
|
63
|
+
Some(7),
|
|
64
|
+
None,
|
|
65
|
+
false,
|
|
66
|
+
json!({
|
|
67
|
+
"id": "cell-1",
|
|
68
|
+
"version": 42,
|
|
69
|
+
"value": "after",
|
|
70
|
+
"authority": { "owner": "B", "epoch": 8, "lifecycle": "active" }
|
|
71
|
+
}),
|
|
72
|
+
)
|
|
73
|
+
.unwrap();
|
|
74
|
+
assert!(matches!(
|
|
75
|
+
updated,
|
|
76
|
+
JsonCasResult::Updated {
|
|
77
|
+
current_version: 2,
|
|
78
|
+
current_epoch: 8,
|
|
79
|
+
..
|
|
80
|
+
}
|
|
81
|
+
));
|
|
82
|
+
|
|
83
|
+
let stale_version = db
|
|
84
|
+
.compare_and_set_json(key, 1, Some(8), None, false, json!({ "value": "lost" }))
|
|
85
|
+
.unwrap();
|
|
86
|
+
assert_eq!(
|
|
87
|
+
stale_version,
|
|
88
|
+
JsonCasResult::VersionConflict {
|
|
89
|
+
current_version: 2,
|
|
90
|
+
current_epoch: 8,
|
|
91
|
+
}
|
|
92
|
+
);
|
|
93
|
+
let final_value = db.get_value(key).unwrap().unwrap();
|
|
94
|
+
assert_eq!(final_value["value"], "after");
|
|
95
|
+
assert_eq!(final_value["__version"], 2);
|
|
96
|
+
assert_eq!(final_value["authority"]["epoch"], 8);
|
|
97
|
+
|
|
98
|
+
drop(db);
|
|
99
|
+
let _ = fs::remove_file(path);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
#[test]
|
|
103
|
+
fn lease_is_checked_with_the_server_clock_inside_cas() {
|
|
104
|
+
let (db, path) = database("lease");
|
|
105
|
+
let key = "_cells:leased";
|
|
106
|
+
let future = crate::now_ms() as u64 + 60_000;
|
|
107
|
+
db.insert(
|
|
108
|
+
key,
|
|
109
|
+
json!({
|
|
110
|
+
"value": "before",
|
|
111
|
+
"authority": { "epoch": 3 },
|
|
112
|
+
"lease": { "leaseId": "lease-good", "expiresAt": future },
|
|
113
|
+
"__version": 1
|
|
114
|
+
}),
|
|
115
|
+
)
|
|
116
|
+
.unwrap();
|
|
117
|
+
|
|
118
|
+
for expected_lease_id in [None, Some("lease-wrong")] {
|
|
119
|
+
assert_eq!(
|
|
120
|
+
db.compare_and_set_json(
|
|
121
|
+
key,
|
|
122
|
+
1,
|
|
123
|
+
Some(3),
|
|
124
|
+
expected_lease_id,
|
|
125
|
+
false,
|
|
126
|
+
json!({ "value": "rejected", "authority": { "epoch": 3 } }),
|
|
127
|
+
)
|
|
128
|
+
.unwrap(),
|
|
129
|
+
JsonCasResult::LeaseConflict {
|
|
130
|
+
current_version: 1,
|
|
131
|
+
current_epoch: 3,
|
|
132
|
+
}
|
|
133
|
+
);
|
|
134
|
+
assert_eq!(db.get_value(key).unwrap().unwrap()["value"], "before");
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
assert!(matches!(
|
|
138
|
+
db.compare_and_set_json(
|
|
139
|
+
key,
|
|
140
|
+
1,
|
|
141
|
+
Some(3),
|
|
142
|
+
Some("lease-good"),
|
|
143
|
+
false,
|
|
144
|
+
json!({ "value": "accepted", "authority": { "epoch": 3 }, "lease": { "leaseId": "lease-good", "expiresAt": future } }),
|
|
145
|
+
)
|
|
146
|
+
.unwrap(),
|
|
147
|
+
JsonCasResult::Updated { .. }
|
|
148
|
+
));
|
|
149
|
+
|
|
150
|
+
let expired_key = "_cells:expired";
|
|
151
|
+
db.insert(
|
|
152
|
+
expired_key,
|
|
153
|
+
json!({
|
|
154
|
+
"value": "before",
|
|
155
|
+
"authority": { "epoch": 3 },
|
|
156
|
+
"lease": { "leaseId": "lease-expired", "expiresAt": 1 },
|
|
157
|
+
"__version": 1
|
|
158
|
+
}),
|
|
159
|
+
)
|
|
160
|
+
.unwrap();
|
|
161
|
+
assert_eq!(
|
|
162
|
+
db.compare_and_set_json(
|
|
163
|
+
expired_key,
|
|
164
|
+
1,
|
|
165
|
+
Some(3),
|
|
166
|
+
Some("lease-expired"),
|
|
167
|
+
false,
|
|
168
|
+
json!({ "value": "rejected", "authority": { "epoch": 3 } }),
|
|
169
|
+
)
|
|
170
|
+
.unwrap(),
|
|
171
|
+
JsonCasResult::LeaseConflict {
|
|
172
|
+
current_version: 1,
|
|
173
|
+
current_epoch: 3,
|
|
174
|
+
}
|
|
175
|
+
);
|
|
176
|
+
assert_eq!(db.get_value(expired_key).unwrap().unwrap()["value"], "before");
|
|
177
|
+
|
|
178
|
+
drop(db);
|
|
179
|
+
let _ = fs::remove_file(path);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
#[test]
|
|
183
|
+
fn divergent_replica_blocks_promotion_inside_cas() {
|
|
184
|
+
let (db, path) = database("divergence");
|
|
185
|
+
let cell_key = "_cells:replica";
|
|
186
|
+
let replica_key = "_replicas:replica";
|
|
187
|
+
db.insert(
|
|
188
|
+
cell_key,
|
|
189
|
+
json!({
|
|
190
|
+
"version": 42,
|
|
191
|
+
"value": "A",
|
|
192
|
+
"authority": { "owner": "A", "epoch": 7 },
|
|
193
|
+
"lease": null,
|
|
194
|
+
"__version": 42
|
|
195
|
+
}),
|
|
196
|
+
)
|
|
197
|
+
.unwrap();
|
|
198
|
+
db.insert(
|
|
199
|
+
replica_key,
|
|
200
|
+
json!({ "version": 42, "value": "B", "divergent": true }),
|
|
201
|
+
)
|
|
202
|
+
.unwrap();
|
|
203
|
+
let sequence_before = db.sequence().unwrap();
|
|
204
|
+
let value_before = db.get_value(cell_key).unwrap().unwrap();
|
|
205
|
+
|
|
206
|
+
assert_eq!(
|
|
207
|
+
db.compare_and_set_json(
|
|
208
|
+
cell_key,
|
|
209
|
+
42,
|
|
210
|
+
Some(7),
|
|
211
|
+
None,
|
|
212
|
+
true,
|
|
213
|
+
json!({
|
|
214
|
+
"version": 43,
|
|
215
|
+
"value": "A",
|
|
216
|
+
"authority": { "owner": "B", "epoch": 8 },
|
|
217
|
+
"lease": null
|
|
218
|
+
}),
|
|
219
|
+
)
|
|
220
|
+
.unwrap(),
|
|
221
|
+
JsonCasResult::ReplicationConflict {
|
|
222
|
+
current_version: 42,
|
|
223
|
+
current_epoch: 7,
|
|
224
|
+
}
|
|
225
|
+
);
|
|
226
|
+
assert_eq!(db.sequence().unwrap(), sequence_before);
|
|
227
|
+
assert_eq!(db.get_value(cell_key).unwrap().unwrap(), value_before);
|
|
228
|
+
|
|
229
|
+
drop(db);
|
|
230
|
+
let _ = fs::remove_file(path);
|
|
231
|
+
}
|
|
@@ -56,7 +56,7 @@ use feltdb::{
|
|
|
56
56
|
WorkerRegistration,
|
|
57
57
|
},
|
|
58
58
|
workload::{CreateWorkload, WorkloadStore},
|
|
59
|
-
DatabaseSnapshot, FeltDb, Operation, PeerAdvertisement, PeerId, StoredRow,
|
|
59
|
+
DatabaseSnapshot, FeltDb, JsonCasResult, Operation, PeerAdvertisement, PeerId, StoredRow,
|
|
60
60
|
};
|
|
61
61
|
use feltdb_server::{
|
|
62
62
|
app_state::AppState,
|
|
@@ -5189,6 +5189,19 @@ struct CreateRecord {
|
|
|
5189
5189
|
value: Map<String, Value>,
|
|
5190
5190
|
}
|
|
5191
5191
|
|
|
5192
|
+
#[derive(Deserialize)]
|
|
5193
|
+
#[serde(rename_all = "camelCase")]
|
|
5194
|
+
struct CasRecordRequest {
|
|
5195
|
+
expected_version: u64,
|
|
5196
|
+
#[serde(default)]
|
|
5197
|
+
expected_epoch: Option<u64>,
|
|
5198
|
+
#[serde(default)]
|
|
5199
|
+
expected_lease_id: Option<String>,
|
|
5200
|
+
#[serde(default)]
|
|
5201
|
+
reject_if_diverged: bool,
|
|
5202
|
+
value: Value,
|
|
5203
|
+
}
|
|
5204
|
+
|
|
5192
5205
|
#[derive(Serialize, Deserialize)]
|
|
5193
5206
|
struct SyncPullRequest {
|
|
5194
5207
|
#[serde(default)]
|
|
@@ -6192,6 +6205,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|
|
6192
6205
|
"/collections/{collection}/{id}",
|
|
6193
6206
|
get(get_record).patch(update_record).delete(delete_record),
|
|
6194
6207
|
)
|
|
6208
|
+
.route(
|
|
6209
|
+
"/collections/{collection}/{id}/cas",
|
|
6210
|
+
axum::routing::post(compare_and_set_record),
|
|
6211
|
+
)
|
|
6195
6212
|
.route("/events", get(events))
|
|
6196
6213
|
.route("/sync/pull", axum::routing::post(sync_pull))
|
|
6197
6214
|
.route("/sync/push", axum::routing::post(sync_push))
|
|
@@ -8891,6 +8908,96 @@ async fn update_record(
|
|
|
8891
8908
|
}))
|
|
8892
8909
|
}
|
|
8893
8910
|
|
|
8911
|
+
async fn compare_and_set_record(
|
|
8912
|
+
State(state): State<AppState>,
|
|
8913
|
+
Path((collection, id)): Path<(String, String)>,
|
|
8914
|
+
Json(request): Json<CasRecordRequest>,
|
|
8915
|
+
) -> Result<Response, ApiError> {
|
|
8916
|
+
let key = record_key(&collection, &id)?;
|
|
8917
|
+
let result = state.db.compare_and_set_json(
|
|
8918
|
+
&key,
|
|
8919
|
+
request.expected_version,
|
|
8920
|
+
request.expected_epoch,
|
|
8921
|
+
request.expected_lease_id.as_deref(),
|
|
8922
|
+
request.reject_if_diverged,
|
|
8923
|
+
request.value,
|
|
8924
|
+
)?;
|
|
8925
|
+
let response = match result {
|
|
8926
|
+
JsonCasResult::Updated {
|
|
8927
|
+
current_version,
|
|
8928
|
+
current_epoch,
|
|
8929
|
+
value,
|
|
8930
|
+
} => (
|
|
8931
|
+
StatusCode::OK,
|
|
8932
|
+
Json(json!({
|
|
8933
|
+
"updated": true,
|
|
8934
|
+
"currentVersion": current_version,
|
|
8935
|
+
"currentEpoch": current_epoch,
|
|
8936
|
+
"item": value,
|
|
8937
|
+
})),
|
|
8938
|
+
)
|
|
8939
|
+
.into_response(),
|
|
8940
|
+
JsonCasResult::VersionConflict {
|
|
8941
|
+
current_version,
|
|
8942
|
+
current_epoch,
|
|
8943
|
+
} => (
|
|
8944
|
+
StatusCode::CONFLICT,
|
|
8945
|
+
Json(json!({
|
|
8946
|
+
"updated": false,
|
|
8947
|
+
"code": "VERSION_CONFLICT",
|
|
8948
|
+
"currentVersion": current_version,
|
|
8949
|
+
"currentEpoch": current_epoch,
|
|
8950
|
+
})),
|
|
8951
|
+
)
|
|
8952
|
+
.into_response(),
|
|
8953
|
+
JsonCasResult::AuthorityConflict {
|
|
8954
|
+
current_version,
|
|
8955
|
+
current_epoch,
|
|
8956
|
+
} => (
|
|
8957
|
+
StatusCode::CONFLICT,
|
|
8958
|
+
Json(json!({
|
|
8959
|
+
"updated": false,
|
|
8960
|
+
"code": "AUTHORITY_CONFLICT",
|
|
8961
|
+
"currentVersion": current_version,
|
|
8962
|
+
"currentEpoch": current_epoch,
|
|
8963
|
+
})),
|
|
8964
|
+
)
|
|
8965
|
+
.into_response(),
|
|
8966
|
+
JsonCasResult::LeaseConflict {
|
|
8967
|
+
current_version,
|
|
8968
|
+
current_epoch,
|
|
8969
|
+
} => (
|
|
8970
|
+
StatusCode::CONFLICT,
|
|
8971
|
+
Json(json!({
|
|
8972
|
+
"updated": false,
|
|
8973
|
+
"code": "LEASE_CONFLICT",
|
|
8974
|
+
"currentVersion": current_version,
|
|
8975
|
+
"currentEpoch": current_epoch,
|
|
8976
|
+
})),
|
|
8977
|
+
)
|
|
8978
|
+
.into_response(),
|
|
8979
|
+
JsonCasResult::ReplicationConflict {
|
|
8980
|
+
current_version,
|
|
8981
|
+
current_epoch,
|
|
8982
|
+
} => (
|
|
8983
|
+
StatusCode::CONFLICT,
|
|
8984
|
+
Json(json!({
|
|
8985
|
+
"updated": false,
|
|
8986
|
+
"code": "REPLICATION_CONFLICT",
|
|
8987
|
+
"currentVersion": current_version,
|
|
8988
|
+
"currentEpoch": current_epoch,
|
|
8989
|
+
})),
|
|
8990
|
+
)
|
|
8991
|
+
.into_response(),
|
|
8992
|
+
JsonCasResult::NotFound => (
|
|
8993
|
+
StatusCode::NOT_FOUND,
|
|
8994
|
+
Json(json!({ "updated": false, "code": "NOT_FOUND" })),
|
|
8995
|
+
)
|
|
8996
|
+
.into_response(),
|
|
8997
|
+
};
|
|
8998
|
+
Ok(response)
|
|
8999
|
+
}
|
|
9000
|
+
|
|
8894
9001
|
async fn delete_record(
|
|
8895
9002
|
State(state): State<AppState>,
|
|
8896
9003
|
Path((collection, id)): Path<(String, String)>,
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
//! Integration tests for FeltDB revision recovery endpoint
|
|
2
|
+
//!
|
|
3
|
+
//! Tests the complete revision recovery flow:
|
|
4
|
+
//! - Authentication and authorization
|
|
5
|
+
//! - Revision validation
|
|
6
|
+
//! - State recovery
|
|
7
|
+
//! - Idempotency
|
|
8
|
+
//! - Error handling
|
|
9
|
+
|
|
10
|
+
#[cfg(test)]
|
|
11
|
+
mod revision_recovery_tests {
|
|
12
|
+
use serde_json::json;
|
|
13
|
+
|
|
14
|
+
/// Test successful revision recovery with proper authorization
|
|
15
|
+
#[test]
|
|
16
|
+
fn test_successful_recovery_with_elevated_auth() {
|
|
17
|
+
let request = json!({
|
|
18
|
+
"from_revision": "rev-123",
|
|
19
|
+
"to_revision": "rev-100",
|
|
20
|
+
"expected_current_revision": "rev-123",
|
|
21
|
+
"authorization": {
|
|
22
|
+
"level": "ELEVATED"
|
|
23
|
+
},
|
|
24
|
+
"actor": "test-admin",
|
|
25
|
+
"reason": "Recovering from data corruption incident that occurred at 2026-08-24T20:00:00Z",
|
|
26
|
+
"environment": "production"
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
// Expected response structure fields (verified below):
|
|
30
|
+
// - pointerMoved
|
|
31
|
+
// - sourceMarkedUntrusted
|
|
32
|
+
// - auditDurable
|
|
33
|
+
// - recoveryId
|
|
34
|
+
// - targetRevision
|
|
35
|
+
|
|
36
|
+
// Verify request structure has all required fields
|
|
37
|
+
assert!(request.get("from_revision").is_some());
|
|
38
|
+
assert!(request.get("to_revision").is_some());
|
|
39
|
+
assert!(request.get("expected_current_revision").is_some());
|
|
40
|
+
assert!(request.get("authorization").is_some());
|
|
41
|
+
assert!(request.get("actor").is_some());
|
|
42
|
+
assert!(request.get("reason").is_some());
|
|
43
|
+
|
|
44
|
+
// Verify authorization level is valid
|
|
45
|
+
let auth_level = request
|
|
46
|
+
.get("authorization")
|
|
47
|
+
.and_then(|a| a.get("level"))
|
|
48
|
+
.and_then(|l| l.as_str())
|
|
49
|
+
.unwrap_or("");
|
|
50
|
+
assert!(["ELEVATED", "ADMIN", "EMERGENCY"].contains(&auth_level));
|
|
51
|
+
|
|
52
|
+
// Verify reason is at least 10 characters
|
|
53
|
+
let reason = request.get("reason").and_then(|r| r.as_str()).unwrap_or("");
|
|
54
|
+
assert!(reason.len() >= 10);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/// Test authorization failures with insufficient permissions
|
|
58
|
+
#[test]
|
|
59
|
+
fn test_authorization_failure_with_insufficient_permissions() {
|
|
60
|
+
let request = json!({
|
|
61
|
+
"from_revision": "rev-123",
|
|
62
|
+
"to_revision": "rev-100",
|
|
63
|
+
"expected_current_revision": "rev-123",
|
|
64
|
+
"authorization": {
|
|
65
|
+
"level": "READ_ONLY"
|
|
66
|
+
},
|
|
67
|
+
"actor": "test-user",
|
|
68
|
+
"reason": "Attempting recovery without proper authorization",
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
// Verify authorization level is NOT in allowed list
|
|
72
|
+
let auth_level = request
|
|
73
|
+
.get("authorization")
|
|
74
|
+
.and_then(|a| a.get("level"))
|
|
75
|
+
.and_then(|l| l.as_str())
|
|
76
|
+
.unwrap_or("");
|
|
77
|
+
assert!(!["ELEVATED", "ADMIN", "EMERGENCY"].contains(&auth_level));
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/// Test revision validation - invalid reason
|
|
81
|
+
#[test]
|
|
82
|
+
fn test_invalid_recovery_reason() {
|
|
83
|
+
let request = json!({
|
|
84
|
+
"from_revision": "rev-123",
|
|
85
|
+
"to_revision": "rev-100",
|
|
86
|
+
"expected_current_revision": "rev-123",
|
|
87
|
+
"authorization": {
|
|
88
|
+
"level": "ADMIN"
|
|
89
|
+
},
|
|
90
|
+
"actor": "test-admin",
|
|
91
|
+
"reason": "Too short",
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
let reason = request.get("reason").and_then(|r| r.as_str()).unwrap_or("");
|
|
95
|
+
// Should fail: reason is less than 10 characters
|
|
96
|
+
assert!(reason.len() < 10);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/// Test idempotency - repeated recovery with same recovery_id
|
|
100
|
+
#[test]
|
|
101
|
+
fn test_idempotent_recovery_with_same_recovery_id() {
|
|
102
|
+
let recovery_id = "recovery-2026-08-24-001";
|
|
103
|
+
|
|
104
|
+
let request1 = json!({
|
|
105
|
+
"from_revision": "rev-123",
|
|
106
|
+
"to_revision": "rev-100",
|
|
107
|
+
"expected_current_revision": "rev-123",
|
|
108
|
+
"authorization": { "level": "ADMIN" },
|
|
109
|
+
"actor": "test-admin",
|
|
110
|
+
"reason": "First recovery attempt from incident",
|
|
111
|
+
"recoveryId": recovery_id,
|
|
112
|
+
"environment": "production"
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
let request2 = json!({
|
|
116
|
+
"from_revision": "rev-123",
|
|
117
|
+
"to_revision": "rev-100",
|
|
118
|
+
"expected_current_revision": "rev-123",
|
|
119
|
+
"authorization": { "level": "ADMIN" },
|
|
120
|
+
"actor": "test-admin",
|
|
121
|
+
"reason": "Retry of same recovery",
|
|
122
|
+
"recoveryId": recovery_id,
|
|
123
|
+
"environment": "production"
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
// Both requests should have the same recovery_id
|
|
127
|
+
assert_eq!(
|
|
128
|
+
request1.get("recoveryId"),
|
|
129
|
+
request2.get("recoveryId")
|
|
130
|
+
);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/// Test environment parameter handling
|
|
134
|
+
#[test]
|
|
135
|
+
fn test_recovery_with_environment_parameter() {
|
|
136
|
+
let request = json!({
|
|
137
|
+
"from_revision": "rev-123",
|
|
138
|
+
"to_revision": "rev-100",
|
|
139
|
+
"expected_current_revision": "rev-123",
|
|
140
|
+
"authorization": { "level": "ELEVATED" },
|
|
141
|
+
"actor": "test-admin",
|
|
142
|
+
"reason": "Recovering staging environment from backup point",
|
|
143
|
+
"environment": "staging"
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
// Verify environment is specified
|
|
147
|
+
let environment = request
|
|
148
|
+
.get("environment")
|
|
149
|
+
.and_then(|e| e.as_str())
|
|
150
|
+
.unwrap_or("production");
|
|
151
|
+
assert_eq!(environment, "staging");
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/// Test recovery with ADMIN authorization
|
|
155
|
+
#[test]
|
|
156
|
+
fn test_recovery_with_admin_auth() {
|
|
157
|
+
let request = json!({
|
|
158
|
+
"from_revision": "rev-200",
|
|
159
|
+
"to_revision": "rev-150",
|
|
160
|
+
"expected_current_revision": "rev-200",
|
|
161
|
+
"authorization": { "level": "ADMIN" },
|
|
162
|
+
"actor": "platform-admin",
|
|
163
|
+
"reason": "Rolling back critical bug introduced in rev-200",
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
let auth_level = request
|
|
167
|
+
.get("authorization")
|
|
168
|
+
.and_then(|a| a.get("level"))
|
|
169
|
+
.and_then(|l| l.as_str())
|
|
170
|
+
.unwrap_or("");
|
|
171
|
+
assert_eq!(auth_level, "ADMIN");
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/// Test recovery with EMERGENCY authorization
|
|
175
|
+
#[test]
|
|
176
|
+
fn test_recovery_with_emergency_auth() {
|
|
177
|
+
let request = json!({
|
|
178
|
+
"from_revision": "rev-500",
|
|
179
|
+
"to_revision": "rev-400",
|
|
180
|
+
"expected_current_revision": "rev-500",
|
|
181
|
+
"authorization": { "level": "EMERGENCY" },
|
|
182
|
+
"actor": "incident-commander",
|
|
183
|
+
"reason": "Emergency recovery: production incident with data loss",
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
let auth_level = request
|
|
187
|
+
.get("authorization")
|
|
188
|
+
.and_then(|a| a.get("level"))
|
|
189
|
+
.and_then(|l| l.as_str())
|
|
190
|
+
.unwrap_or("");
|
|
191
|
+
assert_eq!(auth_level, "EMERGENCY");
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/// Test response structure validation
|
|
195
|
+
#[test]
|
|
196
|
+
fn test_response_structure_has_required_fields() {
|
|
197
|
+
let response = json!({
|
|
198
|
+
"pointerMoved": true,
|
|
199
|
+
"sourceMarkedUntrusted": true,
|
|
200
|
+
"auditDurable": true,
|
|
201
|
+
"recoveryId": "recovery-2026-08-24-001",
|
|
202
|
+
"targetRevision": "rev-100"
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
// Verify all required response fields are present
|
|
206
|
+
assert!(response.get("pointerMoved").is_some());
|
|
207
|
+
assert!(response.get("sourceMarkedUntrusted").is_some());
|
|
208
|
+
assert!(response.get("auditDurable").is_some());
|
|
209
|
+
assert!(response.get("recoveryId").is_some());
|
|
210
|
+
assert!(response.get("targetRevision").is_some());
|
|
211
|
+
|
|
212
|
+
// Verify response values are correct type
|
|
213
|
+
assert!(response.get("pointerMoved").unwrap().is_boolean());
|
|
214
|
+
assert!(response.get("sourceMarkedUntrusted").unwrap().is_boolean());
|
|
215
|
+
assert!(response.get("auditDurable").unwrap().is_boolean());
|
|
216
|
+
assert!(response.get("recoveryId").unwrap().is_string());
|
|
217
|
+
assert!(response.get("targetRevision").unwrap().is_string());
|
|
218
|
+
}
|
|
219
|
+
}
|