journal_machines 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,518 @@
1
+ #![no_std]
2
+ #![cfg_attr(not(test), deny(clippy::indexing_slicing))]
3
+ #![doc = include_str!("../README.md")]
4
+
5
+ mod checked;
6
+ mod pins;
7
+ mod policy;
8
+ mod refusal;
9
+ mod replace;
10
+ mod segment;
11
+
12
+ pub use pins::{Advanced, Checkpointed, Pin, PinKind};
13
+ pub use policy::{Policy, RollReason};
14
+ pub use refusal::Refusal;
15
+ pub use replace::{Confirmed, Durable, Reclaimed, Replacement};
16
+ pub use segment::{Retention, Roll, Rolled, Segment, SegmentState};
17
+
18
+ use core::time::Duration;
19
+
20
+ use periscope_machines::{Issuer, scope_of};
21
+ use pins::PinSlot;
22
+
23
+ /// Journal-issued segment identity, ascending in offset order.
24
+ ///
25
+ /// It carries the scope of the journal that issued it above a sequence
26
+ /// number. The journal draws segment ids, pin generations and attempt numbers
27
+ /// from one sequence, so ids ascend but are not consecutive, and no pin
28
+ /// generation or attempt of the same journal has the same raw value. A
29
+ /// journal of another scope knows no segment by it.
30
+ ///
31
+ /// It has no order: two ids of different journals compare by raw value
32
+ /// without meaning anything, so ids are compared for equality only.
33
+ ///
34
+ /// ```compile_fail
35
+ /// use journal_machines::SegmentId;
36
+ /// let _ = SegmentId::from_raw(1) < SegmentId::from_raw(2);
37
+ /// ```
38
+ #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
39
+ pub struct SegmentId(u64);
40
+
41
+ impl SegmentId {
42
+ /// Adopt an identity read back from storage or another process.
43
+ #[must_use]
44
+ pub const fn from_raw(raw: u64) -> Self {
45
+ Self(raw)
46
+ }
47
+
48
+ /// Raw identity for file naming and persistence.
49
+ #[must_use]
50
+ pub const fn raw(self) -> u64 {
51
+ self.0
52
+ }
53
+
54
+ /// The scope of the journal that issued this identity, see
55
+ /// [`Journal::with_scope`].
56
+ #[must_use]
57
+ pub const fn scope(self) -> u16 {
58
+ scope_of(self.0)
59
+ }
60
+ }
61
+
62
+ #[derive(Debug, Clone, Copy, PartialEq, Eq)]
63
+ pub(crate) enum Phase {
64
+ Active,
65
+ Sealed,
66
+ Proposed(u64),
67
+ Durable(u64, Durable),
68
+ }
69
+
70
+ #[derive(Debug, Clone, Copy, PartialEq, Eq)]
71
+ pub(crate) struct Slot {
72
+ pub(crate) id: u64,
73
+ pub(crate) revision: u64,
74
+ pub(crate) base: u64,
75
+ pub(crate) span: u64,
76
+ pub(crate) stored: u64,
77
+ pub(crate) live: u64,
78
+ pub(crate) bytes: u64,
79
+ pub(crate) opened_at: Duration,
80
+ pub(crate) phase: Phase,
81
+ }
82
+
83
+ impl Slot {
84
+ const fn active(id: u64, base: u64, opened_at: Duration) -> Self {
85
+ Self {
86
+ id,
87
+ revision: 0,
88
+ base,
89
+ span: 0,
90
+ stored: 0,
91
+ live: 0,
92
+ bytes: 0,
93
+ opened_at,
94
+ phase: Phase::Active,
95
+ }
96
+ }
97
+
98
+ /// Never overflows: the range was checked when the segment was sealed.
99
+ pub(crate) const fn end(&self) -> u64 {
100
+ self.base + self.span
101
+ }
102
+ }
103
+
104
+ /// Metadata for up to `SEGMENTS` log segments and `PINS` retention pins.
105
+ ///
106
+ /// The journal writes nothing, compacts nothing and deletes nothing. It keeps
107
+ /// the bookkeeping that says when a segment should roll, which sealed
108
+ /// segments are worth compacting, and when an original may be deleted.
109
+ #[derive(Debug, Clone, PartialEq, Eq)]
110
+ pub struct Journal<const SEGMENTS: usize, const PINS: usize> {
111
+ pub(crate) policy: Policy,
112
+ pub(crate) slots: [Option<Slot>; SEGMENTS],
113
+ pub(crate) len: usize,
114
+ /// The one sequence every segment id, pin generation and replacement
115
+ /// attempt is drawn from, so no two handles share a raw value.
116
+ pub(crate) issuer: Issuer,
117
+ pub(crate) pins: [Option<PinSlot>; PINS],
118
+ }
119
+
120
+ impl<const SEGMENTS: usize, const PINS: usize> Journal<SEGMENTS, PINS> {
121
+ /// Segment slots, including the active one.
122
+ pub const SEGMENT_CAPACITY: usize = SEGMENTS;
123
+ /// Pin slots, readers and checkpoints together.
124
+ pub const PIN_CAPACITY: usize = PINS;
125
+
126
+ /// Start in scope zero with one active segment at `base_offset`, opened
127
+ /// at `now`: monotonic time since an epoch the caller picks.
128
+ #[must_use]
129
+ pub const fn new(policy: Policy, base_offset: u64, now: Duration) -> Self {
130
+ Self::with_scope(policy, base_offset, now, 0)
131
+ }
132
+
133
+ /// Start as [`Journal::new`] does, issuing segment ids, pin generations
134
+ /// and replacement attempts that carry `scope`.
135
+ ///
136
+ /// All three come from one sequence counted from one, so no two handles
137
+ /// of a journal share a raw value; the active segment takes the first.
138
+ /// Two journals in the same scope issue the same values, and a handle
139
+ /// given to the wrong journal names something real there. Give every
140
+ /// journal whose handles could meet its own scope: an identity of another
141
+ /// scope then names nothing, and every call that takes one refuses it
142
+ /// before looking anything up. The journal cannot pick a scope itself;
143
+ /// keeping them distinct is the caller's job.
144
+ #[must_use]
145
+ pub const fn with_scope(policy: Policy, base_offset: u64, now: Duration, scope: u16) -> Self {
146
+ Self::opened(policy, base_offset, now, Issuer::new(scope)).0
147
+ }
148
+
149
+ /// Start as [`Journal::with_scope`] does after a restart, continuing the
150
+ /// sequence of an earlier journal of `scope` that had issued `issued`
151
+ /// identities (its [`Journal::issued`]). Segments and pins are not
152
+ /// restored: the journal has one active segment at `base_offset`, and its
153
+ /// id is the next of the sequence.
154
+ ///
155
+ /// A journal started again with [`Journal::with_scope`] reissues the
156
+ /// identities of the one before it, so a handle that outlived the restart
157
+ /// could name a new entry. Resume from the saved count, or give each
158
+ /// incarnation a scope of its own. Resuming from a count lower than what
159
+ /// was really issued reissues too: save [`Journal::issued`] before
160
+ /// handing an identity out.
161
+ ///
162
+ /// `None` when `issued` is beyond what a scope holds, or leaves no
163
+ /// identity for the active segment.
164
+ #[must_use]
165
+ pub const fn resumed(
166
+ policy: Policy,
167
+ base_offset: u64,
168
+ now: Duration,
169
+ scope: u16,
170
+ issued: u64,
171
+ ) -> Option<Self> {
172
+ let Some(sequence) = Issuer::resumed(scope, issued) else {
173
+ return None;
174
+ };
175
+ match Self::opened(policy, base_offset, now, sequence) {
176
+ (journal, true) => Some(journal),
177
+ (_, false) => None,
178
+ }
179
+ }
180
+
181
+ /// A journal on `issuer`, and whether every segment slot it needs (one,
182
+ /// or none at zero capacity) got its identity.
183
+ const fn opened(policy: Policy, base: u64, now: Duration, mut issuer: Issuer) -> (Self, bool) {
184
+ let mut slots = [None; SEGMENTS];
185
+ let (mut len, mut complete) = (0, true);
186
+ if let Some(first) = slots.first_mut() {
187
+ match issuer.issue() {
188
+ Ok(id) => {
189
+ *first = Some(Slot::active(id, base, now));
190
+ len = 1;
191
+ }
192
+ Err(periscope_machines::Refusal::Exhausted) => complete = false,
193
+ }
194
+ }
195
+ let journal = Self {
196
+ policy,
197
+ slots,
198
+ len,
199
+ issuer,
200
+ pins: [None; PINS],
201
+ };
202
+ (journal, complete)
203
+ }
204
+
205
+ /// The scope carried by every identity this journal issues.
206
+ #[must_use]
207
+ pub const fn scope(&self) -> u16 {
208
+ self.issuer.scope()
209
+ }
210
+
211
+ /// How many identities (segment ids, pin generations and attempts
212
+ /// together) this journal's sequence has issued; what
213
+ /// [`Journal::resumed`] continues from.
214
+ #[must_use]
215
+ pub const fn issued(&self) -> u64 {
216
+ self.issuer.issued()
217
+ }
218
+
219
+ /// The thresholds this journal decides with.
220
+ #[must_use]
221
+ pub const fn policy(&self) -> Policy {
222
+ self.policy
223
+ }
224
+
225
+ /// Segments currently tracked, including the active one.
226
+ #[must_use]
227
+ pub const fn len(&self) -> usize {
228
+ self.len
229
+ }
230
+
231
+ /// True only for a journal with zero segment capacity.
232
+ #[must_use]
233
+ pub const fn is_empty(&self) -> bool {
234
+ self.len == 0
235
+ }
236
+
237
+ /// The segment being written.
238
+ #[must_use]
239
+ pub fn active(&self) -> Option<Segment> {
240
+ self.active_slot().map(|slot| self.view(&slot))
241
+ }
242
+
243
+ /// One segment's metadata; `None` for an identity of another scope.
244
+ #[must_use]
245
+ pub fn segment(&self, id: SegmentId) -> Option<Segment> {
246
+ self.find(id).map(|slot| self.view(&slot))
247
+ }
248
+
249
+ /// The segment whose range holds `offset`. The active segment answers
250
+ /// for every offset at or after its base.
251
+ #[must_use]
252
+ pub fn covering(&self, offset: u64) -> Option<Segment> {
253
+ self.tracked()
254
+ .find(|slot| {
255
+ offset >= slot.base && (slot.phase == Phase::Active || offset < slot.end())
256
+ })
257
+ .map(|slot| self.view(slot))
258
+ }
259
+
260
+ /// Every segment after `after` in ascending identity order. Resume by
261
+ /// passing the last identity yielded.
262
+ ///
263
+ /// `after` is an identity this journal issued: one it never issued (of
264
+ /// another scope, a zero sequence, or one not reached) is no position in
265
+ /// this journal, and the iterator is empty. `None` is the one way to
266
+ /// begin. An issued identity is a position whether or not it still names
267
+ /// a segment, so resuming after a segment that has since left the journal
268
+ /// works, and so does any pin generation or attempt: all three come from
269
+ /// the one sequence that orders segment ids.
270
+ #[must_use]
271
+ pub fn segments(&self, after: Option<SegmentId>) -> Segments<'_, SEGMENTS, PINS> {
272
+ self.scan(after, Filter::All)
273
+ }
274
+
275
+ /// Sealed segments with no replacement in flight whose dead-record share
276
+ /// meets the policy, oldest first. Resume as with [`Journal::segments`].
277
+ #[must_use]
278
+ pub fn compaction_candidates(&self, after: Option<SegmentId>) -> Segments<'_, SEGMENTS, PINS> {
279
+ self.scan(after, Filter::Candidates)
280
+ }
281
+
282
+ /// Segments whose replacement is durable and which no pin retains.
283
+ /// Resume as with [`Journal::segments`].
284
+ #[must_use]
285
+ pub fn reclaimable(&self, after: Option<SegmentId>) -> Segments<'_, SEGMENTS, PINS> {
286
+ self.scan(after, Filter::Reclaimable)
287
+ }
288
+
289
+ /// Decide whether the active segment should roll, from its observed
290
+ /// byte size and record count and the caller's monotonic clock. Reads only.
291
+ ///
292
+ /// An empty segment is never due, whatever its age.
293
+ ///
294
+ /// # Errors
295
+ /// [`Refusal::NoActive`], [`Refusal::TimeRegression`] or
296
+ /// [`Refusal::Overflow`] when the observation cannot be true.
297
+ pub fn roll_decision(&self, bytes: u64, records: u64, now: Duration) -> Result<Roll, Refusal> {
298
+ let active = self.active_slot().ok_or(Refusal::NoActive)?;
299
+ let age = now
300
+ .checked_sub(active.opened_at)
301
+ .ok_or(Refusal::TimeRegression)?;
302
+ active.base.checked_add(records).ok_or(Refusal::Overflow)?;
303
+ if records == 0 {
304
+ return Ok(Roll::Keep);
305
+ }
306
+ Ok(self
307
+ .policy
308
+ .due(bytes, records, age)
309
+ .map_or(Roll::Keep, Roll::Due))
310
+ }
311
+
312
+ /// Record that the caller sealed the active segment at `bytes` and
313
+ /// `records` and opened a new one at `now`. A roll need not be due.
314
+ ///
315
+ /// # Errors
316
+ /// [`Refusal::NoActive`], [`Refusal::Empty`], [`Refusal::TimeRegression`],
317
+ /// [`Refusal::Overflow`], [`Refusal::SegmentsFull`] or
318
+ /// [`Refusal::Exhausted`]; a refusal changes nothing.
319
+ pub fn roll(&mut self, bytes: u64, records: u64, now: Duration) -> Result<Rolled, Refusal> {
320
+ let mut sealed = self.active_slot().ok_or(Refusal::NoActive)?;
321
+ if records == 0 {
322
+ return Err(Refusal::Empty);
323
+ }
324
+ if now < sealed.opened_at {
325
+ return Err(Refusal::TimeRegression);
326
+ }
327
+ let next_offset = sealed.base.checked_add(records).ok_or(Refusal::Overflow)?;
328
+ // The active slot and the one after it; a full journal has no second.
329
+ let Some([active, next]) = self
330
+ .len
331
+ .checked_sub(1)
332
+ .and_then(|last| self.slots.get_mut(last..))
333
+ .and_then(<[_]>::first_chunk_mut)
334
+ else {
335
+ return Err(Refusal::SegmentsFull);
336
+ };
337
+ let opened = self.issuer.issue()?;
338
+ sealed.span = records;
339
+ sealed.stored = records;
340
+ sealed.live = records;
341
+ sealed.bytes = bytes;
342
+ sealed.phase = Phase::Sealed;
343
+ *active = Some(sealed);
344
+ *next = Some(Slot::active(opened, next_offset, now));
345
+ self.len += 1;
346
+ Ok(Rolled {
347
+ sealed: SegmentId(sealed.id),
348
+ opened: SegmentId(opened),
349
+ next_offset,
350
+ })
351
+ }
352
+
353
+ /// Record how many of a sealed segment's stored records are still live.
354
+ ///
355
+ /// # Errors
356
+ /// [`Refusal::Unknown`] (which includes an identity of another scope),
357
+ /// [`Refusal::Active`] or [`Refusal::Exceeds`]; a refusal changes nothing.
358
+ pub fn observe_live(&mut self, id: SegmentId, live_records: u64) -> Result<(), Refusal> {
359
+ let (_, slot) = self.find_mut(id).ok_or(Refusal::Unknown)?;
360
+ if slot.phase == Phase::Active {
361
+ return Err(Refusal::Active);
362
+ }
363
+ if live_records > slot.stored {
364
+ return Err(Refusal::Exceeds);
365
+ }
366
+ slot.live = live_records;
367
+ Ok(())
368
+ }
369
+
370
+ /// The tracked segments, in slot order.
371
+ pub(crate) fn tracked(&self) -> impl Iterator<Item = &Slot> {
372
+ self.slots.iter().take(self.len).flatten()
373
+ }
374
+
375
+ /// A copy of the segment `id` names. An identity this journal never
376
+ /// issued, of another scope or not yet reached, names nothing here: no
377
+ /// slot is read.
378
+ pub(crate) fn find(&self, id: SegmentId) -> Option<Slot> {
379
+ if !self.issuer.owns(id.0) {
380
+ return None;
381
+ }
382
+ self.tracked().find(|slot| slot.id == id.0).copied()
383
+ }
384
+
385
+ /// The slot index and the segment `id` names, as [`Journal::find`].
386
+ pub(crate) fn find_mut(&mut self, id: SegmentId) -> Option<(usize, &mut Slot)> {
387
+ locate(&self.issuer, &mut self.slots, self.len, id)
388
+ }
389
+
390
+ fn active_slot(&self) -> Option<Slot> {
391
+ let last = self.len.checked_sub(1)?;
392
+ let slot = self.slots.get(last).copied().flatten()?;
393
+ (slot.phase == Phase::Active).then_some(slot)
394
+ }
395
+
396
+ fn view(&self, slot: &Slot) -> Segment {
397
+ let active = slot.phase == Phase::Active;
398
+ let pinned = !active && self.low_pin().is_some_and(|offset| offset < slot.end());
399
+ let id = SegmentId(slot.id);
400
+ let (state, attempt) = match slot.phase {
401
+ Phase::Active => (SegmentState::Active, None),
402
+ Phase::Sealed => (SegmentState::Sealed, None),
403
+ Phase::Proposed(attempt) => (SegmentState::Proposed, Some(attempt)),
404
+ Phase::Durable(attempt, _) if pinned => (SegmentState::Durable, Some(attempt)),
405
+ Phase::Durable(attempt, _) => (SegmentState::Reclaimable, Some(attempt)),
406
+ };
407
+ Segment {
408
+ id,
409
+ revision: slot.revision,
410
+ base_offset: slot.base,
411
+ records: slot.span,
412
+ stored_records: slot.stored,
413
+ live_records: slot.live,
414
+ bytes: slot.bytes,
415
+ opened_at: slot.opened_at,
416
+ state,
417
+ retention: if pinned {
418
+ Retention::Pinned
419
+ } else {
420
+ Retention::Unpinned
421
+ },
422
+ replacement: attempt.map(|attempt| Replacement {
423
+ segment: id,
424
+ attempt,
425
+ }),
426
+ }
427
+ }
428
+
429
+ fn scan(&self, after: Option<SegmentId>, filter: Filter) -> Segments<'_, SEGMENTS, PINS> {
430
+ let index = match after {
431
+ None => 0,
432
+ Some(after) if !self.issuer.owns(after.0) => self.len,
433
+ Some(after) => self
434
+ .slots
435
+ .iter()
436
+ .take(self.len)
437
+ .position(|slot| slot.is_some_and(|slot| slot.id > after.0))
438
+ .unwrap_or(self.len),
439
+ };
440
+ Segments {
441
+ journal: self,
442
+ index,
443
+ filter,
444
+ }
445
+ }
446
+ }
447
+
448
+ /// The slot index and the segment `id` names among the first `len` of
449
+ /// `slots`, borrowing only `slots` so the caller may still issue from
450
+ /// `issuer`. An identity `issuer` never issued names nothing: no slot is read.
451
+ fn locate<'a>(
452
+ issuer: &Issuer,
453
+ slots: &'a mut [Option<Slot>],
454
+ len: usize,
455
+ id: SegmentId,
456
+ ) -> Option<(usize, &'a mut Slot)> {
457
+ if !issuer.owns(id.0) {
458
+ return None;
459
+ }
460
+ slots
461
+ .iter_mut()
462
+ .take(len)
463
+ .enumerate()
464
+ .find_map(|(index, slot)| {
465
+ slot.as_mut()
466
+ .filter(|slot| slot.id == id.0)
467
+ .map(|slot| (index, slot))
468
+ })
469
+ }
470
+
471
+ #[derive(Debug, Clone, Copy)]
472
+ enum Filter {
473
+ All,
474
+ Candidates,
475
+ Reclaimable,
476
+ }
477
+
478
+ /// Resumable iterator over segment views, bounded by the segment capacity.
479
+ #[derive(Debug, Clone)]
480
+ pub struct Segments<'a, const SEGMENTS: usize, const PINS: usize> {
481
+ journal: &'a Journal<SEGMENTS, PINS>,
482
+ index: usize,
483
+ filter: Filter,
484
+ }
485
+
486
+ impl<const SEGMENTS: usize, const PINS: usize> Iterator for Segments<'_, SEGMENTS, PINS> {
487
+ type Item = Segment;
488
+
489
+ fn next(&mut self) -> Option<Segment> {
490
+ while self.index < self.journal.len {
491
+ let slot = self.journal.slots.get(self.index).copied().flatten();
492
+ self.index += 1;
493
+ let Some(view) = slot.map(|slot| self.journal.view(&slot)) else {
494
+ continue;
495
+ };
496
+ let wanted = match self.filter {
497
+ Filter::All => true,
498
+ Filter::Candidates => {
499
+ view.state == SegmentState::Sealed
500
+ && self
501
+ .journal
502
+ .policy
503
+ .compactable(view.stored_records, view.live_records)
504
+ }
505
+ Filter::Reclaimable => view.state == SegmentState::Reclaimable,
506
+ };
507
+ if wanted {
508
+ return Some(view);
509
+ }
510
+ }
511
+ None
512
+ }
513
+ }
514
+
515
+ #[cfg(test)]
516
+ extern crate std;
517
+ #[cfg(test)]
518
+ mod tests;