pigeonhole-db 1.1.0

data/vendor/pager/src/alloc.rs

@@ -0,0 +1,1008 @@
+//! The in-memory free-space map: a buddy allocator over 64 KiB units.
+//!
+//! A *unit* is the smallest extent (64 KiB, 16 pages); unit `u` covers pages `16u..16u+16`.
+//! An extent of size class `c` is `2^c` units aligned to `2^c` units, so its buddy is
+//! `u ^ 2^c`. Unit 0 holds the superblocks, the lock page and the reserved pages and is never
+//! free. Free blocks are kept per class in ordered sets (lowest address first) and coalesce
+//! with their buddy on free, so the map never holds two free buddies.
+//!
+//! Every unit below `frontier` (the file's end) is in exactly one of: unit 0, a free block,
+//! or a used extent (live or retired). Nothing here does I/O; the pager grows and truncates
+//! the file around these calls.
+
+use std::collections::{BTreeMap, BTreeSet};
+
+use pigeonhole_format::ManifestVersion;
+
+use crate::Extent;
+
+/// Pages per unit (64 KiB / 4 KiB).
+pub(crate) const UNIT_PAGES: u64 = 16;
+/// Bytes per unit.
+pub(crate) const UNIT_BYTES: u64 = 64 * 1024;
+/// Number of size classes (0..=10).
+const CLASSES: usize = Extent::MAX_CLASS as usize + 1;
+
+/// A used extent: live, or retired at a manifest version and awaiting reclamation.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+struct Used {
+    class: u8,
+    retired: Option<ManifestVersion>,
+    /// Known to be referenced by a durable root: loaded at open, or named by a committed
+    /// root. A live extent that is not is *pending* (allocated in this session); only
+    /// pending extents may be abandoned or trimmed. An SST published in this session stays
+    /// pending here, since the pager never sees the manifest's contents.
+    published: bool,
+    /// Holds free space of a region cleared for a larger extent (`Alloc::clear_for`): never
+    /// published, and taken by `Alloc::claim_region` or freed by `Alloc::release_region`.
+    placeholder: bool,
+}
+
+/// Why a set of live extents cannot be loaded.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum LoadError {
+    /// Not page 16 or later, unknown class, or misaligned.
+    Invalid,
+    /// Two live extents overlap without being equal.
+    Overlap,
+    /// A live extent ends past the file.
+    PastEnd,
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct Alloc {
+    free: FreeLists,
+    used: BTreeMap<u64, Used>,
+    /// `(superseded_at, unit)` of every retired extent.
+    retired: BTreeSet<(ManifestVersion, u64)>,
+    /// End of the file, in units (at least 1).
+    frontier: u64,
+    used_units: u64,
+    retired_units: u64,
+    /// Regions held for a larger extent (`clear_for`): first unit -> class. Space freed
+    /// inside one becomes a placeholder, not free, so no allocation lands there.
+    held: BTreeMap<u64, u8>,
+    /// Growth ahead of need ([`Alloc::wants_ahead`]): the class of the block it keeps free,
+    /// and the file length (units) from which it does.
+    ahead_class: u8,
+    ahead_from: u64,
+}
+
+/// A region `Alloc::clear_for` could clear: occupied units, first unit, and occupants as
+/// `(unit, class)`, largest first.
+type Candidate = (u64, u64, Vec<(u64, u8)>);
+
+/// Free blocks per class, lowest address first.
+type FreeLists = [BTreeSet<u64>; CLASSES];
+
+/// Takes a block of `class` from `free` as `Alloc::alloc_lowest` would: the lowest-addressed
+/// free block that fits, split down. Returns its unit.
+fn take_lowest(free: &mut FreeLists, class: u8) -> Option<u64> {
+    let (unit, from) = (class..=Extent::MAX_CLASS)
+        .filter_map(|k| free[k as usize].first().map(|&u| (u, k)))
+        .min()?;
+    free[from as usize].remove(&unit);
+    let mut k = from;
+    while k > class {
+        k -= 1;
+        free[k as usize].insert(unit + units(k));
+    }
+    Some(unit)
+}
+
+/// Units in an extent of `class`.
+fn units(class: u8) -> u64 {
+    1 << class
+}
+
+/// The largest class of an aligned block starting at `from` and ending by `to`.
+fn block_class(from: u64, to: u64) -> u8 {
+    let mut class = 0u8;
+    while class < Extent::MAX_CLASS
+        && from.is_multiple_of(units(class + 1))
+        && from + units(class + 1) <= to
+    {
+        class += 1;
+    }
+    class
+}
+
+/// The file length, in units, from which it grows ahead of need by default
+/// ([`Alloc::wants_ahead`]): two blocks of the largest class (128 MiB).
+pub(crate) const AHEAD_FROM_UNITS: u64 = 2 << Extent::MAX_CLASS;
+
+/// The extent's first unit, if it is a well-formed extent.
+pub(crate) fn unit_of(e: Extent) -> Option<u64> {
+    if e.size_class > Extent::MAX_CLASS || !e.page.is_multiple_of(UNIT_PAGES) {
+        return None;
+    }
+    let u = e.page / UNIT_PAGES;
+    (u != 0 && u.is_multiple_of(units(e.size_class))).then_some(u)
+}
+
+fn extent(unit: u64, class: u8) -> Extent {
+    Extent {
+        page: unit * UNIT_PAGES,
+        size_class: class,
+    }
+}
+
+impl Alloc {
+    /// An empty map over a file of `frontier` units (at least 1).
+    pub(crate) fn empty(frontier: u64) -> Self {
+        let mut a = Self {
+            free: Default::default(),
+            used: BTreeMap::new(),
+            retired: BTreeSet::new(),
+            frontier: frontier.max(1),
+            used_units: 0,
+            retired_units: 0,
+            held: BTreeMap::new(),
+            ahead_class: Extent::MAX_CLASS,
+            ahead_from: AHEAD_FROM_UNITS,
+        };
+        a.free_range(1, a.frontier);
+        a
+    }
+
+    /// A map over a file of `frontier` units in which exactly `live` is used. Exact duplicates
+    /// in `live` are accepted (tablets may share an SST after a split).
+    pub(crate) fn load(
+        frontier: u64,
+        live: impl IntoIterator<Item = Extent>,
+    ) -> Result<Self, LoadError> {
+        let frontier = frontier.max(1);
+        let mut extents: Vec<(u64, u8)> = Vec::new();
+        for e in live {
+            let u = unit_of(e).ok_or(LoadError::Invalid)?;
+            extents.push((u, e.size_class));
+        }
+        extents.sort_unstable();
+        extents.dedup();
+        let mut a = Self {
+            free: Default::default(),
+            used: BTreeMap::new(),
+            retired: BTreeSet::new(),
+            frontier,
+            used_units: 0,
+            retired_units: 0,
+            held: BTreeMap::new(),
+            ahead_class: Extent::MAX_CLASS,
+            ahead_from: AHEAD_FROM_UNITS,
+        };
+        let mut next = 1;
+        for (u, class) in extents {
+            if u < next {
+                return Err(LoadError::Overlap);
+            }
+            let end = u + units(class);
+            if end > frontier {
+                return Err(LoadError::PastEnd);
+            }
+            a.free_range(next, u);
+            a.used.insert(
+                u,
+                Used {
+                    class,
+                    retired: None,
+                    published: true,
+                    placeholder: false,
+                },
+            );
+            a.used_units += units(class);
+            next = end;
+        }
+        a.free_range(next, frontier);
+        Ok(a)
+    }
+
+    /// Adds `[from, to)` to the free map as maximal aligned blocks, coalescing with free
+    /// neighbors.
+    fn free_range(&mut self, mut from: u64, to: u64) {
+        while from < to {
+            let class = block_class(from, to);
+            self.free_block(from, class);
+            from += units(class);
+        }
+    }
+
+    /// Returns one block to the free map, merging it with its buddy while the buddy is free.
+    fn free_block(&mut self, mut unit: u64, mut class: u8) {
+        while class < Extent::MAX_CLASS {
+            let buddy = unit ^ units(class);
+            if !self.free[class as usize].remove(&buddy) {
+                break;
+            }
+            unit = unit.min(buddy);
+            class += 1;
+        }
+        self.free[class as usize].insert(unit);
+    }
+
+    /// Takes `block` (free, of class `from`) out of the free map and splits it down to
+    /// `class`, returning the lower halves' upper buddies to the map. Returns its unit.
+    fn take_split(&mut self, unit: u64, from: u8, class: u8) -> u64 {
+        self.free[from as usize].remove(&unit);
+        let mut k = from;
+        while k > class {
+            k -= 1;
+            self.free[k as usize].insert(unit + units(k));
+        }
+        unit
+    }
+
+    fn mark_used(&mut self, unit: u64, class: u8) -> Extent {
+        self.used.insert(
+            unit,
+            Used {
+                class,
+                retired: None,
+                published: false,
+                placeholder: false,
+            },
+        );
+        self.used_units += units(class);
+        extent(unit, class)
+    }
+
+    /// Allocates from free space: the smallest free class that fits, lowest address within
+    /// it. `None` if the file must grow.
+    pub(crate) fn alloc_free(&mut self, class: u8) -> Option<Extent> {
+        let from = (class..=Extent::MAX_CLASS).find(|&k| !self.free[k as usize].is_empty())?;
+        let unit = *self.free[from as usize].first().expect("non-empty");
+        let unit = self.take_split(unit, from, class);
+        Some(self.mark_used(unit, class))
+    }
+
+    /// Allocates the lowest-addressed free block that fits, regardless of class (for
+    /// relocation toward the start of the file). `None` if no free block fits.
+    pub(crate) fn alloc_lowest(&mut self, class: u8) -> Option<Extent> {
+        let (unit, from) = (class..=Extent::MAX_CLASS)
+            .filter_map(|k| self.free[k as usize].first().map(|&u| (u, k)))
+            .min()?;
+        let unit = self.take_split(unit, from, class);
+        Some(self.mark_used(unit, class))
+    }
+
+    /// Where growing the file would place an extent of `class`, and the new frontier.
+    pub(crate) fn grow_target(&self, class: u8) -> (u64, u64) {
+        let unit = self.frontier.next_multiple_of(units(class));
+        (unit, unit + units(class))
+    }
+
+    /// Records a growth planned by [`grow_target`](Self::grow_target) once the file has been
+    /// extended: the alignment gap becomes free and the extent is used.
+    pub(crate) fn alloc_grown(&mut self, class: u8) -> Extent {
+        let (unit, end) = self.grow_target(class);
+        let old = self.frontier;
+        self.frontier = end;
+        self.free_range(old, unit);
+        self.mark_used(unit, class)
+    }
+
+    /// Sets the growth ahead of need: the class of the block it keeps free, and the file
+    /// length (units) from which it does.
+    pub(crate) fn set_ahead(&mut self, class: u8, from: u64) {
+        (self.ahead_class, self.ahead_from) = (class.min(Extent::MAX_CLASS), from);
+    }
+
+    /// Whether the file should grow ahead of need: no free block of the ahead class (the
+    /// largest, by default) or larger is left, so the next allocation that large would grow
+    /// the file inline, and the file is long enough ([`AHEAD_FROM_UNITS`] by default: a
+    /// small database never grows ahead).
+    pub(crate) fn wants_ahead(&self) -> bool {
+        self.frontier >= self.ahead_from
+            && (self.ahead_class..=Extent::MAX_CLASS).all(|k| self.free[k as usize].is_empty())
+    }
+
+    /// Claims the tail for a growth ahead of need: `[frontier, end)`, through the next block
+    /// of the ahead class, becomes the new frontier, held as placeholders so no allocation
+    /// lands there before the file has grown. Returns `(from, end)` for
+    /// [`release_ahead`](Self::release_ahead).
+    pub(crate) fn claim_ahead(&mut self) -> (u64, u64) {
+        let (_, end) = self.grow_target(self.ahead_class);
+        let from = self.frontier;
+        self.frontier = end;
+        let mut at = from;
+        while at < end {
+            let class = block_class(at, end);
+            self.used.insert(
+                at,
+                Used {
+                    class,
+                    retired: None,
+                    published: false,
+                    placeholder: true,
+                },
+            );
+            self.used_units += units(class);
+            at += units(class);
+        }
+        (from, end)
+    }
+
+    /// Ends a growth claimed by [`claim_ahead`](Self::claim_ahead). Grown (`grown`), its
+    /// placeholders become free space. Not grown, the claim is undone if it is still the
+    /// file's tail; otherwise (an inline growth went past it meanwhile) the placeholders stay,
+    /// unused until the next open.
+    pub(crate) fn release_ahead(&mut self, from: u64, end: u64, grown: bool) {
+        if !grown && self.frontier != end {
+            return;
+        }
+        let blocks: Vec<u64> = self
+            .used
+            .range(from..end)
+            .filter(|(_, u)| u.placeholder)
+            .map(|(&u, _)| u)
+            .collect();
+        for unit in blocks {
+            let used = self.used.remove(&unit).expect("placeholder");
+            self.used_units -= units(used.class);
+            if grown {
+                self.free_block(unit, used.class);
+            }
+        }
+        if !grown {
+            self.frontier = from;
+        }
+    }
+
+    /// Whether `e` is exactly a pending extent: live and not known to be published.
+    pub(crate) fn is_pending(&self, e: Extent) -> bool {
+        unit_of(e)
+            .and_then(|u| self.used.get(&u))
+            .is_some_and(|u| u.class == e.size_class && u.retired.is_none() && !u.published)
+    }
+
+    /// Records that a durable root references the live extent `e`. Returns false if `e` is
+    /// not live.
+    pub(crate) fn publish(&mut self, e: Extent) -> bool {
+        let Some(unit) = unit_of(e) else { return false };
+        match self.used.get_mut(&unit) {
+            Some(u) if u.class == e.size_class && u.retired.is_none() => {
+                u.published = true;
+                true
+            }
+            _ => false,
+        }
+    }
+
+    /// Frees a pending extent at once. Returns false (and changes nothing) if `e` is not
+    /// pending.
+    pub(crate) fn release_live(&mut self, e: Extent) -> bool {
+        let Some(unit) = unit_of(e) else { return false };
+        if !self.is_pending(e) {
+            return false;
+        }
+        self.free_used(unit);
+        true
+    }
+
+    /// Frees the used extent at `unit`, or keeps it as a placeholder inside a held region.
+    fn free_used(&mut self, unit: u64) {
+        if self.in_held(unit) {
+            let u = self.used.get_mut(&unit).expect("used");
+            *u = Used {
+                class: u.class,
+                retired: None,
+                published: false,
+                placeholder: true,
+            };
+            return;
+        }
+        let used = self.used.remove(&unit).expect("used");
+        self.used_units -= units(used.class);
+        self.free_block(unit, used.class);
+    }
+
+    /// Whether `unit` lies inside a held region.
+    fn in_held(&self, unit: u64) -> bool {
+        self.held
+            .range(..=unit)
+            .next_back()
+            .is_some_and(|(&at, &class)| unit < at + units(class))
+    }
+
+    /// Shrinks a pending extent to `class` in place, freeing its upper halves at once.
+    /// Returns false (and changes nothing) if `e` is not pending or `class` is larger than
+    /// its class.
+    pub(crate) fn shrink_live(&mut self, e: Extent, class: u8) -> bool {
+        if class > e.size_class || !self.is_pending(e) {
+            return false;
+        }
+        let unit = unit_of(e).expect("pending extent is well formed");
+        self.used.insert(
+            unit,
+            Used {
+                class,
+                retired: None,
+                published: false,
+                placeholder: false,
+            },
+        );
+        self.used_units -= units(e.size_class) - units(class);
+        // Each upper half's buddy is the lower part, still used, so none coalesces.
+        for k in class..e.size_class {
+            self.free_block(unit + units(k), k);
+        }
+        true
+    }
+
+    /// Marks a live extent retired at `at`. Returns false if `e` is not live.
+    pub(crate) fn retire(&mut self, e: Extent, at: ManifestVersion) -> bool {
+        let Some(unit) = unit_of(e) else { return false };
+        match self.used.get_mut(&unit) {
+            Some(u) if u.class == e.size_class && u.retired.is_none() => {
+                u.retired = Some(at);
+                self.retired.insert((at, unit));
+                self.retired_units += units(e.size_class);
+                true
+            }
+            _ => false,
+        }
+    }
+
+    /// Frees every retired extent with `superseded_at <= oldest_live`. Returns how many.
+    pub(crate) fn reclaim(&mut self, oldest_live: ManifestVersion) -> usize {
+        let keep = match oldest_live.checked_add(1) {
+            Some(v) => self.retired.split_off(&(v, 0)),
+            None => BTreeSet::new(),
+        };
+        let done = std::mem::replace(&mut self.retired, keep);
+        for &(_, unit) in &done {
+            let n = units(self.used[&unit].class);
+            self.retired_units -= n;
+            self.free_used(unit);
+        }
+        done.len()
+    }
+
+    /// Whether `e` is exactly a live (used, not retired) extent.
+    pub(crate) fn is_live(&self, e: Extent) -> bool {
+        unit_of(e)
+            .and_then(|u| self.used.get(&u))
+            .is_some_and(|u| u.class == e.size_class && u.retired.is_none())
+    }
+
+    /// Whether `e` is exactly a used extent (live or retired).
+    pub(crate) fn is_used(&self, e: Extent) -> bool {
+        unit_of(e)
+            .and_then(|u| self.used.get(&u))
+            .is_some_and(|u| u.class == e.size_class)
+    }
+
+    /// End of the last used extent, in units (1 if none).
+    pub(crate) fn used_end(&self) -> u64 {
+        self.used
+            .last_key_value()
+            .map_or(1, |(&u, used)| u + units(used.class))
+    }
+
+    /// Drops every free block at or past `end` and makes `end` the frontier. `end` must be at
+    /// least [`used_end`](Self::used_end), so no free block straddles it.
+    pub(crate) fn truncate(&mut self, end: u64) {
+        debug_assert!(end >= self.used_end());
+        for set in &mut self.free {
+            set.split_off(&end);
+        }
+        self.frontier = end;
+    }
+
+    /// Live extents past the point the live set would end at if packed toward the start of
+    /// the file, largest first. Retired extents are not moved: reclaiming them frees their
+    /// space.
+    pub(crate) fn shrink_plan(&self) -> Vec<Extent> {
+        let mut classes: Vec<u8> = self
+            .used
+            .values()
+            .filter(|u| u.retired.is_none() && !u.placeholder)
+            .map(|u| u.class)
+            .collect();
+        classes.sort_unstable_by(|a, b| b.cmp(a));
+        let mut packed = Alloc::empty(1);
+        for class in classes {
+            if packed.alloc_free(class).is_none() {
+                packed.alloc_grown(class);
+            }
+        }
+        let target = packed.frontier;
+        let mut plan: Vec<Extent> = self
+            .used
+            .iter()
+            .filter(|(u, used)| {
+                used.retired.is_none() && !used.placeholder && **u + units(used.class) > target
+            })
+            .map(|(&u, used)| extent(u, used.class))
+            .collect();
+        plan.sort_unstable_by(|a, b| b.size_class.cmp(&a.size_class).then(b.page.cmp(&a.page)));
+        plan
+    }
+
+    /// Clears room for `big`, a live extent no free block of its class lies below. A
+    /// candidate is a region of its class and alignment below it whose occupants are all
+    /// live, smaller and `movable`, and can each move to a free block outside the region and
+    /// below `big`: tried on a copy of the free lists, largest first, taking the lowest free
+    /// block as the moves will (`alloc_lowest`; a snapshot rewrite of the manifest's
+    /// extents allocates the same way, though its new snapshot may need a larger class
+    /// than the old one, in which case it can land above `big`). Of the first `tries` candidates
+    /// among the lowest `scan` regions, the one with the fewest occupied units wins (the
+    /// lowest on a tie). It is held: its free blocks, and the occupants' old extents once they
+    /// are freed, become placeholders, so nothing else is allocated there until
+    /// `claim_region` or `release_region`. Returns the region and the occupants to move.
+    pub(crate) fn clear_for(
+        &mut self,
+        big: Extent,
+        movable: &dyn Fn(Extent) -> bool,
+        tries: usize,
+        scan: usize,
+    ) -> Option<(Extent, Vec<Extent>)> {
+        let limit = unit_of(big)?;
+        let size = units(big.size_class);
+        // The first minimum: the lowest region on a tie.
+        let (_, at, occupants) = (size..limit.saturating_sub(size - 1))
+            .step_by(size as usize)
+            .take(scan)
+            .filter_map(|at| Some((at, self.occupants_to_move(at, big.size_class, movable)?)))
+            .take(tries)
+            .filter_map(|(at, occupants)| self.relocate(at, size, limit, occupants))
+            .min_by_key(|(cost, ..)| *cost)?;
+        self.held.insert(at, big.size_class);
+        for class in 0..CLASSES {
+            let inside: Vec<u64> = self.free[class].range(at..at + size).copied().collect();
+            for u in inside {
+                self.free[class].remove(&u);
+                self.mark_used(u, class as u8);
+                self.used.get_mut(&u).expect("marked").placeholder = true;
+            }
+        }
+        let occupants = occupants.into_iter().map(|(u, c)| extent(u, c)).collect();
+        Some((extent(at, big.size_class), occupants))
+    }
+
+    /// The occupants of the region of `class` at unit `at`, if `clear_for` may clear it: the
+    /// region is not inside a used extent that starts below it, and it holds at least one
+    /// occupant, each live, not a placeholder, smaller than `class` and `movable`.
+    fn occupants_to_move(
+        &self,
+        at: u64,
+        class: u8,
+        movable: &dyn Fn(Extent) -> bool,
+    ) -> Option<Vec<(u64, Used)>> {
+        let inside_below = self
+            .used
+            .range(..at)
+            .next_back()
+            .is_some_and(|(&u, used)| u + units(used.class) > at);
+        if inside_below {
+            return None;
+        }
+        let occupants: Vec<(u64, Used)> = self
+            .used
+            .range(at..at + units(class))
+            .map(|(&u, used)| (u, *used))
+            .collect();
+        let all_movable = occupants.iter().all(|(u, used)| {
+            used.retired.is_none()
+                && !used.placeholder
+                && used.class < class
+                && movable(extent(*u, used.class))
+        });
+        (!occupants.is_empty() && all_movable).then_some(occupants)
+    }
+
+    /// The candidate for clearing the region of `size` units at `at`, if its occupants fit
+    /// below `limit` with the region taken: placed largest first (the lowest unit on a tie),
+    /// each as `alloc_lowest` would.
+    fn relocate(
+        &self,
+        at: u64,
+        size: u64,
+        limit: u64,
+        occupants: Vec<(u64, Used)>,
+    ) -> Option<Candidate> {
+        let mut free = self.free.clone();
+        for set in &mut free {
+            set.retain(|u| !(at..at + size).contains(u));
+        }
+        let mut by_size: Vec<(u64, u8)> =
+            occupants.iter().map(|(u, used)| (*u, used.class)).collect();
+        by_size.sort_unstable_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(&b.0)));
+        let fits = by_size.iter().all(|&(_, class)| {
+            take_lowest(&mut free, class).is_some_and(|t| t + units(class) <= limit)
+        });
+        let cost = by_size.iter().map(|&(_, c)| units(c)).sum();
+        fits.then_some((cost, at, by_size))
+    }
+
+    /// Takes the held `region` as one used (pending) extent once nothing but placeholders
+    /// is inside it (the occupants' old extents are freed). `None`, changing nothing, while
+    /// anything else is.
+    pub(crate) fn claim_region(&mut self, region: Extent) -> Option<Extent> {
+        let at = unit_of(region)?;
+        let class = region.size_class;
+        if self.held.get(&at) != Some(&class) {
+            return None;
+        }
+        let size = units(class);
+        let inside: Vec<(u64, Used)> = self
+            .used
+            .range(at..at + size)
+            .map(|(&u, used)| (u, *used))
+            .collect();
+        if inside.iter().any(|(_, used)| !used.placeholder) {
+            return None;
+        }
+        self.held.remove(&at);
+        for (u, used) in inside {
+            self.used.remove(&u);
+            self.used_units -= units(used.class);
+        }
+        Some(self.mark_used(at, class))
+    }
+
+    /// Stops holding `region`: its placeholders become free space. Returns false if it was
+    /// not held.
+    pub(crate) fn release_region(&mut self, region: Extent) -> bool {
+        let Some(at) = unit_of(region) else {
+            return false;
+        };
+        if self.held.get(&at) != Some(&region.size_class) {
+            return false;
+        }
+        self.held.remove(&at);
+        let placeholders: Vec<u64> = self
+            .used
+            .range(at..at + units(region.size_class))
+            .filter(|(_, used)| used.placeholder)
+            .map(|(&u, _)| u)
+            .collect();
+        for u in placeholders {
+            self.free_used(u);
+        }
+        true
+    }
+
+    pub(crate) fn frontier(&self) -> u64 {
+        self.frontier
+    }
+
+    pub(crate) fn used_units(&self) -> u64 {
+        self.used_units
+    }
+
+    pub(crate) fn retired_units(&self) -> u64 {
+        self.retired_units
+    }
+
+    /// Checks every invariant (tests only): the free blocks, used extents and unit 0 tile
+    /// `[0, frontier)` exactly, and no two free blocks are buddies.
+    #[cfg(test)]
+    pub(crate) fn check(&self) {
+        let mut spans: Vec<(u64, u64)> = vec![(0, 1)];
+        for (k, set) in self.free.iter().enumerate() {
+            for &u in set {
+                assert!(u.is_multiple_of(1 << k), "misaligned free block {u}/{k}");
+                if k < CLASSES - 1 {
+                    assert!(
+                        !set.contains(&(u ^ (1 << k))),
+                        "uncoalesced buddies {u}/{k}"
+                    );
+                }
+                spans.push((u, u + (1 << k)));
+            }
+        }
+        let (mut used, mut retired) = (0, 0);
+        for (&u, x) in &self.used {
+            spans.push((u, u + units(x.class)));
+            used += units(x.class);
+            if let Some(v) = x.retired {
+                assert!(self.retired.contains(&(v, u)));
+                retired += units(x.class);
+            }
+        }
+        assert_eq!(used, self.used_units);
+        assert_eq!(retired, self.retired_units);
+        spans.sort_unstable();
+        let mut next = 0;
+        for (s, e) in spans {
+            assert_eq!(s, next, "gap or overlap at unit {s}");
+            next = e;
+        }
+        assert_eq!(next, self.frontier);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn ext(unit: u64, class: u8) -> Extent {
+        extent(unit, class)
+    }
+
+    #[test]
+    fn empty_file_grows_with_alignment() {
+        let mut a = Alloc::empty(1);
+        assert_eq!(a.alloc_free(0), None);
+        assert_eq!(a.alloc_grown(0), ext(1, 0));
+        // Class 2 is 4 units aligned to 4: units 2-3 become a free class-1 block.
+        assert_eq!(a.alloc_grown(2), ext(4, 2));
+        a.check();
+        assert_eq!(a.frontier(), 8);
+        assert_eq!(a.alloc_free(1), Some(ext(2, 1)));
+        a.check();
+    }
+
+    #[test]
+    fn free_coalesces_back_to_one_block() {
+        let mut a = Alloc::empty(2048);
+        a.check();
+        let xs: Vec<_> = (0..20).map(|_| a.alloc_free(0).unwrap()).collect();
+        a.check();
+        for x in xs {
+            assert!(a.release_live(x));
+        }
+        a.check();
+        assert_eq!(a.free[10].len(), 1, "units 1024..2048 whole again");
+    }
+
+    #[test]
+    fn shrink_live_frees_the_upper_halves() {
+        let mut a = Alloc::empty(1);
+        let x = a.alloc_grown(3);
+        assert_eq!(x, ext(8, 3));
+        assert!(!a.shrink_live(x, 4), "cannot grow");
+        assert!(a.shrink_live(x, 0));
+        a.check();
+        assert_eq!(a.used_units(), 1);
+        assert!(a.is_live(ext(8, 0)) && !a.is_live(x));
+        assert!(!a.shrink_live(x, 0), "no longer live at its old class");
+        // The freed halves (9, 10-11, 12-15) are free blocks, and coalesce once the rest goes.
+        assert!(a.free[0].contains(&9) && a.free[1].contains(&10) && a.free[2].contains(&12));
+        assert!(a.retire(ext(8, 0), 1));
+        assert!(
+            !a.shrink_live(ext(8, 0), 0),
+            "retired extents are not trimmed"
+        );
+        assert_eq!(a.reclaim(1), 1);
+        a.check();
+        assert_eq!(a.used_units(), 0);
+        assert!(a.free[3].contains(&8), "units 8..16 whole again");
+    }
+
+    #[test]
+    fn published_extents_are_neither_released_nor_shrunk() {
+        let mut a = Alloc::load(32, [ext(16, 4)]).unwrap();
+        let loaded = ext(16, 4);
+        assert!(a.is_live(loaded) && !a.is_pending(loaded));
+        assert!(!a.release_live(loaded), "loaded at open: published");
+        assert!(!a.shrink_live(loaded, 0));
+        let fresh = a.alloc_free(2).unwrap();
+        assert!(a.is_pending(fresh));
+        assert!(a.publish(fresh));
+        assert!(!a.release_live(fresh), "named by a committed root");
+        assert!(!a.shrink_live(fresh, 0));
+        // Published extents still retire and reclaim.
+        assert!(a.retire(fresh, 1) && a.retire(loaded, 1));
+        assert_eq!(a.reclaim(1), 2);
+        a.check();
+        assert_eq!(a.used_units(), 0);
+    }
+
+    #[test]
+    fn load_rejects_bad_sets() {
+        assert_eq!(
+            Alloc::load(8, [ext(4, 2), ext(5, 0)]).unwrap_err(),
+            LoadError::Overlap
+        );
+        assert_eq!(Alloc::load(8, [ext(8, 0)]).unwrap_err(), LoadError::PastEnd);
+        let bad = Extent {
+            page: 16,
+            size_class: 1,
+        };
+        assert_eq!(Alloc::load(8, [bad]).unwrap_err(), LoadError::Invalid);
+        let a = Alloc::load(8, [ext(4, 2), ext(4, 2), ext(1, 0)]).unwrap();
+        a.check();
+        assert_eq!(a.used_units(), 5);
+    }
+
+    #[test]
+    fn retire_then_reclaim_by_version() {
+        let mut a = Alloc::empty(1);
+        let x = a.alloc_grown(0);
+        let y = a.alloc_grown(0);
+        assert!(a.retire(x, 5));
+        assert!(!a.retire(x, 6), "already retired");
+        assert!(a.retire(y, 7));
+        assert!(!a.release_live(x), "retired extents are not abandoned");
+        assert_eq!(a.reclaim(4), 0);
+        assert_eq!(a.reclaim(6), 1);
+        assert!(a.is_used(y) && !a.is_used(x));
+        assert_eq!(a.reclaim(u64::MAX), 1);
+        a.check();
+        assert_eq!(a.used_units(), 0);
+    }
+
+    #[test]
+    fn truncate_drops_tail_free_blocks() {
+        let mut a = Alloc::empty(1);
+        let xs: Vec<_> = (0..8).map(|_| a.alloc_grown(0)).collect();
+        for x in &xs[2..] {
+            a.release_live(*x);
+        }
+        a.truncate(a.used_end());
+        a.check();
+        assert_eq!(a.frontier(), 3);
+    }
+
+    /// Issue #314's layout (64 KiB units): every 16-aligned hole below the 1 MiB extent at
+    /// 96 holds a small extent, so it cannot move until one region is cleared.
+    fn fragmented() -> (Alloc, Extent) {
+        let live = [
+            (2, 1),
+            (8, 3),
+            (16, 4),
+            (32, 2),
+            (36, 1),
+            (38, 0),
+            (39, 0),
+            (40, 0),
+            (42, 0),
+            (44, 0),
+            (46, 0),
+            (48, 0),
+            (52, 2),
+            (64, 5),
+            (96, 4),
+        ]
+        .map(|(u, c)| extent(u, c));
+        let a = Alloc::load(112, live).unwrap();
+        (a, extent(96, 4))
+    }
+
+    #[test]
+    fn clearing_a_region_lets_a_large_extent_move_down() {
+        let (mut a, big) = fragmented();
+        assert!(a.clone().alloc_lowest(4).is_none_or(|t| t.page > big.page));
+        let (region, occupants) = a.clear_for(big, &|_| true, 16, 512).unwrap();
+        // 32..48 could be cleared too, but 48..64 moves less: the SST at 48 and the manifest
+        // log at 52, into the holes at 41 and 4..8.
+        assert_eq!(region, extent(48, 4));
+        assert_eq!(occupants, vec![extent(52, 2), extent(48, 0)]);
+        a.check();
+        // Both move into the lowest hole that fits: below 48.
+        let log = a.alloc_lowest(2).unwrap();
+        let sst = a.alloc_lowest(0).unwrap();
+        assert!(
+            log.page < 48 * UNIT_PAGES && sst.page < 48 * UNIT_PAGES,
+            "{log:?} {sst:?}"
+        );
+        // Published at load: the moves' commit retires the occupants; until a reclaim frees
+        // them, the region cannot be taken.
+        for o in &occupants {
+            assert!(a.retire(*o, 1));
+        }
+        assert_eq!(a.claim_region(region), None);
+        a.check();
+        // Reclaimed, they stay held: a flush between the reclaim and the large move
+        // allocates elsewhere (here, everything left below the file's end).
+        a.reclaim(u64::MAX);
+        a.check();
+        while let Some(e) = a.alloc_free(0) {
+            assert!(
+                !(48 * UNIT_PAGES..64 * UNIT_PAGES).contains(&e.page),
+                "{e:?}"
+            );
+        }
+        assert_eq!(a.claim_region(region), Some(extent(48, 4)));
+        assert_eq!(a.claim_region(region), None, "taken");
+        a.check();
+    }
+
+    #[test]
+    fn a_released_region_is_free_space_again() {
+        let (mut a, big) = fragmented();
+        let used = a.used_units;
+        let (region, occupants) = a.clear_for(big, &|_| true, 16, 512).unwrap();
+        assert!(a.used_units > used, "held");
+        // Placeholders are not extents to move.
+        assert!(a.shrink_plan().iter().all(|e| e.page >= 64 * UNIT_PAGES));
+        for o in &occupants {
+            assert!(a.retire(*o, 1));
+        }
+        a.reclaim(u64::MAX);
+        assert!(a.release_region(region));
+        assert!(!a.release_region(region), "no longer held");
+        a.check();
+        // The occupants (retired, then reclaimed) and the held space are all free.
+        assert_eq!(a.used_units, used - 5);
+        assert_eq!(a.alloc_lowest(4), Some(extent(48, 4)));
+    }
+
+    #[test]
+    fn a_region_with_an_occupant_that_cannot_move_is_skipped() {
+        let (mut a, big) = fragmented();
+        // The SST at 48 is pinned: 32..48 is cleared instead, moving more.
+        let pinned = extent(48, 0);
+        let (region, occupants) = a.clear_for(big, &|e| e != pinned, 16, 512).unwrap();
+        assert_eq!(region, extent(32, 4));
+        assert_eq!(occupants.len(), 8, "{occupants:?}");
+        a.check();
+        // With the manifest snapshot at 39 pinned too, no region can be cleared.
+        let (mut a, big) = fragmented();
+        let used = a.used_units;
+        let pinned = [extent(48, 0), extent(39, 0)];
+        assert!(
+            a.clear_for(big, &|e| !pinned.contains(&e), 16, 512)
+                .is_none()
+        );
+        assert_eq!(a.used_units, used, "nothing reserved");
+        a.check();
+        // Nor when the scan stops before the regions that qualify.
+        let (mut a, big) = fragmented();
+        assert!(a.clear_for(big, &|_| true, 16, 1).is_none());
+        a.check();
+    }
+
+    #[test]
+    fn shrink_plan_names_tail_extents() {
+        let mut a = Alloc::empty(1);
+        let xs: Vec<_> = (0..16).map(|_| a.alloc_grown(0)).collect();
+        for x in &xs[..12] {
+            a.release_live(*x);
+        }
+        // Four live units pack into units 1..5.
+        let plan = a.shrink_plan();
+        assert_eq!(plan.len(), 4);
+        for e in plan {
+            let moved = a.alloc_lowest(0).unwrap();
+            assert!(moved.page < e.page);
+            a.release_live(e);
+        }
+        a.truncate(a.used_end());
+        a.check();
+        assert_eq!(a.frontier(), 5);
+    }
+
+    #[test]
+    fn a_growth_ahead_claims_the_tail_until_released() {
+        let mut a = Alloc::empty(3);
+        a.set_ahead(2, 0);
+        assert!(a.wants_ahead());
+        let (from, end) = a.claim_ahead();
+        assert_eq!((from, end, a.frontier()), (3, 8, 8));
+        a.check();
+        // Claimed space is not handed out: an allocation of the class grows past it.
+        assert!(a.alloc_free(2).is_none());
+        assert!(a.alloc_free(0).is_some());
+        a.release_ahead(from, end, true);
+        a.check();
+        assert!(!a.wants_ahead());
+        assert_eq!(a.alloc_free(2).map(|e| e.page / UNIT_PAGES), Some(4));
+        a.check();
+    }
+
+    #[test]
+    fn a_failed_growth_ahead_undoes_its_claim_only_at_the_tail() {
+        let mut a = Alloc::empty(3);
+        a.set_ahead(2, 0);
+        let (from, end) = a.claim_ahead();
+        a.release_ahead(from, end, false);
+        assert_eq!(a.frontier(), 3);
+        a.check();
+        // An inline growth went past the claim: it stays, unused, below the new tail.
+        let (from, end) = a.claim_ahead();
+        a.alloc_grown(2);
+        a.release_ahead(from, end, false);
+        assert_eq!(a.frontier(), 12);
+        a.check();
+    }
+
+    #[test]
+    fn a_small_file_does_not_grow_ahead() {
+        let a = Alloc::empty(3);
+        assert!(!a.wants_ahead());
+        // Long enough, but with a free block of the largest class: not yet.
+        assert!(!Alloc::empty(AHEAD_FROM_UNITS + 1).wants_ahead());
+    }
+}