pigeonhole-db 1.1.0

data/vendor/sst/src/iter.rs

@@ -0,0 +1,803 @@
+//! The SST cursor: partitioned-index seeks, filter pushdown in the block decoder, row skipping
+//! and forward readahead.
+
+use std::collections::VecDeque;
+use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
+
+use pigeonhole_cache::{BlockHandle, Cell};
+use pigeonhole_format::Cursor;
+use pigeonhole_format::block::{Block, BlockAddr, BlockIter, BlockKind};
+use pigeonhole_format::key::row_prefix_len;
+
+use crate::{Error, Fetch, ReadOptions, Result, ScanFilter, SstReader};
+use pigeonhole_format::scan::{CompiledFilter, Verdict};
+
+/// A zero-copy cursor over one SST, from [`SstReader::iter`].
+///
+/// It owns an `Arc<SstReader>` and the pinned blocks it is positioned in, so it can be stored
+/// anywhere (scan cursors, compaction jobs) without borrowing (decision D32). Keys and values
+/// borrow the current data block; [`SstIter::value_cell`] hands out a pin that outlives the
+/// cursor.
+///
+/// The cursor only ever stops on entries its [`ScanFilter`] admits: excluded qualifiers are
+/// skipped with seeks (the compiled filter's hints, [`ScanFilter::compile`]), so a filtered
+/// scan returns exactly the unfiltered entries the compiled filter admits (decision D22).
+/// [`Cursor::skip_row`] uses each block's row-start table and crosses into the next block only
+/// if the row continues there.
+///
+/// ```
+/// use std::sync::Arc;
+/// use pigeonhole_cache::{BlockCache, Priority};
+/// use pigeonhole_format::key::{Kind, encode_key};
+/// use pigeonhole_format::manifest::FamilyOptions;
+/// use pigeonhole_format::superblock::ExtentRef;
+/// use pigeonhole_format::{Cursor, FamilyId, SstId, TableId, TabletId};
+/// use pigeonhole_io::{OpenOptions, Vfs};
+/// use pigeonhole_io::sim::SimVfs;
+/// use pigeonhole_sst::{QualifierFilter, ReadOptions, ScanFilter, SstReader, SstWriter, SstWriterOptions};
+///
+/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
+/// let vfs = SimVfs::new(1);
+/// let file = vfs.open("/db".as_ref(), OpenOptions::read_write_create())?;
+/// let options = SstWriterOptions::for_family(&FamilyOptions::default(), TableId(1), FamilyId(1), TabletId(1));
+/// let extent = ExtentRef { page: 16, size_class: 0 };
+/// let mut w = SstWriter::new(file.clone(), extent, SstId(1), options);
+/// let mut key = Vec::new();
+/// for row in [&b"a"[..], b"b"] {
+///     for q in [&b"meta:x"[..], b"body"] {
+///         key.clear();
+///         encode_key(&mut key, row, q, 7, 1, Kind::Put)?;
+///         // (keys must arrive sorted: "body" < "meta:x")
+///     }
+/// }
+/// # let mut keys = Vec::new();
+/// # for row in [&b"a"[..], b"b"] { for q in [&b"body"[..], b"meta:x"] {
+/// #     let mut k = Vec::new(); encode_key(&mut k, row, q, 7, 1, Kind::Put)?; keys.push(k);
+/// # } }
+/// # for k in &keys { w.add(k, b"\x00v")?; }
+/// let meta = w.finish()?;
+///
+/// let reader = Arc::new(SstReader::open(file, &meta, Arc::new(BlockCache::new(1 << 20, 1)), Priority::Normal)?);
+/// let mut filter = ScanFilter::all();
+/// filter.qualifiers = QualifierFilter::Prefix(b"meta:".to_vec());
+/// let mut it = reader.iter(filter, ReadOptions::default());
+/// it.seek_to_first()?;
+/// let mut n = 0;
+/// while it.valid() {
+///     assert!(it.key().windows(5).any(|w| w == b"meta:"));
+///     let pinned = it.value_cell(); // outlives the cursor
+///     assert_eq!(&pinned[..], b"\x00v");
+///     n += 1;
+///     it.next()?;
+/// }
+/// assert_eq!(n, 2);
+/// # Ok(())
+/// # }
+/// ```
+#[derive(Debug)]
+pub struct SstIter {
+    reader: Arc<SstReader>,
+    filter: CompiledFilter,
+    filter_all: bool,
+    options: ReadOptions,
+    top: Option<BlockIter<BlockHandle>>,
+    index: Option<BlockIter<BlockHandle>>,
+    data: Option<BlockIter<BlockHandle>>,
+    /// Seek hints from the filter.
+    hint: Vec<u8>,
+    /// The row being skipped.
+    row: Vec<u8>,
+    /// Readahead submitted from the miss path (#402); `None` until the first one, so a scan
+    /// that never misses pays one check per block.
+    ahead: Option<Box<Ahead>>,
+    /// Data blocks the cursor moved into forward since it was last positioned (saturating),
+    /// and the size of its next readahead batch: readahead ramps up (#712).
+    stepped: u32,
+    window: u32,
+}
+
+/// Where a cursor's readahead ramp stands ([`SstIter::readahead_ramp`]), to carry it to the
+/// cursor of the next SST a level scan reads on into.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub struct ReadaheadRamp {
+    stepped: u32,
+    window: u32,
+}
+
+/// Forward blocks a cursor reads after a seek before it reads ahead, by default
+/// ([`ReadOptions::readahead_after`]), and the first batch's size, which doubles with each
+/// batch up to [`ReadOptions::readahead_blocks`] (#712). A short scan (a few rows past its
+/// seek, the common case) then fetches nothing it does not read; at 16 blocks from the first
+/// miss, ycsb-e's scans fetched, verified and decompressed mostly blocks they never reached,
+/// and the I/O backend's single completion thread did all of it.
+pub(crate) const RAMP_AFTER: u32 = 2;
+const RAMP_FIRST: u32 = 2;
+
+/// A cursor's readahead (#402): fetches it submitted from its miss path for the blocks it
+/// reads next ([`SstIter::upcoming`]), each admitted by its completion's continuation, so the
+/// scan thread never waits for them unless it reaches a block still in flight.
+#[derive(Default)]
+struct Ahead {
+    /// Blocks whose fetch completed, by offset, in completion order. Pinned until the cursor
+    /// reaches or passes them: for a cursor that must not fill the cache, the only copy.
+    arrived: Arc<Mutex<Vec<(u64, BlockHandle)>>>,
+    /// Fetches still in flight, in submission order (ascending: `upcoming` order): the first
+    /// and last block offsets each covers (one block, or a merged run of adjacent ones).
+    in_flight: VecDeque<(u64, u64, pigeonhole_io::Completion<()>)>,
+    /// Blocks submitted and not yet taken or dropped; zero skips the lock.
+    unclaimed: usize,
+}
+
+impl std::fmt::Debug for Ahead {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Ahead")
+            .field("in_flight", &self.in_flight.len())
+            .field("unclaimed", &self.unclaimed)
+            .finish()
+    }
+}
+
+fn lock<T>(m: &Mutex<T>) -> MutexGuard<'_, T> {
+    m.lock().unwrap_or_else(PoisonError::into_inner)
+}
+
+/// A readahead batch's fetches in runs of blocks adjacent on disk (with `merge`; else one
+/// block each), each run one read.
+fn runs_of(fetches: Vec<(BlockAddr, Fetch)>, merge: bool) -> Vec<Vec<(BlockAddr, Fetch)>> {
+    let mut runs: Vec<Vec<(BlockAddr, Fetch)>> = Vec::new();
+    for (addr, fetch) in fetches {
+        match runs.last_mut() {
+            Some(run)
+                if merge
+                    && run.last().is_some_and(|(prev, _)| {
+                        prev.offset + u64::from(prev.len) == addr.offset
+                    }) =>
+            {
+                run.push((addr, fetch));
+            }
+            _ => runs.push(vec![(addr, fetch)]),
+        }
+    }
+    runs
+}
+
+/// A run's bytes: its first block's offset and the length to the end of its last.
+fn span(run: &[(BlockAddr, Fetch)]) -> (u64, usize) {
+    let (first, last) = (run[0].0, run[run.len() - 1].0);
+    (
+        first.offset,
+        (last.offset + u64::from(last.len) - first.offset) as usize,
+    )
+}
+
+/// Admits a run's blocks into `arrived` when its read completes: a lone block takes the
+/// buffer as read, a longer run's blocks are copied out of it (one copy each). A failed read
+/// or block is dropped; the cursor reads it when it gets there.
+fn admit_run(
+    run: Vec<(BlockAddr, Fetch)>,
+    read: pigeonhole_io::Completion,
+    arrived: Arc<Mutex<Vec<(u64, BlockHandle)>>>,
+) -> pigeonhole_io::Completion<()> {
+    let first = run[0].0.offset;
+    read.map(move |read| {
+        let Ok(buf) = read else { return Ok(()) };
+        if let [(addr, fetch)] = run.as_slice() {
+            if let Ok(Some(h)) = fetch.admit(buf) {
+                lock(&arrived).push((addr.offset, h));
+            }
+            return Ok(());
+        }
+        for (addr, fetch) in &run {
+            let at = (addr.offset - first) as usize;
+            let mut piece = fetch.buffer(addr.len as usize);
+            piece.copy_from_slice(&buf[at..at + addr.len as usize]);
+            if let Ok(Some(h)) = fetch.admit(piece) {
+                lock(&arrived).push((addr.offset, h));
+            }
+        }
+        Ok(())
+    })
+}
+
+impl Ahead {
+    /// The block at `offset` if its readahead arrived, waiting for its fetch if it is still in
+    /// flight. Blocks before it are dropped: the cursor only moves forward between seeks.
+    fn take(&mut self, offset: u64) -> Option<BlockHandle> {
+        if self.unclaimed == 0 {
+            return None;
+        }
+        while self
+            .in_flight
+            .front()
+            .is_some_and(|(first, ..)| *first <= offset)
+        {
+            let (_, last, done) = self.in_flight.pop_front()?;
+            if offset <= last {
+                // Reached while still in flight: wait (its continuation admits it) rather
+                // than read it a second time.
+                if !done.is_ready() {
+                    crate::note_readahead(|c| c.waited += 1);
+                }
+                let _ = done.wait();
+            }
+            // A fetch the cursor passed completes on its own; its block is dropped below or
+            // at the next seek.
+        }
+        let mut arrived = lock(&self.arrived);
+        let mut found = None;
+        arrived.retain(|(o, h)| {
+            if *o == offset && found.is_none() {
+                found = Some(h.clone());
+            }
+            *o > offset
+        });
+        self.unclaimed = arrived.len() + self.in_flight.len();
+        if found.is_some() {
+            crate::note_readahead(|c| c.used += 1);
+        }
+        found
+    }
+
+    /// Forgets everything (a seek): fetches still in flight complete into a queue no cursor
+    /// reads any more.
+    fn clear(&mut self) {
+        if self.unclaimed != 0 || !self.in_flight.is_empty() {
+            self.arrived = Arc::default();
+            self.in_flight.clear();
+            self.unclaimed = 0;
+        }
+    }
+}
+
+/// Points `slot` at `block`, reusing the cursor (and its key buffer) if there is one.
+fn load(slot: &mut Option<BlockIter<BlockHandle>>, block: BlockHandle) -> Result<()> {
+    match slot {
+        Some(it) => it.reset(block)?,
+        None => *slot = Some(Block::new(block)?.into_cursor()),
+    }
+    Ok(())
+}
+
+/// Whether `sep` is a row-end separator: an escaped row followed by `00 02`, the bound past
+/// every key of that row (ICR 0039). An escaped row holds `00` only as `00 FF`.
+#[inline]
+fn row_end(sep: &[u8]) -> bool {
+    let mut i = 0;
+    while let Some(p) = sep[i..].iter().position(|&b| b == 0) {
+        let at = i + p;
+        match sep.get(at + 1) {
+            Some(0xFF) => i = at + 2,
+            Some(0x02) => return at + 2 == sep.len(),
+            _ => return false,
+        }
+    }
+    false
+}
+
+fn valid(slot: &Option<BlockIter<BlockHandle>>) -> bool {
+    slot.as_ref().is_some_and(|it| it.valid())
+}
+
+/// The loaded cursor in `slot`. Every caller loads it first, so `None` never happens; it is
+/// an error rather than a panic all the same.
+fn loaded(slot: &mut Option<BlockIter<BlockHandle>>) -> Result<&mut BlockIter<BlockHandle>> {
+    // Built only if it happens: an `Error` built eagerly (`ok_or`) is dropped on every call,
+    // and its drop is not free since `Error::WouldBlock` holds a box (ICR 0014).
+    slot.as_mut().ok_or_else(|| {
+        Error::Format(pigeonhole_format::Error::Corrupt {
+            what: "sst cursor state",
+        })
+    })
+}
+
+impl SstIter {
+    #[inline]
+    pub(crate) fn new(reader: Arc<SstReader>, filter: ScanFilter, options: ReadOptions) -> Self {
+        Self {
+            reader,
+            filter_all: filter.is_all(),
+            filter: filter.into_compiled(),
+            options,
+            top: None,
+            index: None,
+            data: None,
+            hint: Vec::new(),
+            row: Vec::new(),
+            ahead: None,
+            stepped: 0,
+            window: 0,
+        }
+    }
+
+    /// Switches cache-only reads on or off ([`ReadOptions::cache_only`]): an async scan
+    /// positions its cursors cache-only, then steps them reading normally (ICR 0014).
+    pub fn set_cache_only(&mut self, on: bool) {
+        self.options.cache_only = on;
+    }
+
+    /// The blocks the cursor reads next, past its current data block, that the cache does
+    /// not hold: up to `max` of the following data blocks its index names, in order, appended
+    /// to `out` as fetches. Cached blocks are skipped. The walk stops at the SST's end, or at
+    /// a next index partition that is not cached (its fetch is the last one appended: the
+    /// blocks after it cannot be named without it). Read-only (it peeks a copy of the index
+    /// cursor), and nothing if the cursor is not positioned.
+    ///
+    /// The one prediction of what a scan reads next (ICR 0017): an async scan fetches
+    /// `max = 1` ahead before each step (D196), and a sync scan's readahead hint
+    /// (#402) fetches more from its miss path.
+    pub fn upcoming(&self, max: usize, out: &mut Vec<crate::Fetch>) {
+        self.upcoming_with(max, true, false, |_, fetch| out.push(fetch));
+    }
+
+    /// Where this cursor's readahead ramp stands (#712), for [`SstIter::resume_readahead`].
+    pub fn readahead_ramp(&self) -> ReadaheadRamp {
+        ReadaheadRamp {
+            stepped: self.stepped,
+            window: self.window,
+        }
+    }
+
+    /// Resumes a ramp taken with [`SstIter::readahead_ramp`] (#712): for a level's cursor
+    /// that read one SST to its end and is now positioned at the next one's first entry. That
+    /// is the scan reading on, not a seek, so readahead keeps the size it had reached.
+    pub fn resume_readahead(&mut self, ramp: ReadaheadRamp) {
+        (self.stepped, self.window) = (ramp.stepped, ramp.window);
+    }
+
+    /// [`SstIter::upcoming`], each fetch handed to `f` with its block's address. With
+    /// `fill_cache` false the fetches hand their blocks out unshared. With `read_partitions`,
+    /// an uncached next index partition is read synchronously (into the cache as
+    /// `fill_cache` says) and the walk goes on through it, instead of ending at its fetch: the
+    /// sync readahead, whose scan reads that partition next anyway, then names all `max`
+    /// blocks even when partitions are small.
+    pub(crate) fn upcoming_with(
+        &self,
+        max: usize,
+        fill_cache: bool,
+        read_partitions: bool,
+        mut f: impl FnMut(BlockAddr, Fetch),
+    ) {
+        let blocks = &self.reader.inner.blocks;
+        let Some(index) = self.index.as_ref().filter(|i| i.valid()) else {
+            return;
+        };
+        let mut index = index.clone();
+        let mut top = self.top.clone();
+        for _ in 0..max {
+            if index.next().is_err() {
+                return;
+            }
+            if !index.valid() {
+                let Some(next) =
+                    self.next_partition(top.as_mut(), fill_cache, read_partitions, &mut f)
+                else {
+                    return;
+                };
+                index = next;
+            }
+            let Ok(addr) = BlockAddr::decode_varint(index.value()) else {
+                return;
+            };
+            if let Some(fetch) =
+                blocks.fetch_if_missing(addr, BlockKind::Data, self.options.priority, fill_cache)
+            {
+                f(addr, fetch);
+            }
+        }
+    }
+
+    /// The index partition after `top`'s, positioned on its first entry, with `top` moved
+    /// onto it. An uncached partition is read now with `read_partitions`; without, its fetch
+    /// goes to `f`. `None` ends the walk: no next partition, an empty one, a failure, or a
+    /// fetch handed to `f`.
+    fn next_partition(
+        &self,
+        top: Option<&mut BlockIter<BlockHandle>>,
+        fill_cache: bool,
+        read_partitions: bool,
+        f: &mut impl FnMut(BlockAddr, Fetch),
+    ) -> Option<BlockIter<BlockHandle>> {
+        let blocks = &self.reader.inner.blocks;
+        let priority = self.options.priority;
+        let top = top?;
+        top.next().ok()?;
+        if !top.valid() {
+            return None;
+        }
+        let addr = BlockAddr::decode_varint(top.value()).ok()?;
+        let h = match blocks.lookup(addr).ok()? {
+            Some(h) => h,
+            None if read_partitions => blocks
+                .read_uncached(addr, BlockKind::Index, fill_cache, priority)
+                .ok()?,
+            None => {
+                f(addr, blocks.fetch(addr, BlockKind::Index, priority));
+                return None;
+            }
+        };
+        let mut index = Block::new(h).ok()?.into_cursor();
+        index.seek_to_first().ok()?;
+        index.valid().then_some(index)
+    }
+
+    /// Whether the current data block is the SST's last: the cursor's next block is past the
+    /// SST (a level cursor moves on to its next one).
+    pub fn at_last_block(&self) -> bool {
+        let next = |slot: &Option<BlockIter<BlockHandle>>| {
+            slot.as_ref().is_some_and(|it| {
+                let mut peek = it.clone();
+                peek.next().is_ok() && peek.valid()
+            })
+        };
+        self.data.is_some() && !next(&self.index) && !next(&self.top)
+    }
+
+    /// The current value as a pinned [`Cell`] that outlives the cursor (no copy). Empty if
+    /// the cursor is not valid.
+    pub fn value_cell(&self) -> Cell {
+        match &self.data {
+            Some(it) if it.valid() => Cell::in_block(it.bytes().clone(), it.value_range()),
+            _ => Cell::owned(Vec::new()),
+        }
+    }
+
+    fn load_partition(&mut self) -> Result<()> {
+        let addr = BlockAddr::decode_varint(loaded(&mut self.top)?.value())?;
+        let h = match self.reader.inner.blocks.lookup(addr)? {
+            Some(h) => h,
+            None => self.load_missed(addr, BlockKind::Index)?,
+        };
+        load(&mut self.index, h)
+    }
+
+    /// Loads the data block the index cursor points at. `ahead` allows readahead (forward
+    /// scans only, never for seeks).
+    fn load_data(&mut self, ahead: bool) -> Result<()> {
+        let addr = BlockAddr::decode_varint(loaded(&mut self.index)?.value())?;
+        if ahead {
+            self.stepped = self.stepped.saturating_add(1);
+        }
+        let h = match self.ahead.as_mut().and_then(|a| a.take(addr.offset)) {
+            Some(h) => h,
+            None => match self.reader.inner.blocks.lookup(addr)? {
+                Some(h) => h,
+                None => {
+                    if ahead
+                        && self.options.readahead_blocks > 0
+                        && !self.options.cache_only
+                        && self.stepped > self.options.readahead_after
+                    {
+                        self.read_ahead();
+                    }
+                    self.load_missed(addr, BlockKind::Data)?
+                }
+            },
+        };
+        load(&mut self.data, h)
+    }
+
+    /// A block `lookup` did not find: read from the file, or, in cache-only mode, read in
+    /// place if that doesn't wait on the device, else the `Error::WouldBlock` that names it
+    /// with its read in flight (ICR 0014, #709). Out of line, so a hit never reads the mode.
+    #[cold]
+    #[inline(never)]
+    fn load_missed(&self, addr: BlockAddr, kind: BlockKind) -> Result<BlockHandle> {
+        let blocks = &self.reader.inner.blocks;
+        if self.options.cache_only {
+            return blocks.fetch_now(addr, kind, self.options.priority);
+        }
+        blocks.read_uncached(addr, kind, self.options.fill_cache, self.options.priority)
+    }
+
+    /// Submits fetches for up to the ramp's window (at most `readahead_blocks`) of the blocks
+    /// past the current one that the cache lacks ([`SstIter::upcoming`], the one prediction
+    /// of what a scan reads next), unless this cursor's previous ones are still in flight:
+    /// one batch at a time. A failed fetch is dropped; the block is read when the cursor
+    /// reaches it.
+    #[cold]
+    #[inline(never)]
+    fn read_ahead(&mut self) {
+        let ahead = self.ahead.get_or_insert_with(Box::default);
+        ahead.in_flight.retain(|(.., done)| !done.is_ready());
+        if !ahead.in_flight.is_empty() {
+            return;
+        }
+        let max = self.options.readahead_blocks;
+        let window = if self.window == 0 {
+            RAMP_FIRST.min(max)
+        } else {
+            self.window
+        };
+        self.window = window.saturating_mul(2).min(max);
+        let mut fetches = Vec::new();
+        self.upcoming_with(
+            window as usize,
+            self.options.fill_cache,
+            // A partition read for a cursor that does not fill the cache would be read again
+            // when the scan reaches it: such a walk ends there instead (partitions hold
+            // about 100 blocks at the default sizes, so that is rare).
+            self.options.fill_cache,
+            |addr, fetch| fetches.push((addr, fetch)),
+        );
+        let runs = runs_of(fetches, self.options.readahead_merge);
+        // Every run's read in one batch (ICR 0077): one submission, each read off this thread
+        // where the backend can (#974).
+        let ranges: Vec<(u64, usize)> = runs.iter().map(|run| span(run)).collect();
+        let submitted = self.reader.inner.blocks.submit_ahead(&ranges);
+        let Some(ahead) = self.ahead.as_mut() else {
+            return;
+        };
+        for (run, read) in runs.into_iter().zip(submitted) {
+            let (first, last) = (run[0].0.offset, run[run.len() - 1].0.offset);
+            crate::note_readahead(|c| c.issued += run.len() as u64);
+            ahead.unclaimed += run.len();
+            let done = admit_run(run, read, Arc::clone(&ahead.arrived));
+            ahead.in_flight.push_back((first, last, done));
+        }
+    }
+
+    /// Positions on the first entry `>= target` (or the first entry), ignoring the filter.
+    fn position(&mut self, target: Option<&[u8]>) -> Result<()> {
+        // Readahead is only useful in front of a forward scan; a seek (possibly backwards)
+        // drops its pins.
+        if let Some(ahead) = self.ahead.as_mut() {
+            ahead.clear();
+        }
+        (self.stepped, self.window) = (0, 0);
+        if self.top.is_none() {
+            load(&mut self.top, self.reader.inner.top.clone())?;
+        }
+        let top = loaded(&mut self.top)?;
+        match target {
+            Some(t) => top.seek(t)?,
+            None => top.seek_to_first()?,
+        }
+        loop {
+            if !valid(&self.top) {
+                self.data = None;
+                return Ok(());
+            }
+            self.load_partition()?;
+            let index = loaded(&mut self.index)?;
+            match target {
+                Some(t) => index.seek(t)?,
+                None => index.seek_to_first()?,
+            }
+            if index.valid() {
+                break;
+            }
+            loaded(&mut self.top)?.next()?;
+        }
+        self.load_data(false)?;
+        let data = loaded(&mut self.data)?;
+        match target {
+            Some(t) => data.seek(t)?,
+            None => data.seek_to_first()?,
+        }
+        if !data.valid() {
+            // A separator may sort above the block's last key; the answer starts the next.
+            self.advance_block(false)?;
+        }
+        Ok(())
+    }
+
+    /// Moves to the first entry of the next data block. With `may_end` (the cursor has
+    /// reached its read's row: every key it passed sorts at or after that row's start) and a
+    /// one-row read ([`ReadOptions::one_row`]), it ends instead when the block it leaves has
+    /// a row-end separator: every key of a later block sorts after the separator, so past
+    /// the row of that block's last key, and so past the read's row (ICR 0039).
+    fn advance_block(&mut self, may_end: bool) -> Result<()> {
+        if may_end
+            && self.options.one_row
+            && self
+                .index
+                .as_ref()
+                .is_some_and(|i| i.valid() && row_end(i.key()))
+        {
+            self.data = None;
+            return Ok(());
+        }
+        loop {
+            loaded(&mut self.index)?.next()?;
+            while !valid(&self.index) {
+                loaded(&mut self.top)?.next()?;
+                if !valid(&self.top) {
+                    self.data = None;
+                    return Ok(());
+                }
+                self.load_partition()?;
+                loaded(&mut self.index)?.seek_to_first()?;
+            }
+            self.load_data(true)?;
+            let data = loaded(&mut self.data)?;
+            data.seek_to_first()?;
+            if data.valid() {
+                return Ok(());
+            }
+        }
+    }
+
+    fn next_raw(&mut self) -> Result<()> {
+        let Some(data) = self.data.as_mut() else {
+            return Ok(());
+        };
+        data.next()?;
+        if !data.valid() {
+            self.advance_block(true)?;
+        }
+        Ok(())
+    }
+
+    fn seek_forward(&mut self, target: &[u8]) -> Result<()> {
+        let data = loaded(&mut self.data)?;
+        data.seek_forward(target)?;
+        if !data.valid() {
+            self.seek_past_block(target)?;
+        }
+        Ok(())
+    }
+
+    /// Positions on the first entry `>= target`, which lies past the current data block
+    /// (#29): the index partition is walked forward from the current block, past every block
+    /// whose separator is below `target` (none of them is read), and only a target past the
+    /// partition goes back to the top index. Readahead is kept: it drops the blocks a forward
+    /// move passes.
+    fn seek_past_block(&mut self, target: &[u8]) -> Result<()> {
+        // Plain seeks, not forward ones: the target is usually blocks away, where a binary
+        // search over the partition (or the new block) is the cheapest way there.
+        let index = loaded(&mut self.index)?;
+        index.next()?;
+        if index.valid() && index.key() < target {
+            index.seek(target)?;
+        }
+        if !index.valid() {
+            return self.position(Some(target));
+        }
+        self.load_data(true)?;
+        let data = loaded(&mut self.data)?;
+        data.seek(target)?;
+        if !data.valid() {
+            // A separator may sort above the block's last key; the answer starts the next.
+            self.advance_block(true)?;
+        }
+        Ok(())
+    }
+
+    fn skip_row_raw(&mut self) -> Result<()> {
+        let Some(data) = self.data.as_mut().filter(|d| d.valid()) else {
+            return Ok(());
+        };
+        let Ok(n) = row_prefix_len(data.key()) else {
+            return self.next_raw();
+        };
+        self.row.clear();
+        self.row.extend_from_slice(&data.key()[..n]);
+        data.skip_row()?;
+        if data.valid() {
+            return Ok(());
+        }
+        // The rest of the block is this row's. `escaped row ++ 00 02` sorts after every key
+        // of the row and before every later row (a later row continues with `00 FF` or a
+        // byte >= 01 at that point, or differs earlier): the blocks the row fills are passed
+        // through the index, unread.
+        let mut row = std::mem::take(&mut self.row);
+        if let Some(last) = row.last_mut() {
+            *last = 0x02;
+        }
+        let r = self.seek_past_block(&row);
+        self.row = row;
+        r
+    }
+
+    /// Advances to the first admitted entry at or after the current one.
+    fn settle(&mut self) -> Result<()> {
+        if self.filter_all {
+            return Ok(());
+        }
+        while let Some(data) = self.data.as_ref().filter(|d| d.valid()) {
+            match self.filter.check(data.key(), &mut self.hint) {
+                Verdict::Admit => return Ok(()),
+                Verdict::SkipRow => self.skip_row_raw()?,
+                Verdict::Seek => {
+                    let hint = std::mem::take(&mut self.hint);
+                    let r = self.seek_forward(&hint);
+                    self.hint = hint;
+                    r?;
+                }
+                Verdict::Next => self.next_raw()?,
+            }
+        }
+        Ok(())
+    }
+
+    /// Runs a move and settles; on error the cursor becomes invalid.
+    fn run(&mut self, f: impl FnOnce(&mut Self) -> Result<()>) -> Result<()> {
+        let r = f(self).and_then(|()| self.settle());
+        if r.is_err() {
+            self.data = None;
+        }
+        r
+    }
+}
+
+impl Cursor for SstIter {
+    type Error = Error;
+
+    fn valid(&self) -> bool {
+        valid(&self.data)
+    }
+
+    fn key(&self) -> &[u8] {
+        self.data.as_ref().map_or(&[], |d| d.key())
+    }
+
+    fn value(&self) -> &[u8] {
+        self.data.as_ref().map_or(&[], |d| d.value())
+    }
+
+    fn seek_to_first(&mut self) -> Result<()> {
+        self.run(|it| it.position(None))
+    }
+
+    fn seek(&mut self, target: &[u8]) -> Result<()> {
+        self.run(|it| it.position(Some(target)))
+    }
+
+    fn next(&mut self) -> Result<()> {
+        let Some(data) = self.data.as_mut().filter(|d| d.valid()) else {
+            return Ok(());
+        };
+        // The usual step: the next entry of the same block, no filter to apply.
+        match data.next() {
+            Ok(()) if data.valid() && self.filter_all => Ok(()),
+            Ok(()) => self.run(|it| {
+                if valid(&it.data) {
+                    Ok(())
+                } else {
+                    it.advance_block(true)
+                }
+            }),
+            Err(e) => {
+                self.data = None;
+                Err(e.into())
+            }
+        }
+    }
+
+    /// Passes the column's later entries in the current block from their headers
+    /// ([`BlockIter`]'s `skip_column`, ICR 0041); at the block's end it moves to the next
+    /// block, as `next` does, where the column may continue. Like `next`, a one-row read
+    /// ends there instead when the block's separator ends its row: the cursor is on the
+    /// read's row, so every key it passes sorts at or after it (ICR 0039).
+    #[inline]
+    fn skip_column(&mut self, column: &[u8]) -> Result<bool> {
+        let Some(data) = self.data.as_mut().filter(|d| d.valid()) else {
+            return Ok(false);
+        };
+        match data.skip_column(column) {
+            Ok(_) if data.valid() && self.filter_all => Ok(true),
+            Ok(_) => self
+                .run(|it| {
+                    if valid(&it.data) {
+                        Ok(())
+                    } else {
+                        it.advance_block(true)
+                    }
+                })
+                .map(|()| true),
+            Err(e) => {
+                self.data = None;
+                Err(e.into())
+            }
+        }
+    }
+
+    /// Uses the block's row-start table; crosses into the next block only if the row
+    /// continues there.
+    fn skip_row(&mut self) -> Result<()> {
+        if !self.valid() {
+            return Ok(());
+        }
+        self.run(Self::skip_row_raw)
+    }
+}