pigeonhole-db 1.1.0

data/vendor/engine/src/read/keys.rs

@@ -0,0 +1,1049 @@
+//! Frontier multi-get (D299, #489): many rows read in key order under one snapshot, with the
+//! read set up once per tablet rather than once per row.
+//!
+//! A [`KeySet`] is sorted and distinct. [`key_groups`] cuts it into runs of keys that route
+//! to the same tablet. For each run, per family, a [`KeyCursor`] keeps its memtable sources
+//! and one SST iterator per SST it has read from, and reuses them for every key of the run:
+//!
+//! - an SST takes part in a key's read only if the row is within its key range and its row
+//!   filter may hold the row, as for a point row read (`sst_sources_row`);
+//! - a source that takes part moves forward to the row: `Cursor::seek_forward` from where the
+//!   previous key left it (usually inside the block it already decoded), a full seek if it is
+//!   new or ran past its end;
+//! - the merge is rebuilt over just those sources ([`MergingCursor::rebuild_positioned`]),
+//!   and the family's resolver reads the row as `read_row_with` does.
+//!
+//! The scan's and the row read's own code are untouched (D296's R3): this is a sibling that
+//! drives the same [`Lane`]s.
+//!
+//! An async multi-get ([`KeyCursor::poll_next_key`]) positions each key cache-only, as an
+//! async scan positions a tablet (D196): a miss unwinds the key's positioning and fetches the
+//! block before positioning it again. Before it does, it fetches the blocks of the next
+//! 8 keys ([`LOOKAHEAD`]) that the cache lacks, found from the index alone
+//! ([`SstReader::fetch_for`]), so a cold frontier's reads are in flight together rather than
+//! one per key.
+//!
+//! [`MergingCursor::rebuild_positioned`]: pigeonhole_compaction::MergingCursor::rebuild_positioned
+
+use std::ops::Range;
+use std::sync::Arc;
+use std::sync::atomic::Ordering;
+use std::task::{Context, Poll};
+
+use pigeonhole_format::Cursor;
+use pigeonhole_format::filter::row_hash;
+use pigeonhole_format::key::{Escaped, TERMINATOR};
+use pigeonhole_format::scan::ScanFilter;
+use pigeonhole_format::{FamilyId, TableId, TabletId, Timestamp};
+use pigeonhole_runtime::ShardId;
+use pigeonhole_sst::SstReader;
+
+use super::{
+    CellData, FORK_CHECK_ROWS, Lane, MergeKind, ReadSpec, ResolverBlobs, RowSink, ScanCell,
+    as_async_read, give_back, skips_columns, take_scan_resolver,
+};
+use crate::snapshot::{FamilySsts, OpenSst, Snapshot, SstSet};
+use crate::source::{Source, mem_sources, read_options, row_prefix};
+use crate::{Error, Result};
+
+/// A sorted set of distinct row keys in one flat buffer: the input of a multi-get (D299) and
+/// of a bounded read's `Keys` input (D296).
+///
+/// ```
+/// use pigeonhole_engine::KeySet;
+///
+/// let keys = KeySet::from_unsorted([&b"c"[..], b"a", b"c", b"b"]);
+/// assert_eq!(keys.len(), 3);
+/// assert_eq!(keys.iter().collect::<Vec<_>>(), [&b"a"[..], b"b", b"c"]);
+/// assert!(KeySet::from_sorted([&b"b"[..], b"a"]).is_err());
+/// ```
+#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)]
+pub struct KeySet {
+    bytes: Vec<u8>,
+    /// Where each key ends in `bytes`.
+    ends: Vec<usize>,
+}
+
+impl KeySet {
+    /// The set of `keys`, sorted and deduplicated.
+    pub fn from_unsorted<K: AsRef<[u8]>>(keys: impl IntoIterator<Item = K>) -> Self {
+        let mut flat = Vec::new();
+        let mut spans: Vec<Range<usize>> = Vec::new();
+        for k in keys {
+            let start = flat.len();
+            flat.extend_from_slice(k.as_ref());
+            spans.push(start..flat.len());
+        }
+        spans.sort_unstable_by(|a, b| flat[a.clone()].cmp(&flat[b.clone()]));
+        spans.dedup_by(|a, b| flat[a.clone()] == flat[b.clone()]);
+        let mut out = KeySet {
+            bytes: Vec::with_capacity(spans.iter().map(|s| s.len()).sum()),
+            ends: Vec::with_capacity(spans.len()),
+        };
+        for s in spans {
+            out.bytes.extend_from_slice(&flat[s]);
+            out.ends.push(out.bytes.len());
+        }
+        out
+    }
+
+    /// The set of `keys`, which must already be strictly increasing (`InvalidArgument`
+    /// otherwise).
+    pub fn from_sorted<K: AsRef<[u8]>>(keys: impl IntoIterator<Item = K>) -> Result<Self> {
+        let mut out = KeySet::default();
+        for k in keys {
+            let k = k.as_ref();
+            if !out.is_empty() && out.get(out.len() - 1) >= k {
+                return Err(Error::InvalidArgument(
+                    "a key set must be strictly increasing (KeySet::from_unsorted sorts)"
+                        .to_owned(),
+                ));
+            }
+            out.bytes.extend_from_slice(k);
+            out.ends.push(out.bytes.len());
+        }
+        Ok(out)
+    }
+
+    /// The number of keys.
+    pub fn len(&self) -> usize {
+        self.ends.len()
+    }
+
+    /// Whether there are none.
+    pub fn is_empty(&self) -> bool {
+        self.ends.is_empty()
+    }
+
+    /// The `i`th key (panics past the end).
+    pub fn get(&self, i: usize) -> &[u8] {
+        let start = if i == 0 { 0 } else { self.ends[i - 1] };
+        &self.bytes[start..self.ends[i]]
+    }
+
+    /// The keys, in order.
+    pub fn iter(&self) -> impl Iterator<Item = &[u8]> + '_ {
+        (0..self.len()).map(|i| self.get(i))
+    }
+}
+
+/// A run of keys that route to one tablet in a view: `keys` indexes the [`KeySet`].
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(crate) struct KeyGroup {
+    pub(crate) tablet: TabletId,
+    pub(crate) shard: ShardId,
+    pub(crate) keys: Range<usize>,
+}
+
+/// Cuts `keys` into runs of keys that route to the same tablet of `table` in `snapshot`'s
+/// view, in key order.
+pub(crate) fn key_groups(
+    snapshot: &Snapshot,
+    table: TableId,
+    keys: &KeySet,
+) -> Result<Vec<KeyGroup>> {
+    let mut out: Vec<KeyGroup> = Vec::new();
+    for i in 0..keys.len() {
+        let (tablet, shard) = snapshot
+            .view
+            .tablets()
+            .route(table, keys.get(i))
+            .ok_or_else(|| Error::TableNotFound(format!("table {}", table.0)))?;
+        match out.last_mut() {
+            Some(g) if g.tablet == tablet => g.keys.end = i + 1,
+            _ => out.push(KeyGroup {
+                tablet,
+                shard,
+                keys: i..i + 1,
+            }),
+        }
+    }
+    Ok(out)
+}
+
+/// One SST a family's read in the current group may use, with its reader and iterator kept
+/// from key to key.
+struct SstSlot {
+    sst: Arc<OpenSst>,
+    reader: Option<Arc<SstReader>>,
+    source: Option<Source>,
+}
+
+impl SstSlot {
+    /// The SST's reader, opened on first use (from the cache only, for an async attempt).
+    fn reader(
+        &mut self,
+        set: &SstSet,
+        priority: pigeonhole_cache::Priority,
+        cache_only: bool,
+    ) -> Result<Arc<SstReader>> {
+        if let Some(r) = &self.reader {
+            return Ok(Arc::clone(r));
+        }
+        let r = if cache_only {
+            self.sst.reader_cache_only(set, priority)?
+        } else {
+            self.sst.reader(set, priority)?
+        };
+        self.reader = Some(Arc::clone(&r));
+        Ok(r)
+    }
+}
+
+/// Calls `f` on each SST of a family's read whose key range holds the row `prefix`, by
+/// level, newest first (the row filter is `f`'s to check).
+fn each_sst(
+    fam: Option<&FamilySsts>,
+    levels: &[Range<usize>],
+    ssts: &mut [SstSlot],
+    prefix: &[u8],
+    mut f: impl FnMut(usize, &mut SstSlot) -> Result<()>,
+) -> Result<()> {
+    let Some(fam) = fam else {
+        return Ok(());
+    };
+    for (l, range) in levels.iter().enumerate() {
+        let (from, search) = fam.level_start(l, prefix);
+        for (i, slot) in ssts
+            .iter_mut()
+            .enumerate()
+            .take(range.end)
+            .skip(range.start + from)
+        {
+            if search && slot.sst.first_row() > prefix {
+                break;
+            }
+            if slot.sst.first_row() <= prefix && prefix <= slot.sst.last_row() {
+                f(i, slot)?;
+            }
+        }
+    }
+    Ok(())
+}
+
+/// Where a source in a lane's merge came from, to put it back after the key.
+#[derive(Clone, Copy)]
+enum Slot {
+    Mem(usize),
+    Sst(usize),
+}
+
+/// A family's read within one group: its lane (resolver and held cell) and the sources it
+/// keeps across keys.
+struct FamilyRead {
+    lane: Lane,
+    filter: ScanFilter,
+    opts: pigeonhole_compaction::ResolveOptions,
+    priority: pigeonhole_cache::Priority,
+    mems: Vec<Option<Source>>,
+    /// The family's SSTs in the tablet, by level (level 0 newest first), flattened; `levels`
+    /// gives each level's range.
+    ssts: Vec<SstSlot>,
+    levels: Vec<Range<usize>>,
+    /// Per level, `FamilySsts::level_start`'s answer needs the level's files: kept here.
+    fam: Option<Arc<crate::snapshot::FamilySsts>>,
+    /// The sources in the lane's merge for the current key, in order.
+    in_merge: Vec<Slot>,
+    /// An async attempt at a key was unwound (a cache miss): its sources may have moved past
+    /// the row's start, so the next positioning seeks every source afresh.
+    reseek: bool,
+}
+
+/// A multi-get over a snapshot (D299): positioned on one key at a time ([`KeyCursor::next_key`]),
+/// its cells handed out one at a time like a scan's, by family, then qualifier, then newest
+/// version first.
+pub struct KeyCursor {
+    snapshot: Snapshot,
+    families: Vec<FamilyId>,
+    spec: ReadSpec,
+    now: Timestamp,
+    keys: Arc<KeySet>,
+    /// One past the last key this cursor reads: the key set's length, or a group's end
+    /// ([`KeyCursor::for_group`]).
+    end: usize,
+    groups: Vec<KeyGroup>,
+    group: usize,
+    /// The next key to read.
+    next: usize,
+    /// The key the cursor is on, if any.
+    current: Option<usize>,
+    reads: Vec<FamilyRead>,
+    /// The current row, escaped with its terminator (a row prefix), and unescaped.
+    prefix: Vec<u8>,
+    /// The first key past the current row (the resolvers' upper bound).
+    past: Vec<u8>,
+    row_esc_len: usize,
+    lane_idx: usize,
+    last_lane: Option<usize>,
+    qual_buf: Vec<u8>,
+    rows_read: u64,
+    /// The group whose reads are open.
+    opened: Option<usize>,
+    shared: Arc<crate::shard::Shared>,
+    /// An async multi-get's state ([`KeyCursor::poll_next_key`]), made on its first poll.
+    nb: Option<Box<AsyncKeys>>,
+}
+
+/// Keys past the one being positioned whose uncached blocks an async multi-get fetches
+/// ahead ([`SstReader::fetch_for`]).
+pub(crate) const LOOKAHEAD: usize = 8;
+
+/// Keys with no cell to read that one poll of an async multi-get passes over before it
+/// yields (#1237): a run of absent keys never holds the executor for long.
+pub const POLL_ABSENT_KEYS: u32 = 4_096;
+
+/// Fetches an async multi-get makes positioning one key before it reads synchronously.
+const MAX_KEY_FETCHES: u32 = 64;
+
+/// What an async multi-get keeps between polls.
+#[derive(Default)]
+struct AsyncKeys {
+    /// Reads in flight, each with the first key that needs it.
+    in_flight: Vec<(usize, pigeonhole_io::Completion, Box<pigeonhole_sst::Fetch>)>,
+    /// Blocks fetched for keys not positioned yet, with their key.
+    pinned: Vec<(usize, pigeonhole_cache::BlockHandle)>,
+    /// Keys below this one have been probed for the blocks they need.
+    probed: usize,
+    /// Fetches while positioning the current key.
+    fetches: u32,
+    /// The cache keeps nothing, or a key needed too many fetches: read synchronously from now
+    /// on (counted).
+    sync: bool,
+}
+
+/// Where positioning on one key left the cursor.
+enum Step {
+    /// On the key with this index.
+    Key(usize),
+    /// The key has no cell to read.
+    Absent,
+    /// Past the last key.
+    End,
+}
+
+impl std::fmt::Debug for KeyCursor {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("KeyCursor")
+            .field("keys", &self.keys.len())
+            .field("next", &self.next)
+            .finish_non_exhaustive()
+    }
+}
+
+impl KeyCursor {
+    pub(crate) fn new(
+        snapshot: Snapshot,
+        table: TableId,
+        families: Vec<FamilyId>,
+        spec: ReadSpec,
+        now: Timestamp,
+        keys: KeySet,
+        shared: Arc<crate::shard::Shared>,
+    ) -> Result<Self> {
+        let groups = key_groups(&snapshot, table, &keys)?;
+        let end = keys.len();
+        Ok(Self::with_groups(
+            snapshot,
+            families,
+            spec,
+            now,
+            Arc::new(keys),
+            groups,
+            (0, end),
+            shared,
+        ))
+    }
+
+    /// [`KeyCursor::new`] over a shared key set (D304's async traversal reads each hop's
+    /// frontier, which it also keeps).
+    pub(crate) fn over(
+        snapshot: Snapshot,
+        table: TableId,
+        families: Vec<FamilyId>,
+        spec: ReadSpec,
+        now: Timestamp,
+        keys: Arc<KeySet>,
+        shared: Arc<crate::shard::Shared>,
+    ) -> Result<Self> {
+        let groups = key_groups(&snapshot, table, &keys)?;
+        let end = keys.len();
+        Ok(Self::with_groups(
+            snapshot,
+            families,
+            spec,
+            now,
+            keys,
+            groups,
+            (0, end),
+            shared,
+        ))
+    }
+
+    /// A cursor over one key group of `keys` (D296's `Keys` input: each tablet group of a
+    /// bounded read gets its own cursor, all sharing one key set).
+    pub(crate) fn for_group(
+        snapshot: Snapshot,
+        families: Vec<FamilyId>,
+        spec: ReadSpec,
+        now: Timestamp,
+        keys: Arc<KeySet>,
+        group: KeyGroup,
+        shared: Arc<crate::shard::Shared>,
+    ) -> Self {
+        let range = (group.keys.start, group.keys.end);
+        Self::with_groups(
+            snapshot,
+            families,
+            spec,
+            now,
+            keys,
+            vec![group],
+            range,
+            shared,
+        )
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn with_groups(
+        snapshot: Snapshot,
+        families: Vec<FamilyId>,
+        spec: ReadSpec,
+        now: Timestamp,
+        keys: Arc<KeySet>,
+        groups: Vec<KeyGroup>,
+        (next, end): (usize, usize),
+        shared: Arc<crate::shard::Shared>,
+    ) -> Self {
+        Self {
+            snapshot,
+            families,
+            spec,
+            now,
+            keys,
+            end,
+            groups,
+            group: 0,
+            next,
+            current: None,
+            reads: Vec::new(),
+            prefix: Vec::new(),
+            past: Vec::new(),
+            row_esc_len: 0,
+            lane_idx: 0,
+            last_lane: None,
+            qual_buf: Vec::new(),
+            rows_read: 0,
+            opened: None,
+            shared,
+            nb: None,
+        }
+    }
+
+    /// The key set.
+    pub fn keys(&self) -> &KeySet {
+        &self.keys
+    }
+
+    /// Opens the reads of group `g` (its tablet), one per family with data there.
+    fn open_group(&mut self, g: usize) -> Result<()> {
+        self.close_reads();
+        let (tablet, shard) = (self.groups[g].tablet, self.groups[g].shard);
+        let view = Arc::clone(&self.snapshot.view);
+        for &family in &self.families {
+            let located = view.locate(shard, tablet, family);
+            if !located.has_data() {
+                continue;
+            }
+            let Some(meta) = view.catalog.family(family) else {
+                continue;
+            };
+            let (mut opts, filter) = self.spec.resolve_opts(meta, self.snapshot.seqno, self.now);
+            let resolver_blobs = ResolverBlobs::attach(&mut opts, &view.ssts);
+            let mut mems = Vec::new();
+            if let Some(set) = located.mems {
+                let mut out = Vec::new();
+                mem_sources(set, &filter, &mut out);
+                mems.extend(out.into_iter().map(Some));
+            }
+            let (mut ssts, mut levels) = (Vec::new(), Vec::new());
+            let fam = located.ssts.cloned();
+            if let Some(f) = &fam {
+                for files in &f.levels {
+                    let start = ssts.len();
+                    ssts.extend(files.iter().map(|sst| SstSlot {
+                        sst: Arc::clone(sst),
+                        reader: None,
+                        source: None,
+                    }));
+                    levels.push(start..ssts.len());
+                }
+            }
+            self.reads.push(FamilyRead {
+                lane: Lane {
+                    family,
+                    unknown_merge: (meta.merge == MergeKind::Unknown)
+                        .then(|| meta.options.merge_operator.clone()),
+                    resolver: take_scan_resolver(),
+                    grew: false,
+                    ssts: Arc::clone(&view.ssts),
+                    resolver_blobs,
+                    ts: 0,
+                    pinned: None,
+                    pending: false,
+                    done: true,
+                },
+                filter,
+                opts,
+                priority: located.priority,
+                mems,
+                ssts,
+                levels,
+                fam,
+                in_merge: Vec::new(),
+                reseek: false,
+            });
+        }
+        self.opened = Some(g);
+        Ok(())
+    }
+
+    /// Ends the current group's reads, giving their resolvers back.
+    fn close_reads(&mut self) {
+        self.opened = None;
+        for mut r in self.reads.drain(..) {
+            r.lane.resolver.cursor_mut().sources_mut().clear();
+            if !r.lane.grew {
+                give_back(r.lane.resolver);
+            }
+        }
+    }
+
+    /// Positions on the next key with a cell to read, and returns its index in the key set;
+    /// `None` after the last.
+    pub fn next_key(&mut self) -> Result<Option<usize>> {
+        self.current = None;
+        loop {
+            match self.step(false)? {
+                Step::Key(i) => return Ok(Some(i)),
+                Step::Absent => {}
+                Step::End => return Ok(None),
+            }
+        }
+    }
+
+    /// The next key of an async multi-get: `Ready(Ok(Some(i)))` when the cursor is on key
+    /// `i` (read its cells inside [`KeyCursor::counted`]), `Ready(Ok(None))` after the last.
+    ///
+    /// Each key is positioned cache-only: what its sources' seeks and first cells miss is
+    /// fetched asynchronously, and the key positioned again from scratch. First, the blocks
+    /// the next 8 keys' seeks read that the cache lacks are fetched, all in flight
+    /// together. A key's further cells are read synchronously, and the blocks read so are
+    /// counted in `Metrics::async_sync_reads`. When the cache keeps nothing, or a key needs
+    /// more than 64 fetches, the rest is read synchronously (counted), as an async scan does
+    /// (D196).
+    pub fn poll_next_key(&mut self, cx: &mut Context<'_>) -> Poll<Result<Option<usize>>> {
+        let _poll = pigeonhole_io::async_poll_scope();
+        let mut nb = self.nb.take().unwrap_or_default();
+        self.current = None;
+        let mut absent = 0;
+        let out = loop {
+            if let Err(e) = admit_arrived(&mut nb, &self.shared, cx) {
+                break Err(e);
+            }
+            if nb.sync {
+                break self.counted(Self::next_key);
+            }
+            let i = self.next;
+            if i < self.end {
+                let entered = self.enter(i);
+                if let Err(e) = self.snapshot.checked(entered) {
+                    break Err(e);
+                }
+                if self.look_ahead(&mut nb, i) {
+                    // Poll the new reads before waiting on them.
+                    continue;
+                }
+                if nb.in_flight.iter().any(|f| f.0 <= i) {
+                    // Invariant: the wakers are registered last. Poll the reads waited on
+                    // once more, as the last thing before `Pending`, as a scan or a get
+                    // returns right after its one poll. The work since the first poll takes
+                    // the block cache's shard locks, and a contended std `RwLock` waits with
+                    // `Thread::park` where std has no futex (macOS and other non-Linux Unix,
+                    // its queue lock), using up an unpark a read's waker delivered meanwhile:
+                    // an executor that parks its thread on `Pending` then sleeps for ever
+                    // (#1226, seen with deferred I/O in the model test).
+                    let waiting = nb.in_flight.len();
+                    if let Err(e) = admit_arrived(&mut nb, &self.shared, cx) {
+                        break Err(e);
+                    }
+                    if nb.in_flight.len() < waiting {
+                        continue;
+                    }
+                    self.nb = Some(nb);
+                    return Poll::Pending;
+                }
+            }
+            let (stepped, sync_reads) = as_async_read(true, || self.step(true));
+            if sync_reads > 0 {
+                self.shared
+                    .async_sync_reads
+                    .fetch_add(sync_reads, Ordering::Relaxed);
+            }
+            match stepped {
+                Ok(Step::Key(i)) => {
+                    nb.fetches = 0;
+                    nb.pinned.retain(|(k, _)| *k > i);
+                    for r in &mut self.reads {
+                        for source in r.lane.resolver.cursor_mut().sources_mut() {
+                            source.set_cache_only(false);
+                        }
+                    }
+                    break Ok(Some(i));
+                }
+                Ok(Step::Absent) => {
+                    nb.fetches = 0;
+                    nb.pinned.retain(|(k, _)| *k > i);
+                    absent += 1;
+                    if absent >= POLL_ABSENT_KEYS {
+                        // Many keys in a row with nothing to read: yield rather than walk
+                        // them all in one poll (#1237); the next poll goes on from here.
+                        cx.waker().wake_by_ref();
+                        self.nb = Some(nb);
+                        return Poll::Pending;
+                    }
+                }
+                Ok(Step::End) => break Ok(None),
+                Err(Error::WouldBlock(mut fetch)) => {
+                    nb.fetches += 1;
+                    if nb.fetches > MAX_KEY_FETCHES {
+                        nb.sync = true;
+                        self.shared.async_sync_reads.fetch_add(1, Ordering::Relaxed);
+                    } else {
+                        let completion = fetch.start();
+                        nb.in_flight.push((i, completion, fetch));
+                    }
+                }
+                Err(e) => break Err(e),
+            }
+        };
+        self.nb = Some(nb);
+        Poll::Ready(out)
+    }
+
+    /// Runs `f` on this cursor counting the file reads it makes synchronously into
+    /// `Metrics::async_sync_reads` (an async multi-get reading a key's cells after
+    /// [`KeyCursor::poll_next_key`]); on a cursor never polled it just runs `f`.
+    pub fn counted<T>(&mut self, f: impl FnOnce(&mut Self) -> T) -> T {
+        if self.nb.is_none() {
+            return f(self);
+        }
+        let ((out, blob_reads), file_reads) =
+            pigeonhole_sst::counting_file_reads(|| as_async_read(false, || f(self)));
+        if blob_reads + file_reads > 0 {
+            self.shared
+                .async_sync_reads
+                .fetch_add(blob_reads + file_reads, Ordering::Relaxed);
+        }
+        out
+    }
+
+    /// Moves to key `i`'s group, opening its reads if they are not open.
+    fn enter(&mut self, i: usize) -> Result<()> {
+        while self.groups[self.group].keys.end <= i {
+            self.group += 1;
+        }
+        if self.opened != Some(self.group) {
+            self.open_group(self.group)?;
+        }
+        Ok(())
+    }
+
+    /// Positions on the next key, from the cache only if `cache_only`; a miss there unwinds
+    /// the attempt (the key is read again, every source seeking afresh) and is returned as
+    /// `Error::WouldBlock`.
+    fn step(&mut self, cache_only: bool) -> Result<Step> {
+        let i = self.next;
+        if i >= self.end {
+            self.close_reads();
+            return Ok(Step::End);
+        }
+        let entered = self.enter(i);
+        self.snapshot.checked(entered)?;
+        self.rows_read += 1;
+        if self.rows_read.is_multiple_of(FORK_CHECK_ROWS) {
+            self.snapshot.check_process()?;
+        }
+        match self.position(i, cache_only) {
+            Ok(any) => {
+                self.next = i + 1;
+                if !any {
+                    return Ok(Step::Absent);
+                }
+                self.current = Some(i);
+                self.lane_idx = 0;
+                self.last_lane = None;
+                Ok(Step::Key(i))
+            }
+            Err(e @ Error::WouldBlock(_)) if cache_only => {
+                for r in &mut self.reads {
+                    r.unwind();
+                }
+                Err(e)
+            }
+            Err(e) => {
+                self.next = i + 1;
+                self.snapshot.checked(Err(e))
+            }
+        }
+    }
+
+    /// Fetches the blocks the seeks of keys `i..i + LOOKAHEAD` (within `i`'s group) read
+    /// that the cache lacks and no read in flight brings, each key probed once (again after
+    /// a fetch lands: an index partition fetched names the data block next). Whether it
+    /// submitted any.
+    fn look_ahead(&mut self, nb: &mut AsyncKeys, i: usize) -> bool {
+        let end = (i + LOOKAHEAD).min(self.groups[self.group].keys.end);
+        let from = nb.probed.max(i);
+        let before = nb.in_flight.len();
+        for k in from..end {
+            let Ok(prefix) = row_prefix(self.keys.get(k)) else {
+                continue;
+            };
+            let hash = row_hash(&prefix[..prefix.len() - TERMINATOR.len()]);
+            for r in &mut self.reads {
+                r.probe(&prefix, hash, |mut fetch| {
+                    if !nb.in_flight.iter().any(|f| f.2.offset() == fetch.offset()) {
+                        let completion = fetch.start();
+                        nb.in_flight.push((k, completion, Box::new(fetch)));
+                    }
+                });
+            }
+        }
+        nb.probed = nb.probed.max(end);
+        nb.in_flight.len() > before
+    }
+
+    /// Positions every family's read on key `i`: whether any has a cell in its row.
+    fn position(&mut self, i: usize, cache_only: bool) -> Result<bool> {
+        let prefix = row_prefix(self.keys.get(i))?;
+        self.prefix.clear();
+        self.prefix.extend_from_slice(&prefix);
+        self.row_esc_len = prefix.len() - TERMINATOR.len();
+        let hash = row_hash(&self.prefix[..self.row_esc_len]);
+        self.past.clear();
+        self.past.extend_from_slice(&self.prefix);
+        if let Some(last) = self.past.last_mut() {
+            *last += 1;
+        }
+        let mut any = false;
+        for r in &mut self.reads {
+            r.position(&self.prefix, hash, &self.past, cache_only)?;
+            if r.lane.pending && in_row(&r.lane, &self.prefix) {
+                any = true;
+            }
+        }
+        Ok(any)
+    }
+
+    /// The current row key (escaped prefix aside: the key as given).
+    pub fn row(&self) -> &[u8] {
+        self.current.map_or(&[], |i| self.keys.get(i))
+    }
+
+    /// The next cell of the current key's row, or `None` when the row is done.
+    pub fn next_cell(&mut self) -> Result<Option<ScanCell<'_>>> {
+        let mut qual = std::mem::take(&mut self.qual_buf);
+        qual.clear();
+        let next = self.next_cell_into(&mut qual);
+        self.qual_buf = qual;
+        Ok(next?.map(|family| {
+            let read = &self.reads[self.last_lane.expect("set by next_cell_into")];
+            ScanCell {
+                family,
+                qualifier: &self.qual_buf,
+                ts: read.lane.ts,
+                stored: read.lane.value(),
+            }
+        }))
+    }
+
+    /// Like [`KeyCursor::next_cell`], appending the cell's unescaped qualifier to
+    /// `qualifiers` and returning its family; [`KeyCursor::current_data`] has its version.
+    pub fn next_cell_into(&mut self, qualifiers: &mut Vec<u8>) -> Result<Option<FamilyId>> {
+        if self.current.is_none() {
+            return Ok(None);
+        }
+        if let Some(i) = self.last_lane.take() {
+            let fetched = self.reads[i].lane.fetch();
+            self.snapshot.checked(fetched)?;
+        }
+        let row_len = self.prefix.len();
+        while self.lane_idx < self.reads.len() {
+            let i = self.lane_idx;
+            let lane = &self.reads[i].lane;
+            if in_row(lane, &self.prefix) {
+                self.last_lane = Some(i);
+                let col = lane.col();
+                let end = col.len().saturating_sub(2).max(row_len);
+                Escaped::new(&col[row_len..end]).unescape_into(qualifiers);
+                return Ok(Some(lane.family));
+            }
+            self.lane_idx += 1;
+        }
+        Ok(None)
+    }
+
+    /// Appends the current row's next cell to `sink` (its qualifier unescaped into
+    /// [`RowSink::qualifiers`]); `false` when the row is done.
+    pub fn push_next_cell(&mut self, sink: &mut impl RowSink) -> Result<bool> {
+        if self.current.is_none() {
+            return Ok(false);
+        }
+        if let Some(i) = self.last_lane.take() {
+            let fetched = self.reads[i].lane.fetch();
+            self.snapshot.checked(fetched)?;
+        }
+        let row_len = self.prefix.len();
+        while self.lane_idx < self.reads.len() {
+            let i = self.lane_idx;
+            let lane = &self.reads[i].lane;
+            if in_row(lane, &self.prefix) {
+                let col = lane.col();
+                let end = col.len().saturating_sub(2).max(row_len);
+                let qualifiers = sink.qualifiers();
+                let start = qualifiers.len();
+                Escaped::new(&col[row_len..end]).unescape_into(qualifiers);
+                let qualifier = start..qualifiers.len();
+                let bytes = lane.value();
+                if bytes.len() <= CellData::INLINE_MAX {
+                    sink.push_inline(lane.family, qualifier, lane.ts, bytes);
+                } else {
+                    let view = Arc::clone(&self.snapshot.view);
+                    sink.push(lane.family, qualifier, lane.data(|| view));
+                }
+                self.last_lane = Some(i);
+                return Ok(true);
+            }
+            self.lane_idx += 1;
+        }
+        Ok(false)
+    }
+
+    /// The cell last returned by [`KeyCursor::next_cell`], as a [`CellData`].
+    ///
+    /// # Panics
+    /// If no cell has been returned for the current key.
+    pub fn current_data(&self) -> CellData {
+        let i = self.last_lane.expect("current_data before next_cell");
+        let view = Arc::clone(&self.snapshot.view);
+        self.reads[i].lane.data(|| view)
+    }
+
+    /// D308: positions the current key's reads to resume after a column (see
+    /// [`ReduceSpec::resume`](crate::ReduceSpec)): whether the row has a cell left to read.
+    pub(super) fn resume_in_row(&mut self, resume: &super::reduce::Resume) -> Result<bool> {
+        if self.current.is_none() {
+            return Ok(false);
+        }
+        if let Some(i) = self.last_lane.take() {
+            let fetched = self.reads[i].lane.fetch();
+            self.snapshot.checked(fetched)?;
+        }
+        let mut any = false;
+        for r in &mut self.reads {
+            let rank = self
+                .families
+                .iter()
+                .position(|f| *f == r.lane.family)
+                .unwrap_or(usize::MAX) as u32;
+            // The escaped row without its terminator, as the reduce loop's rows are.
+            let row_esc = &self.prefix[..self.row_esc_len];
+            let applied = resume.apply(&mut r.lane, rank, row_esc);
+            self.snapshot.checked(applied)?;
+            any |= super::reduce::in_row(&r.lane, row_esc);
+        }
+        self.lane_idx = 0;
+        Ok(any)
+    }
+
+    /// Drops the current key's remaining cells without reading them: the next
+    /// [`KeyCursor::next_key`] moves every source on from where it is (D296's `NextRow`).
+    pub fn skip_rest_of_key(&mut self) {
+        for r in &mut self.reads {
+            r.lane.pending = false;
+        }
+        self.last_lane = None;
+        self.lane_idx = self.reads.len();
+    }
+}
+
+impl Drop for KeyCursor {
+    fn drop(&mut self) {
+        self.close_reads();
+    }
+}
+
+/// Admits the async multi-get's reads that have completed (registering `cx` with the rest):
+/// their blocks pinned until their key is read, and the keys they serve probed again.
+fn admit_arrived(
+    nb: &mut AsyncKeys,
+    shared: &crate::shard::Shared,
+    cx: &mut Context<'_>,
+) -> Result<()> {
+    use std::future::Future;
+    let mut j = 0;
+    while j < nb.in_flight.len() {
+        let buf = match std::pin::Pin::new(&mut nb.in_flight[j].1).poll(cx) {
+            Poll::Pending => {
+                j += 1;
+                continue;
+            }
+            Poll::Ready(buf) => buf,
+        };
+        let (k, _, fetch) = nb.in_flight.swap_remove(j);
+        let handle = buf.map_err(Error::from).and_then(|b| Ok(fetch.admit(b)?))?;
+        nb.pinned.extend(handle.map(|h| (k, h)));
+        nb.probed = nb.probed.min(k);
+        if !fetch.is_kept() {
+            // The cache keeps nothing: fetching ahead or positioning cache-only would fetch
+            // the same block again; read synchronously from now on (counted).
+            nb.sync = true;
+            shared.async_sync_reads.fetch_add(1, Ordering::Relaxed);
+        }
+    }
+    Ok(())
+}
+
+/// Whether the lane's held cell is in the row whose prefix (escaped row and terminator) is
+/// `prefix`.
+fn in_row(lane: &Lane, prefix: &[u8]) -> bool {
+    lane.pending && lane.col().starts_with(prefix)
+}
+
+impl FamilyRead {
+    /// Returns the sources of the last key's merge to their slots, then gathers and
+    /// positions this key's (the row filter prunes SSTs, as a point row read does), and
+    /// reads its first cell. With `cache_only`, a block the cache lacks fails it with
+    /// `Error::WouldBlock` (then [`FamilyRead::unwind`]); every source stays in the merge, to
+    /// go back to its slot next time.
+    fn position(&mut self, prefix: &[u8], hash: u64, past: &[u8], cache_only: bool) -> Result<()> {
+        let FamilyRead {
+            lane,
+            filter,
+            opts,
+            priority,
+            mems,
+            ssts,
+            levels,
+            fam,
+            in_merge,
+            reseek,
+        } = self;
+        let sources = lane.resolver.cursor_mut().sources_mut();
+        for (src, slot) in sources.drain(..).zip(in_merge.drain(..)) {
+            match slot {
+                Slot::Mem(i) => mems[i] = Some(src),
+                Slot::Sst(i) => ssts[i].source = Some(src),
+            }
+        }
+        let reseek = std::mem::take(reseek);
+        lane.pending = false;
+        lane.pinned = None;
+        for (i, m) in mems.iter_mut().enumerate() {
+            let mut src = m.take().expect("a memtable source is in its slot");
+            let moved = move_to(&mut src, prefix, reseek);
+            sources.push(src);
+            in_merge.push(Slot::Mem(i));
+            moved?;
+        }
+        // A point read's options without `one_row`: a cursor that stopped at its row's end
+        // could not move forward to the next key, and would seek from the top.
+        let mut read_opts = read_options(*priority, false, false, cache_only);
+        read_opts.one_row = false;
+        let set = Arc::clone(&lane.ssts);
+        each_sst(fam.as_deref(), levels, ssts, prefix, |i, slot| {
+            let reader = slot.reader(&set, *priority, cache_only)?;
+            if !reader.may_contain_row(hash) {
+                return Ok(());
+            }
+            let fresh = reseek || slot.source.is_none();
+            let mut src = match slot.source.take() {
+                Some(mut src) => {
+                    src.set_cache_only(cache_only);
+                    src
+                }
+                None => Source::Sst(reader.iter(filter.clone(), read_opts).into()),
+            };
+            // Kept in the merge even if the move fails: an SST cursor whose move failed is
+            // invalid, and seeks next time.
+            let moved = move_to(&mut src, prefix, fresh);
+            sources.push(src);
+            in_merge.push(Slot::Sst(i));
+            moved
+        })?;
+        if sources.is_empty() {
+            lane.done = true;
+            return Ok(());
+        }
+        lane.done = false;
+        let mut o = opts.clone();
+        o.skip_columns = skips_columns(sources);
+        let resolver = &mut lane.resolver;
+        resolver.cursor_mut().reset();
+        resolver.reset(o);
+        resolver.set_upper_bound(Some(past));
+        resolver.seek_with(|cursor| cursor.rebuild_positioned())?;
+        lane.fetch()
+    }
+
+    /// Undoes a positioning that failed cache-only: the lane holds no cell, and the key's
+    /// next positioning seeks every source afresh (one may have stepped past the row's start
+    /// reading its first cell).
+    fn unwind(&mut self) {
+        self.reseek = true;
+        self.lane.pending = false;
+        self.lane.pinned = None;
+        self.lane.done = true;
+    }
+
+    /// Hands `f` the fetch of each block a seek to the row `prefix` would read first that
+    /// the cache lacks: per SST that may hold the row, its index partition or data block
+    /// ([`SstReader::fetch_for`]), or what opening its reader needs.
+    fn probe(&mut self, prefix: &[u8], hash: u64, mut f: impl FnMut(pigeonhole_sst::Fetch)) {
+        let set = Arc::clone(&self.lane.ssts);
+        let priority = self.priority;
+        let _ = each_sst(
+            self.fam.as_deref(),
+            &self.levels,
+            &mut self.ssts,
+            prefix,
+            |_, slot| {
+                match slot.reader(&set, priority, true) {
+                    Ok(reader) => {
+                        if reader.may_contain_row(hash)
+                            && let Some(fetch) = reader.fetch_for(prefix, priority)
+                        {
+                            f(fetch);
+                        }
+                    }
+                    Err(Error::WouldBlock(fetch)) => f(*fetch),
+                    Err(_) => {}
+                }
+                Ok(())
+            },
+        );
+    }
+}
+
+/// Moves `src` to the first entry at or past `target` (a row prefix): a full seek if it is
+/// `fresh` or has run past its end, a forward one from where it is if it is behind, nothing
+/// if it is already there (keys come in increasing order).
+fn move_to(src: &mut Source, target: &[u8], fresh: bool) -> Result<()> {
+    if fresh || !src.valid() {
+        src.seek(target)
+    } else if src.key() < target {
+        src.seek_forward(target)
+    } else {
+        Ok(())
+    }
+}