pigeonhole-db 1.1.0

data/vendor/engine/src/read/reduce.rs

@@ -0,0 +1,2057 @@
+//! Bounded-result read pushdown (D296, #490 A): a visitor runs on each resolved cell of the
+//! input, where the scan would have handed it out, and a reducer keeps a bounded result (the
+//! k best, the first n, one per key). Only the reducer's result leaves the read.
+//!
+//! The scan's own code ([`super::ScanCursor`]) is untouched: this module's loop is a sibling
+//! that drives the same per-family [`Lane`]s (resolver over the tablet's sources), so the
+//! visitor sees exactly what a scan returns (snapshot, deletes, TTL, versions, merges and
+//! value predicates applied) and reads each value where the lane holds it, without assembling
+//! rows. #1167 measured that any change to the scan's per-cell loop costs its shapes.
+//!
+//! Work is cut at tablet boundaries into groups. Each group has its own visitor and reducer,
+//! and stops only on its own account (a visitor's [`Visit::Stop`], or its reducer's `Break`):
+//! no group stops because of another, so a group's result is a function of its key range at
+//! the snapshot, and merging the groups' results gives exactly the sequential result at any
+//! fan-out (D296 rev 1.2). The one shared flag ends every group on an error.
+
+use std::ops::{Bound, ControlFlow};
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+
+use pigeonhole_format::key::{Escaped, TERMINATOR, encode_row_prefix};
+use pigeonhole_format::{Cursor, FamilyId, TableId, Timestamp};
+
+use super::keys::{KeyCursor, KeyGroup, KeySet, key_groups};
+use super::{
+    FORK_CHECK_ROWS, Lane, LaneValue, MergeKind, ReadSpec, clamp_into, column_of, give_back,
+    past_row, row_of, skips_columns, take_scan_resolver,
+};
+use crate::catalog::FamilyMeta;
+use crate::snapshot::{Snapshot, SstSet, TabletEntry, View};
+use crate::source::{Located, Pinned, Source};
+use crate::{ColumnPosition, Error, Result};
+
+/// A row range on unescaped row keys, `(start, end)`.
+pub type RowRange = (Bound<Vec<u8>>, Bound<Vec<u8>>);
+
+/// What a bounded read reads.
+#[derive(Debug, Clone)]
+#[non_exhaustive]
+pub enum ReadInput {
+    /// Row ranges, sorted and disjoint, each `(start, end)` on unescaped row keys. One range
+    /// is a scan's input; several cover, for example, an IVF index's probed partitions in one
+    /// read under one snapshot.
+    Ranges(Vec<RowRange>),
+    /// Rows by key (D299's multi-get, #489): sorted and distinct by construction, read with
+    /// one forward cursor per source per tablet group.
+    Keys(Arc<KeySet>),
+}
+
+/// What a [`CellVisitor`] tells the read to do after one cell.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Visit<I> {
+    /// Nothing from this cell; go on to the next.
+    Skip,
+    /// Push `I` to the reducer and go on.
+    Emit(I),
+    /// Push `I`, then skip the rest of this row (a seek past it, not a walk).
+    EmitAndNextRow(I),
+    /// Skip the rest of this row.
+    NextRow,
+    /// End this group's read.
+    Stop,
+}
+
+/// Runs on each resolved cell of a bounded read.
+pub trait CellVisitor {
+    /// What it hands the reducer.
+    type Item;
+
+    /// Whether [`visit`](Self::visit) reads its `qualifier`. A visitor that doesn't (a
+    /// nearest-neighbour search scores the value alone) gets an empty one, and a range read
+    /// unescapes the qualifier only for an item the reducer [`admits`](Reducer::admits)
+    /// (#1283 lever 1). A constant, so a visitor that reads it compiles as before.
+    const QUALIFIER: bool = true;
+
+    /// One cell, borrowed for the call: `row` and `qualifier` unescaped, `value` the stored
+    /// value with its tag byte (in place where the source holds it).
+    fn visit(
+        &mut self,
+        row: &[u8],
+        family: FamilyId,
+        qualifier: &[u8],
+        ts: Timestamp,
+        value: &[u8],
+    ) -> Visit<Self::Item>;
+}
+
+/// The cell an item came from, borrowed for a [`Reducer::push`]. Cells order as a scan hands
+/// them out: by row, then family in the read's family order, then qualifier, then newest
+/// version first. That is a total order over a read's cells.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct CellRef<'a> {
+    /// Unescaped row key.
+    pub row: &'a [u8],
+    /// The family's position in the read's family order (the scan's).
+    pub family_rank: u32,
+    /// The family.
+    pub family: FamilyId,
+    /// Unescaped qualifier.
+    pub qualifier: &'a [u8],
+    /// The version's timestamp.
+    pub ts: Timestamp,
+}
+
+impl CellRef<'_> {
+    fn order(&self) -> (&[u8], u32, &[u8], std::cmp::Reverse<Timestamp>) {
+        (
+            self.row,
+            self.family_rank,
+            self.qualifier,
+            std::cmp::Reverse(self.ts),
+        )
+    }
+
+    /// An owned copy.
+    pub fn to_key(&self) -> CellKey {
+        CellKey {
+            row: self.row.to_vec(),
+            family_rank: self.family_rank,
+            family: self.family,
+            qualifier: self.qualifier.to_vec(),
+            ts: self.ts,
+        }
+    }
+}
+
+impl PartialOrd for CellRef<'_> {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl Ord for CellRef<'_> {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        self.order().cmp(&other.order())
+    }
+}
+
+/// An owned [`CellRef`]: a result's cell, ordered the same way.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct CellKey {
+    /// Unescaped row key.
+    pub row: Vec<u8>,
+    /// The family's position in the read's family order.
+    pub family_rank: u32,
+    /// The family.
+    pub family: FamilyId,
+    /// Unescaped qualifier.
+    pub qualifier: Vec<u8>,
+    /// The version's timestamp.
+    pub ts: Timestamp,
+}
+
+impl CellKey {
+    /// This key, borrowed.
+    pub fn as_ref(&self) -> CellRef<'_> {
+        CellRef {
+            row: &self.row,
+            family_rank: self.family_rank,
+            family: self.family,
+            qualifier: &self.qualifier,
+            ts: self.ts,
+        }
+    }
+}
+
+impl PartialOrd for CellKey {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl Ord for CellKey {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        self.as_ref().cmp(&other.as_ref())
+    }
+}
+
+/// Keeps a bounded result. `merge` must be associative and commutative, and the result a
+/// function of the multiset of pushes, so fan-out equals the sequential read.
+pub trait Reducer: Send {
+    /// What it keeps.
+    type Item: Send;
+
+    /// One item from `cell`. A group pushes in cell order. `Break` ends this group's read
+    /// (the reducer is full, or has all it can take).
+    fn push(&mut self, cell: CellRef<'_>, item: Self::Item) -> ControlFlow<()>;
+
+    /// Whether `item` could be kept, judged before its cell is built: `false` only when
+    /// [`push`](Self::push) would drop it whatever its cell, and then it isn't pushed (#1283
+    /// lever 1). It never changes what is kept: an item that ties the bound is admitted, and
+    /// `push` breaks the tie by cell.
+    #[inline]
+    fn admits(&self, _item: &Self::Item) -> bool {
+        true
+    }
+
+    /// Folds another group's result in.
+    fn merge(&mut self, other: Self);
+
+    /// The result, in the reducer's order.
+    fn finish(self) -> Vec<(CellKey, Self::Item)>;
+}
+
+/// A bounded read's input, per-row selection and fan-out.
+#[derive(Debug, Clone)]
+#[non_exhaustive]
+pub struct ReduceSpec {
+    /// The rows read.
+    pub input: ReadInput,
+    /// Families, qualifier selection, time range, value predicates and versions, as for a scan.
+    pub read: ReadSpec,
+    /// Threads the call may use (1 = the caller's only). More run tablet groups on scoped
+    /// threads owned by the call, never the engine's: only when the input is large enough to
+    /// pay for them (`parallel_floor` estimated entries per thread), and never on a thread
+    /// that drives a shard (D88).
+    pub parallelism: usize,
+    /// Estimated entries per thread below which a read does not fan out
+    /// ([`PARALLEL_FLOOR_ENTRIES`] by default; 0 fans out whenever there are groups). For a
+    /// [`ReadInput::Keys`] input it counts keys: a key set's rows are not estimated before
+    /// they are read. Key groups are per tablet, so keys inside one tablet never fan out.
+    pub parallel_floor: u64,
+    /// D308: the read of this row (unescaped), if the input reaches it, resumes after this
+    /// column: families the read selects before the position's skip the row, the
+    /// position's family starts past the column (all its versions), and later families
+    /// start at the selection's own bound. Other rows are read as usual. A position from
+    /// another table, or whose family the read doesn't select, is `InvalidArgument`; so is
+    /// one whose row a range excludes by its start (`Bound::Excluded(row)`): the row would be
+    /// skipped whole, so a range resumed in a row starts at `Bound::Included(row)`.
+    pub resume: Option<(Vec<u8>, ColumnPosition)>,
+}
+
+impl ReduceSpec {
+    /// `input` with the default selection, on the caller's thread.
+    pub fn new(input: ReadInput) -> Self {
+        Self {
+            input,
+            read: ReadSpec::default(),
+            parallelism: 1,
+            parallel_floor: PARALLEL_FLOOR_ENTRIES,
+            resume: None,
+        }
+    }
+}
+
+/// A bounded read's result and counts.
+#[derive(Debug)]
+#[non_exhaustive]
+pub struct Reduced<I> {
+    /// `(cell, item)`, in the reducer's order.
+    pub items: Vec<(CellKey, I)>,
+    /// Counts over every group.
+    pub stats: ReduceStats,
+}
+
+/// Counts of a bounded read.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+#[non_exhaustive]
+pub struct ReduceStats {
+    /// Tablet groups read.
+    pub groups: u64,
+    /// Rows the visitor saw a cell of.
+    pub rows: u64,
+    /// Cells the visitor saw.
+    pub cells: u64,
+    /// Threads used (1 when the read ran on the caller's thread).
+    pub threads: u64,
+    /// Lanes read straight from their one SST, with nothing to resolve (#1283 lever 3).
+    pub plain_lanes: u64,
+}
+
+impl ReduceStats {
+    fn add(&mut self, o: &ReduceStats) {
+        self.groups += o.groups;
+        self.rows += o.rows;
+        self.cells += o.cells;
+        self.plain_lanes += o.plain_lanes;
+    }
+}
+
+/// Estimated entries a tablet group must hold, on average, before a read fans out: about
+/// 20,000 rows at today's per-row cost is some 2 ms, against 10-20 µs to start a thread
+/// (D296 rev 1, perf287's work floor).
+pub const PARALLEL_FLOOR_ENTRIES: u64 = 20_000;
+
+/// One tablet's share of one input range.
+struct Group<'a> {
+    tablet: &'a TabletEntry,
+    /// The range, unescaped (for the excluded start and the end check).
+    start: &'a Bound<Vec<u8>>,
+    end: &'a Bound<Vec<u8>>,
+    /// The range clamped to the tablet, as row prefixes (`None` = unbounded).
+    lo: Option<Vec<u8>>,
+    hi: Option<Vec<u8>>,
+}
+
+/// A unit of a bounded read: one tablet's share of a range, or one tablet's run of keys.
+enum Work<'a> {
+    Range(Group<'a>),
+    Keys(KeyGroup),
+}
+
+/// Cuts `input` into work units, in key order.
+fn work<'a>(snapshot: &'a Snapshot, table: TableId, input: &'a ReadInput) -> Result<Vec<Work<'a>>> {
+    match input {
+        ReadInput::Ranges(ranges) => Ok(groups(&snapshot.view, table, ranges)?
+            .into_iter()
+            .map(Work::Range)
+            .collect()),
+        ReadInput::Keys(keys) => Ok(key_groups(snapshot, table, keys)?
+            .into_iter()
+            .map(Work::Keys)
+            .collect()),
+    }
+}
+
+/// Cuts row ranges into tablet groups, in key order.
+fn groups<'a>(view: &'a View, table: TableId, ranges: &'a [RowRange]) -> Result<Vec<Group<'a>>> {
+    // Sorted and disjoint, or a cell would be visited (and counted by a reducer) twice.
+    for pair in ranges.windows(2) {
+        if !before(&pair[0].1, &pair[1].0) {
+            return Err(Error::InvalidArgument(
+                "bounded read: the ranges must be sorted and disjoint".to_owned(),
+            ));
+        }
+    }
+    let mut out = Vec::new();
+    let (mut lo, mut hi) = (Vec::new(), Vec::new());
+    for (start, end) in ranges {
+        let s = match start {
+            Bound::Included(s) | Bound::Excluded(s) => {
+                let mut k = Vec::new();
+                encode_row_prefix(&mut k, s)?;
+                Some(k)
+            }
+            Bound::Unbounded => None,
+        };
+        let e = match end {
+            Bound::Excluded(e) => {
+                let mut k = Vec::new();
+                encode_row_prefix(&mut k, e)?;
+                Some(k)
+            }
+            Bound::Included(e) => {
+                let mut k = Vec::new();
+                encode_row_prefix(&mut k, e)?;
+                let mut past = Vec::new();
+                past_row(&k, &mut past);
+                Some(past)
+            }
+            Bound::Unbounded => None,
+        };
+        for tablet in view.tablets().tablets_of(table) {
+            let (has_lo, has_hi) =
+                clamp_into(tablet, s.as_deref(), e.as_deref(), &mut lo, &mut hi)?;
+            if has_lo && has_hi && lo >= hi {
+                continue;
+            }
+            out.push(Group {
+                tablet,
+                start,
+                end,
+                lo: has_lo.then(|| lo.clone()),
+                hi: has_hi.then(|| hi.clone()),
+            });
+        }
+    }
+    Ok(out)
+}
+
+/// Whether every row a range ending at `end` holds comes before every row a range starting
+/// at `start` holds.
+fn before(end: &Bound<Vec<u8>>, start: &Bound<Vec<u8>>) -> bool {
+    match (end, start) {
+        (Bound::Unbounded, _) | (_, Bound::Unbounded) => false,
+        (Bound::Included(e), Bound::Included(s)) => e < s,
+        (Bound::Included(e) | Bound::Excluded(e), Bound::Excluded(s))
+        | (Bound::Excluded(e), Bound::Included(s)) => e <= s,
+    }
+}
+
+/// Entries the group's sources hold (SSTs it may read, memtables): an over-estimate after a
+/// split, enough to decide whether fan-out pays.
+fn estimate(view: &View, families: &[FamilyId], g: &Group<'_>) -> u64 {
+    let mut n = 0;
+    for &family in families {
+        let located = view.locate(g.tablet.shard, g.tablet.id, family);
+        if let Some(ssts) = located.ssts {
+            n += ssts
+                .levels
+                .iter()
+                .flatten()
+                .map(|s| s.meta.entries)
+                .sum::<u64>();
+        }
+        if let Some(mems) = located.mems {
+            n += mems.readers.iter().map(|r| r.len() as u64).sum::<u64>();
+        }
+    }
+    n
+}
+
+/// What a group's read needs from the call.
+struct Ctx<'a> {
+    snapshot: &'a Snapshot,
+    /// For the `Keys` input's cursors (their async-read counters).
+    shared: &'a Arc<crate::shard::Shared>,
+    input: &'a ReadInput,
+    families: &'a [FamilyId],
+    read: &'a ReadSpec,
+    now: Timestamp,
+    abort: &'a AtomicBool,
+    resume: Option<&'a Resume>,
+}
+
+/// [`ReduceSpec::resume`], checked and encoded: the row, the position's family's rank in the
+/// read's family order, and the key just past every version of the column.
+pub(super) struct Resume {
+    row: Vec<u8>,
+    rank: u32,
+    past: Vec<u8>,
+}
+
+impl Resume {
+    pub(super) fn new(
+        table: TableId,
+        families: &[FamilyId],
+        (row, pos): &(Vec<u8>, ColumnPosition),
+    ) -> Result<Self> {
+        if pos.table() != table {
+            return Err(Error::InvalidArgument(
+                "a position from another table".to_owned(),
+            ));
+        }
+        let Some(rank) = families.iter().position(|f| *f == pos.family()) else {
+            return Err(Error::InvalidArgument(
+                "a position whose family the read does not select".to_owned(),
+            ));
+        };
+        let mut col = Vec::new();
+        pigeonhole_format::key::encode_column_prefix(&mut col, row, pos.qualifier())?;
+        let mut past = Vec::new();
+        past_row(&col, &mut past);
+        Ok(Self {
+            row: row.clone(),
+            rank: rank as u32,
+            past,
+        })
+    }
+
+    /// Whether `row` (unescaped) is the position's row.
+    pub(super) fn is(&self, row: &[u8]) -> bool {
+        self.row == row
+    }
+
+    /// Positions the lane of family rank `rank`, pending in the position's row (escaped as
+    /// `row_esc`), as the resume says: dropped from the row, moved past the column, or left.
+    pub(super) fn apply<L: ReduceLane>(
+        &self,
+        lane: &mut L,
+        rank: u32,
+        row_esc: &[u8],
+    ) -> Result<()> {
+        if !in_row(lane, row_esc) {
+            return Ok(());
+        }
+        match rank.cmp(&self.rank) {
+            std::cmp::Ordering::Less => lane.drop_row(),
+            std::cmp::Ordering::Equal => {
+                if lane.col() < self.past.as_slice() {
+                    lane.reseek(&self.past)?;
+                }
+                Ok(())
+            }
+            std::cmp::Ordering::Greater => Ok(()),
+        }
+    }
+}
+
+/// A lane `visit_lanes` reads: a [`Lane`] (through the MVCC resolver), or a [`PlainLane`]
+/// (#1283 lever 3). A group reads through one kind only, so each is its own instance of the
+/// loop, and neither pays a dispatch per cell.
+pub(super) trait ReduceLane {
+    /// The lane's family.
+    fn family(&self) -> FamilyId;
+    /// A cell is held (fetched, not yet handed on).
+    fn pending(&self) -> bool;
+    /// No cell is left.
+    fn done(&self) -> bool;
+    /// The held cell's timestamp.
+    fn ts(&self) -> Timestamp;
+    /// Moves to the next visible cell.
+    fn fetch(&mut self) -> Result<()>;
+    /// The held cell's column prefix (only while pending).
+    fn col(&self) -> &[u8];
+    /// The held cell's stored value (only while pending).
+    fn value(&self) -> &[u8];
+    /// Drops the held cell and the rest of its row.
+    fn drop_row(&mut self) -> Result<()>;
+    /// Drops the held cell and holds the first visible one at or after `key`.
+    fn reseek(&mut self, key: &[u8]) -> Result<()>;
+}
+
+impl ReduceLane for Lane {
+    #[inline]
+    fn family(&self) -> FamilyId {
+        self.family
+    }
+    #[inline]
+    fn pending(&self) -> bool {
+        self.pending
+    }
+    #[inline]
+    fn done(&self) -> bool {
+        self.done
+    }
+    #[inline]
+    fn ts(&self) -> Timestamp {
+        self.ts
+    }
+    #[inline]
+    fn fetch(&mut self) -> Result<()> {
+        Lane::fetch(self)
+    }
+    #[inline]
+    fn col(&self) -> &[u8] {
+        Lane::col(self)
+    }
+    #[inline]
+    fn value(&self) -> &[u8] {
+        Lane::value(self)
+    }
+    fn drop_row(&mut self) -> Result<()> {
+        self.pending = false;
+        self.resolver.skip_row()
+    }
+    fn reseek(&mut self, key: &[u8]) -> Result<()> {
+        self.pending = false;
+        self.pinned = None;
+        self.resolver.seek(key)?;
+        Lane::fetch(self)
+    }
+}
+
+/// A lane over the one SST source of its family in the tablet, when nothing in it needs the
+/// MVCC resolver (#1283 lever 3, [`plain_lane`]). A column's first entry in key order is
+/// then its version (newest timestamp, then newest seqno), so the lane takes it from the
+/// block as it lies and steps past the column's older entries: no merge, group walk or copy.
+struct PlainLane {
+    family: FamilyId,
+    src: Source,
+    /// The view's SSTs and blob files, to read separated values.
+    ssts: Arc<SstSet>,
+    /// The column prefix of the cell last taken: an entry still in it is an older version.
+    col: Vec<u8>,
+    /// The group's end (an encoded row prefix), as `CellResolver::set_upper_bound`.
+    hi: Option<Vec<u8>>,
+    /// The source is on the held cell (it moves past it on the next fetch).
+    held: bool,
+    ts: Timestamp,
+    /// A separated value, read from its blob file.
+    pinned: Option<LaneValue>,
+    pending: bool,
+    done: bool,
+}
+
+impl PlainLane {
+    /// The lane `lane` (opened by `open_lanes`, plain by [`plain_lane`]) as a plain one: its
+    /// SST source, seeked to the group's start. The resolver goes back to the pool.
+    fn from_lane(mut lane: Lane, g: &Group<'_>) -> Result<PlainLane> {
+        let mut sources = std::mem::take(lane.resolver.cursor_mut().sources_mut());
+        give_back(lane.resolver);
+        // The empty memtables' sources go (`plain_lane` checked they hold nothing).
+        sources.retain(|s| matches!(s, Source::Sst(_)));
+        let mut src = sources.pop().expect("plain_lane checked one SST source");
+        src.seek(g.lo.as_deref().unwrap_or(&[]))?;
+        Ok(PlainLane {
+            family: lane.family,
+            src,
+            ssts: lane.ssts,
+            col: Vec::new(),
+            hi: g.hi.clone(),
+            held: false,
+            ts: 0,
+            pinned: None,
+            pending: false,
+            done: false,
+        })
+    }
+}
+
+impl ReduceLane for PlainLane {
+    #[inline]
+    fn family(&self) -> FamilyId {
+        self.family
+    }
+    #[inline]
+    fn pending(&self) -> bool {
+        self.pending
+    }
+    #[inline]
+    fn done(&self) -> bool {
+        self.done
+    }
+    #[inline]
+    fn ts(&self) -> Timestamp {
+        self.ts
+    }
+    fn fetch(&mut self) -> Result<()> {
+        self.pending = false;
+        self.pinned = None;
+        if self.done {
+            return Ok(());
+        }
+        if std::mem::take(&mut self.held) {
+            self.src.next()?;
+        }
+        while self.src.valid() {
+            let key = self.src.key();
+            if self.hi.as_deref().is_some_and(|hi| key >= hi) {
+                break;
+            }
+            let (col, ts, _, kind) = pigeonhole_format::key::split_suffix(key)?;
+            if col == self.col.as_slice() {
+                // An older version of the column just taken.
+                self.src.next()?;
+                continue;
+            }
+            if kind != pigeonhole_format::Kind::Put {
+                // `plain_lane` checked the SSTs hold no deletes, and the family no operands.
+                return Err(Error::Corruption(
+                    "an entry other than a put in an SST counted without deletes or operands"
+                        .into(),
+                ));
+            }
+            self.col.clear();
+            self.col.extend_from_slice(col);
+            self.held = true;
+            self.ts = ts;
+            if let Some(v) = self.ssts.read_blob(self.src.value())? {
+                self.pinned = Some(LaneValue::Pinned(Pinned::Block(v)));
+            }
+            self.pending = true;
+            return Ok(());
+        }
+        self.done = true;
+        Ok(())
+    }
+    #[inline]
+    fn col(&self) -> &[u8] {
+        column_of(self.src.key())
+    }
+    #[inline]
+    fn value(&self) -> &[u8] {
+        match &self.pinned {
+            Some(v) => v,
+            None => self.src.value(),
+        }
+    }
+    fn drop_row(&mut self) -> Result<()> {
+        // The source is on the held cell, in the row: skip from there.
+        self.pending = false;
+        self.held = false;
+        self.src.skip_row()
+    }
+    fn reseek(&mut self, key: &[u8]) -> Result<()> {
+        self.pending = false;
+        self.held = false;
+        self.col.clear();
+        self.src.seek(key)?;
+        self.fetch()
+    }
+}
+
+/// Whether a lane with `sources` (filled for `located` with `filter`) and options `opts` can
+/// read plain ([`PlainLane`]) at read point `snapshot`. Checked once, when the lane opens
+/// (simp724's review on #1283):
+/// 1. one SST source (an SST, or one level's SSTs), and every memtable of the family here
+///    empty. `scan_sources_located` adds a source for every memtable, the empty active one
+///    included; one empty when the lane opens holds nothing at or below the read point,
+///    since every commit there applied before the read point became visible;
+/// 2. no delete entries (cell, column or family: `SstMeta::deletes` counts every kind) in
+///    any of the family's SSTs here;
+/// 3. every entry at or below the read point;
+/// 4. a standard or vector family (plain puts) with no merge operator, known or not;
+/// 5. one version, no time range (the resolver's or the filter's), no column limit and no
+///    value predicate;
+/// 6. no TTL.
+fn plain_lane(
+    located: &Located<'_>,
+    meta: &FamilyMeta,
+    opts: &pigeonhole_compaction::ResolveOptions,
+    filter: &pigeonhole_format::scan::ScanFilter,
+    sources: &[Source],
+    snapshot: pigeonhole_format::Seqno,
+) -> bool {
+    let Some(ssts) = located.ssts else {
+        return false;
+    };
+    sources.iter().filter(|s| matches!(s, Source::Sst(_))).count() == 1
+        && located
+            .mems
+            .is_none_or(|m| m.readers.iter().all(|r| r.is_empty()))
+        && ssts
+            .iter()
+            .all(|s| s.meta.deletes == 0 && s.meta.seqno_range.1 <= snapshot)
+        // A kind added later stays out until listed here.
+        && matches!(
+            meta.options.kind,
+            pigeonhole_format::manifest::FamilyKind::Standard
+                | pigeonhole_format::manifest::FamilyKind::Vector
+        )
+        // `merge_op` alone is also `None` for an operator this build doesn't know
+        // (`MergeKind::Unknown`), whose operands are still there.
+        && meta.merge == MergeKind::None
+        && meta.merge_op.is_none()
+        && meta.options.ttl_micros == 0
+        && opts.versions == 1
+        && opts.time_range.is_none()
+        && filter.time_range.is_none()
+        && opts.columns_per_row == 0
+        && opts.value.is_none()
+}
+
+/// Opens the group's lanes, one per family with data in the tablet's range, each seeked to
+/// the group's start: [`super::ScanCursor`]'s tablet opening, for one tablet.
+fn open_lanes(
+    ctx: &Ctx<'_>,
+    g: &Group<'_>,
+    lanes: &mut Vec<Lane>,
+    plain: &mut Vec<bool>,
+) -> Result<()> {
+    let view = &ctx.snapshot.view;
+    for &family in ctx.families {
+        let located = view.locate(g.tablet.shard, g.tablet.id, family);
+        if !located.has_data() {
+            continue;
+        }
+        let Some(meta) = view.catalog.family(family) else {
+            continue;
+        };
+        let (mut opts, filter) = ctx.read.resolve_opts(meta, ctx.snapshot.seqno, ctx.now);
+        let resolver_blobs = super::ResolverBlobs::attach(&mut opts, &view.ssts);
+        let mut resolver = take_scan_resolver();
+        let filled = view.scan_sources_located::<false>(
+            &located,
+            &filter,
+            g.lo.as_deref(),
+            g.hi.as_deref(),
+            false,
+            resolver.cursor_mut().sources_mut(),
+        );
+        if let Err(e) = filled {
+            give_back(resolver);
+            return Err(e);
+        }
+        if resolver.cursor_mut().sources().is_empty() {
+            give_back(resolver);
+            continue;
+        }
+        opts.skip_columns = skips_columns(resolver.cursor_mut().sources());
+        plain.push(plain_lane(
+            &located,
+            meta,
+            &opts,
+            &filter,
+            resolver.cursor_mut().sources(),
+            ctx.snapshot.seqno,
+        ));
+        resolver.cursor_mut().reset();
+        resolver.reset(opts);
+        resolver.set_upper_bound(g.hi.as_deref());
+        lanes.push(Lane {
+            family,
+            unknown_merge: (meta.merge == MergeKind::Unknown)
+                .then(|| meta.options.merge_operator.clone()),
+            resolver,
+            grew: false,
+            ssts: Arc::clone(&view.ssts),
+            resolver_blobs,
+            ts: 0,
+            pinned: None,
+            pending: false,
+            done: false,
+        });
+        let lane = lanes.last_mut().expect("pushed above");
+        lane.resolver.seek(g.lo.as_deref().unwrap_or(&[]))?;
+    }
+    Ok(())
+}
+
+/// The family's position in the read's family order.
+fn rank_of(families: &[FamilyId], family: FamilyId) -> u32 {
+    families
+        .iter()
+        .position(|f| *f == family)
+        .unwrap_or(usize::MAX) as u32
+}
+
+/// Whether the lane's pending cell is in the row whose escaped key is `row_esc`.
+pub(super) fn in_row<L: ReduceLane>(lane: &L, row_esc: &[u8]) -> bool {
+    if !lane.pending() {
+        return false;
+    }
+    let col = lane.col();
+    let row_len = row_esc.len() + TERMINATOR.len();
+    col.len() >= row_len && col.starts_with(row_esc) && col[row_esc.len()..row_len] == TERMINATOR
+}
+
+/// Drops the rest of the row in every lane (a seek past it).
+fn skip_row<L: ReduceLane>(lanes: &mut [L], row_esc: &[u8]) -> Result<()> {
+    for lane in lanes {
+        if in_row(lane, row_esc) {
+            lane.drop_row()?;
+        }
+    }
+    Ok(())
+}
+
+/// Reads one group into `reducer`; `true` when the reducer broke (it has all it can take).
+fn run_group<V, R>(
+    ctx: &Ctx<'_>,
+    g: &Group<'_>,
+    visitor: &mut V,
+    reducer: &mut R,
+    stats: &mut ReduceStats,
+) -> Result<bool>
+where
+    V: CellVisitor<Item = R::Item>,
+    R: Reducer,
+{
+    ctx.snapshot.check_process()?;
+    let mut lanes = Vec::new();
+    let mut plain = Vec::new();
+    let opened = open_lanes(ctx, g, &mut lanes, &mut plain);
+    // Each lane's family position in the read's family order, once per group (perf287's
+    // nit on #1209: not a search per cell).
+    let ranks: Vec<u32> = lanes
+        .iter()
+        .map(|l| rank_of(ctx.families, l.family))
+        .collect();
+    if opened.is_ok() && !lanes.is_empty() && plain.iter().all(|p| *p) {
+        // Every lane plain (#1283 lever 3): the group reads through `PlainLane`s, a second
+        // instance of `visit_lanes`, so neither kind of lane pays for the other per cell.
+        let mut plains = Vec::with_capacity(lanes.len());
+        let mut moved = Ok(());
+        for lane in lanes {
+            match moved.and_then(|()| PlainLane::from_lane(lane, g)) {
+                Ok(p) => {
+                    plains.push(p);
+                    moved = Ok(());
+                }
+                Err(e) => moved = Err(e),
+            }
+        }
+        stats.plain_lanes += plains.len() as u64;
+        let read =
+            moved.and_then(|()| visit_lanes(ctx, g, &mut plains, &ranks, visitor, reducer, stats));
+        stats.groups += 1;
+        return ctx.snapshot.checked(read);
+    }
+    let read =
+        opened.and_then(|()| visit_lanes(ctx, g, &mut lanes, &ranks, visitor, reducer, stats));
+    for lane in lanes {
+        if !lane.grew {
+            give_back(lane.resolver);
+        }
+    }
+    stats.groups += 1;
+    ctx.snapshot.checked(read)
+}
+
+fn visit_lanes<V, R, L>(
+    ctx: &Ctx<'_>,
+    g: &Group<'_>,
+    lanes: &mut [L],
+    ranks: &[u32],
+    visitor: &mut V,
+    reducer: &mut R,
+    stats: &mut ReduceStats,
+) -> Result<bool>
+where
+    V: CellVisitor<Item = R::Item>,
+    R: Reducer,
+    L: ReduceLane,
+{
+    let (mut row_esc, mut row, mut qualifier) = (Vec::new(), Vec::new(), Vec::new());
+    loop {
+        if ctx.abort.load(Ordering::Relaxed) {
+            return Ok(false);
+        }
+        for lane in lanes.iter_mut() {
+            if !lane.pending() && !lane.done() {
+                lane.fetch()?;
+            }
+        }
+        // The smallest pending row across lanes.
+        let Some(best) = lanes
+            .iter()
+            .filter(|l| l.pending())
+            .map(|l| row_of(l.col()))
+            .min()
+        else {
+            return Ok(false);
+        };
+        row_esc.clear();
+        row_esc.extend_from_slice(best);
+        row.clear();
+        Escaped::new(&row_esc).unescape_long_into(&mut row);
+        if matches!(g.start, Bound::Excluded(s) if *s == row) {
+            skip_row(lanes, &row_esc)?;
+            continue;
+        }
+        let past = match g.end {
+            Bound::Included(e) => row > *e,
+            Bound::Excluded(e) => row >= *e,
+            Bound::Unbounded => false,
+        };
+        if past {
+            return Ok(false);
+        }
+        if let Some(r) = ctx.resume
+            && r.is(&row)
+        {
+            for (lane, &rank) in lanes.iter_mut().zip(ranks) {
+                r.apply(lane, rank, &row_esc)?;
+            }
+            if !lanes.iter().any(|l| in_row(l, &row_esc)) {
+                continue;
+            }
+        }
+        stats.rows += 1;
+        if stats.rows.is_multiple_of(FORK_CHECK_ROWS) {
+            ctx.snapshot.check_process()?;
+        }
+        let row_len = row_esc.len() + TERMINATOR.len();
+        let mut next_row = false;
+        'lanes: for (lane, &family_rank) in lanes.iter_mut().zip(ranks) {
+            while in_row(lane, &row_esc) {
+                let col = lane.col();
+                let end = col.len().saturating_sub(2).max(row_len);
+                // Unescaped before the visit only for a visitor that reads it; otherwise only
+                // for an item the reducer admits (#1283 lever 1).
+                if V::QUALIFIER {
+                    qualifier.clear();
+                    Escaped::new(&col[row_len..end]).unescape_into(&mut qualifier);
+                }
+                stats.cells += 1;
+                let q: &[u8] = if V::QUALIFIER { &qualifier } else { &[] };
+                let visit = visitor.visit(&row, lane.family(), q, lane.ts(), lane.value());
+                let (item, after) = match visit {
+                    Visit::Skip => (None, None),
+                    Visit::Emit(item) => (Some(item), None),
+                    Visit::EmitAndNextRow(item) => (Some(item), Some(false)),
+                    Visit::NextRow => (None, Some(false)),
+                    Visit::Stop => return Ok(false),
+                };
+                if let Some(item) = item
+                    && (V::QUALIFIER || reducer.admits(&item))
+                {
+                    if !V::QUALIFIER {
+                        let col = lane.col();
+                        qualifier.clear();
+                        Escaped::new(&col[row_len..end]).unescape_into(&mut qualifier);
+                    }
+                    let cell = CellRef {
+                        row: &row,
+                        family_rank,
+                        family: lane.family(),
+                        qualifier: &qualifier,
+                        ts: lane.ts(),
+                    };
+                    if reducer.push(cell, item).is_break() {
+                        return Ok(true);
+                    }
+                }
+                if after.is_some() {
+                    next_row = true;
+                    break 'lanes;
+                }
+                lane.fetch()?;
+            }
+        }
+        if next_row {
+            skip_row(lanes, &row_esc)?;
+        }
+    }
+}
+
+/// Reads one key group into `reducer` through its own [`KeyCursor`] (D296 PR 2): the visitor
+/// sees each present key's resolved cells in place, as for a range; `NextRow` skips the rest
+/// of the key without reading it. `true` when the reducer broke.
+fn run_keys<V, R>(
+    ctx: &Ctx<'_>,
+    group: &KeyGroup,
+    visitor: &mut V,
+    reducer: &mut R,
+    stats: &mut ReduceStats,
+) -> Result<bool>
+where
+    V: CellVisitor<Item = R::Item>,
+    R: Reducer,
+{
+    let ReadInput::Keys(keys) = &ctx.input else {
+        unreachable!("a key group comes from a Keys input");
+    };
+    let mut cur = KeyCursor::for_group(
+        ctx.snapshot.clone(),
+        ctx.families.to_vec(),
+        ctx.read.clone(),
+        ctx.now,
+        Arc::clone(keys),
+        group.clone(),
+        Arc::clone(ctx.shared),
+    );
+    stats.groups += 1;
+    let mut row = Vec::new();
+    // Cells come grouped by family: the rank of the last family seen, looked up once.
+    let mut rank: Option<(FamilyId, u32)> = None;
+    while let Some(i) = cur.next_key()? {
+        if ctx.abort.load(Ordering::Relaxed) {
+            return Ok(false);
+        }
+        row.clear();
+        row.extend_from_slice(keys.get(i));
+        if let Some(r) = ctx.resume
+            && r.is(&row)
+            && !cur.resume_in_row(r)?
+        {
+            continue;
+        }
+        stats.rows += 1;
+        loop {
+            let Some(c) = cur.next_cell()? else {
+                break;
+            };
+            stats.cells += 1;
+            let family_rank = match rank {
+                Some((f, r)) if f == c.family => r,
+                _ => {
+                    let r = rank_of(ctx.families, c.family);
+                    rank = Some((c.family, r));
+                    r
+                }
+            };
+            let cell = CellRef {
+                row: &row,
+                family_rank,
+                family: c.family,
+                qualifier: c.qualifier,
+                ts: c.ts,
+            };
+            let (item, next_row) = match visitor.visit(&row, c.family, c.qualifier, c.ts, c.stored)
+            {
+                Visit::Skip => (None, false),
+                Visit::Emit(item) => (Some(item), false),
+                Visit::EmitAndNextRow(item) => (Some(item), true),
+                Visit::NextRow => (None, true),
+                Visit::Stop => return Ok(false),
+            };
+            if let Some(item) = item
+                && reducer.push(cell, item).is_break()
+            {
+                return Ok(true);
+            }
+            if next_row {
+                cur.skip_rest_of_key();
+                break;
+            }
+        }
+    }
+    Ok(false)
+}
+
+/// [`Engine::reduce`](crate::Engine::reduce): [`reduce_into`], then the merged reducer's
+/// [`Reducer::finish`].
+pub(crate) fn reduce<V, R>(
+    snapshot: &Snapshot,
+    table: TableId,
+    families: &[FamilyId],
+    spec: &ReduceSpec,
+    ctx: (Timestamp, bool, &Arc<crate::shard::Shared>),
+    visitor: &(impl Fn() -> V + Sync),
+    reducer: &(impl Fn() -> R + Sync),
+) -> Result<Reduced<R::Item>>
+where
+    V: CellVisitor<Item = R::Item>,
+    R: Reducer,
+{
+    let (acc, stats) = reduce_into(snapshot, table, families, spec, ctx, visitor, reducer)?;
+    Ok(Reduced {
+        items: acc.finish(),
+        stats,
+    })
+}
+
+/// The groups, sequential or fanned out, merged into one reducer and handed back unfinished,
+/// for a caller that takes its result in another form (traversal's `DedupeLimit::into_nodes`,
+/// D304).
+pub(crate) fn reduce_into<V, R>(
+    snapshot: &Snapshot,
+    table: TableId,
+    families: &[FamilyId],
+    spec: &ReduceSpec,
+    (now, may_fan_out, shared): (Timestamp, bool, &Arc<crate::shard::Shared>),
+    visitor: &(impl Fn() -> V + Sync),
+    reducer: &(impl Fn() -> R + Sync),
+) -> Result<(R, ReduceStats)>
+where
+    V: CellVisitor<Item = R::Item>,
+    R: Reducer,
+{
+    let view = Arc::clone(&snapshot.view);
+    let resume = spec
+        .resume
+        .as_ref()
+        .map(|r| Resume::new(table, families, r))
+        .transpose()?;
+    // perf287's N1 on #1264: an excluded start skips its row before the resume could apply.
+    if let (Some((row, _)), ReadInput::Ranges(ranges)) = (&spec.resume, &spec.input)
+        && ranges
+            .iter()
+            .any(|(start, _)| matches!(start, Bound::Excluded(s) if s == row))
+    {
+        return Err(Error::InvalidArgument(
+            "a resume in a row that a range's excluded start skips: start the range at the \
+             row, included"
+                .to_owned(),
+        ));
+    }
+    let groups = work(snapshot, table, &spec.input)?;
+    let abort = AtomicBool::new(false);
+    let ctx = Ctx {
+        snapshot,
+        shared,
+        input: &spec.input,
+        families,
+        read: &spec.read,
+        now,
+        abort: &abort,
+        resume: resume.as_ref(),
+    };
+    let threads = if may_fan_out && spec.parallelism > 1 && groups.len() > 1 {
+        let by_work = match spec.parallel_floor {
+            0 => usize::MAX,
+            floor => {
+                let total: u64 = groups
+                    .iter()
+                    .map(|w| match w {
+                        Work::Range(g) => estimate(&view, families, g),
+                        Work::Keys(k) => k.keys.len() as u64,
+                    })
+                    .sum();
+                usize::try_from(total / floor).unwrap_or(usize::MAX).max(1)
+            }
+        };
+        spec.parallelism.min(groups.len()).min(by_work)
+    } else {
+        1
+    };
+    // One visitor per group. `true` when the reducer broke.
+    let run_into = |w: &Work<'_>, r: &mut R, stats: &mut ReduceStats| -> Result<bool> {
+        let mut v = visitor();
+        match w {
+            Work::Range(g) => run_group(&ctx, g, &mut v, r, stats),
+            Work::Keys(k) => run_keys(&ctx, k, &mut v, r, stats),
+        }
+    };
+    // Fanned out, one reducer per group as well: a group stops only on its own account.
+    let run = |w: &Work<'_>, stats: &mut ReduceStats| -> Result<R> {
+        let mut r = reducer();
+        run_into(w, &mut r, stats)?;
+        Ok(r)
+    };
+    let mut stats = ReduceStats::default();
+    let mut acc = reducer();
+    if threads <= 1 {
+        // Sequentially, every group pushes into the one accumulator (D308 B2). The groups
+        // are in key order, so its pushes are the read's cells in order, exactly as one
+        // scan would push them; a `Break` means the whole read has what it needs, and the
+        // groups after it are not read. No per-group merge.
+        for g in &groups {
+            if run_into(g, &mut acc, &mut stats)? {
+                break;
+            }
+        }
+        stats.threads = 1;
+    } else {
+        let next = AtomicUsize::new(0);
+        let results: Vec<Result<(R, ReduceStats)>> = std::thread::scope(|s| {
+            let workers: Vec<_> = (0..threads)
+                .map(|_| {
+                    s.spawn(|| -> Result<(R, ReduceStats)> {
+                        let mut mine = reducer();
+                        let mut st = ReduceStats::default();
+                        loop {
+                            let i = next.fetch_add(1, Ordering::Relaxed);
+                            let Some(g) = groups.get(i) else {
+                                return Ok((mine, st));
+                            };
+                            match run(g, &mut st) {
+                                Ok(r) => mine.merge(r),
+                                Err(e) => {
+                                    abort.store(true, Ordering::Relaxed);
+                                    return Err(e);
+                                }
+                            }
+                        }
+                    })
+                })
+                .collect();
+            workers
+                .into_iter()
+                // A worker's panic is the caller's, as it would be on the caller's thread.
+                .map(|w| w.join().unwrap_or_else(|p| std::panic::resume_unwind(p)))
+                .collect()
+        });
+        for r in results {
+            let (r, st) = r?;
+            acc.merge(r);
+            stats.add(&st);
+        }
+        stats.threads = threads as u64;
+    }
+    Ok((acc, stats))
+}
+
+/// The `k` items with the smallest scores, ties broken by cell order: a total order, so the
+/// result is the same at any fan-out and in any push order (D296 R1).
+#[derive(Debug, Clone)]
+pub struct TopK<S> {
+    k: usize,
+    /// Sorted ascending by (score, cell); at most `k`.
+    kept: Vec<(S, CellKey)>,
+}
+
+impl<S: Ord> TopK<S> {
+    /// Keeps the `k` smallest scores.
+    pub fn new(k: usize) -> Self {
+        Self {
+            k,
+            kept: Vec::with_capacity(k.min(1024) + 1),
+        }
+    }
+
+    /// Whether `(score, cell)` would enter.
+    fn enters(&self, score: &S, cell: CellRef<'_>) -> bool {
+        if self.k == 0 {
+            return false;
+        }
+        match self.kept.last() {
+            Some((ws, wc)) if self.kept.len() == self.k => (score, cell) < (ws, wc.as_ref()),
+            _ => true,
+        }
+    }
+
+    fn insert(&mut self, score: S, key: CellKey) {
+        if !self.enters(&score, key.as_ref()) {
+            return;
+        }
+        let at = self.kept.partition_point(|(s, k)| (s, k) < (&score, &key));
+        self.kept.insert(at, (score, key));
+        self.kept.truncate(self.k);
+    }
+}
+
+impl<S: Ord + Send> Reducer for TopK<S> {
+    type Item = S;
+
+    fn push(&mut self, cell: CellRef<'_>, item: S) -> ControlFlow<()> {
+        // Only a candidate that enters is copied.
+        if self.enters(&item, cell) {
+            self.insert(item, cell.to_key());
+        }
+        ControlFlow::Continue(())
+    }
+
+    /// A score above the worst kept one, with `k` kept, can't enter whatever its cell; one
+    /// equal to it can, by a smaller cell.
+    #[inline]
+    fn admits(&self, item: &S) -> bool {
+        match self.kept.last() {
+            Some((worst, _)) if self.kept.len() == self.k => item <= worst,
+            _ => self.k > 0,
+        }
+    }
+
+    fn merge(&mut self, other: Self) {
+        for (s, key) in other.kept {
+            self.insert(s, key);
+        }
+    }
+
+    fn finish(self) -> Vec<(CellKey, S)> {
+        self.kept.into_iter().map(|(s, k)| (k, s)).collect()
+    }
+}
+
+/// The first `n` cells in scan order. A group pushes in that order and stops at its own n-th;
+/// merging keeps the `n` smallest cells, which is the sequential first `n` (D296 rev 1.2).
+#[derive(Debug, Clone)]
+pub struct Limit<I> {
+    n: usize,
+    /// In cell order; at most `n`.
+    kept: Vec<(CellKey, I)>,
+}
+
+impl<I> Limit<I> {
+    /// Keeps the first `n`.
+    pub fn new(n: usize) -> Self {
+        Self {
+            n,
+            kept: Vec::new(),
+        }
+    }
+}
+
+impl<I: Send> Reducer for Limit<I> {
+    type Item = I;
+
+    fn push(&mut self, cell: CellRef<'_>, item: I) -> ControlFlow<()> {
+        if self.kept.len() >= self.n {
+            return ControlFlow::Break(());
+        }
+        // A group pushes in cell order, so this appends.
+        debug_assert!(self.kept.last().is_none_or(|(k, _)| k.as_ref() < cell));
+        self.kept.push((cell.to_key(), item));
+        if self.kept.len() >= self.n {
+            ControlFlow::Break(())
+        } else {
+            ControlFlow::Continue(())
+        }
+    }
+
+    fn merge(&mut self, other: Self) {
+        let mut all = std::mem::take(&mut self.kept);
+        all.extend(other.kept);
+        all.sort_by(|a, b| a.0.cmp(&b.0));
+        all.truncate(self.n);
+        self.kept = all;
+    }
+
+    fn finish(self) -> Vec<(CellKey, I)> {
+        self.kept
+    }
+}
+
+/// D308: the per-item overhead in a budget's size, standing for the timestamp (8 bytes), the
+/// family and the item's framing. Stable from 1.1.0.
+pub const BUDGET_ITEM_OVERHEAD: u64 = 16;
+
+/// The first column past a [`ByteBudget`]'s prefix: its first cell, and its size as the
+/// first item of a page (every version the read selects, plus the row key).
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Overflow {
+    /// The column's first cell.
+    pub key: CellKey,
+    /// Its size, the row key included: a budget of this size returns it whole when a read
+    /// resumes at it.
+    pub bytes: u64,
+}
+
+/// A [`ByteBudget`]'s result.
+#[derive(Debug)]
+#[non_exhaustive]
+pub struct BytePage<I> {
+    /// The kept cells, in cell order.
+    pub items: Vec<(CellKey, I)>,
+    /// Their size by D308's rule: each item's own size, plus each row's key once.
+    pub bytes: u64,
+    /// The first column that didn't fit, when the budget cut the read.
+    pub overflow: Option<Overflow>,
+    /// Whether [`ByteBudget::with_rows`]'s row limit stopped the read first: true only when
+    /// a row followed the `n`-th, so an input of exactly `n` rows gives `false` (the read ran
+    /// out, with nothing to resume). Then `overflow` is `None`.
+    pub cut_by_rows: bool,
+}
+
+struct Kept<I> {
+    key: CellKey,
+    item: I,
+    /// The item's own size, without its row key.
+    size: u64,
+}
+
+/// The longest prefix of the read's cells, in cell order, whose size is at most `budget`
+/// bytes (D308): `Limit` measured in bytes, cut at column boundaries. An item's own size is
+/// `size(cell, item)` (by D308, its qualifier, its value and [`BUDGET_ITEM_OVERHEAD`]); each
+/// row's key counts once, on the row's first kept cell.
+///
+/// A column (every version the read selects) is admitted whole or not at all: the read
+/// pushes one cell per version, so a column that doesn't fit is rolled back, and its
+/// remaining versions are counted into the [`Overflow`] before the group breaks on the next
+/// column. A group stops on its own budget (D296 rev 1.2); `merge` concatenates and cuts the
+/// global prefix again, which is exact: a row never spans two groups, so each cell's size is
+/// the same in its group as in the whole read, and a group's share of the global prefix is a
+/// prefix of its own cells that its budget already covered.
+pub struct ByteBudget<I, F, L: RowLimit = NoRowLimit> {
+    budget: u64,
+    size: F,
+    kept: Vec<Kept<I>>,
+    used: u64,
+    /// The open column's first cell, and its start in `kept`.
+    open: Option<CellKey>,
+    open_at: usize,
+    overflow: Option<Overflow>,
+    /// D308's N4 (#1280): a scan's `limit` bounds the read too. [`NoRowLimit`] (the default,
+    /// zero-sized) compiles every row-limit branch away, so a read without one runs exactly
+    /// the budget's own code (#1284: an in-struct limit cost scan-budget 20 instructions per
+    /// cell, mostly codegen around fields it never used).
+    limit: L,
+}
+
+/// A [`ByteBudget`]'s row limit: [`NoRowLimit`] or [`RowsLimit`].
+pub trait RowLimit: Send {
+    /// Whether this limits rows at all; `false` compiles the limit's branches away.
+    const LIMITED: bool;
+    /// The most rows the read keeps.
+    fn max_rows(&self) -> usize;
+    /// Rows with a kept cell (this group's own account until `cut`).
+    fn kept_rows(&self) -> usize;
+    /// Sets the kept-row count.
+    fn set_kept_rows(&mut self, n: usize);
+    /// Whether more rows followed the limit's last.
+    fn full(&self) -> bool;
+    /// Records whether more rows followed the limit's last.
+    fn set_full(&mut self, full: bool);
+}
+
+/// No row limit: the budget alone stops the read.
+#[derive(Debug, Clone, Copy, Default)]
+pub struct NoRowLimit;
+
+impl RowLimit for NoRowLimit {
+    const LIMITED: bool = false;
+    fn max_rows(&self) -> usize {
+        usize::MAX
+    }
+    fn kept_rows(&self) -> usize {
+        0
+    }
+    fn set_kept_rows(&mut self, _: usize) {}
+    fn full(&self) -> bool {
+        false
+    }
+    fn set_full(&mut self, _: bool) {}
+}
+
+/// At most `n` rows ([`ByteBudget::with_rows`]).
+#[derive(Debug, Clone, Copy)]
+pub struct RowsLimit {
+    n: usize,
+    kept: usize,
+    full: bool,
+}
+
+impl RowLimit for RowsLimit {
+    const LIMITED: bool = true;
+    fn max_rows(&self) -> usize {
+        self.n
+    }
+    fn kept_rows(&self) -> usize {
+        self.kept
+    }
+    fn set_kept_rows(&mut self, n: usize) {
+        self.kept = n;
+    }
+    fn full(&self) -> bool {
+        self.full
+    }
+    fn set_full(&mut self, full: bool) {
+        self.full = full;
+    }
+}
+
+impl<I, F, L: RowLimit> std::fmt::Debug for ByteBudget<I, F, L> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("ByteBudget")
+            .field("budget", &self.budget)
+            .field("kept", &self.kept.len())
+            .field("used", &self.used)
+            .field("overflow", &self.overflow)
+            .finish()
+    }
+}
+
+impl<I, F: Fn(&CellRef<'_>, &I) -> u64> ByteBudget<I, F> {
+    /// Keeps the in-budget prefix, an item's own size given by `size`.
+    pub fn new(budget: u64, size: F) -> Self {
+        Self {
+            budget,
+            size,
+            kept: Vec::new(),
+            used: 0,
+            open: None,
+            open_at: 0,
+            overflow: None,
+            limit: NoRowLimit,
+        }
+    }
+
+    /// Also stops after `n` rows (a scan's `limit`; #1280, perf287's N4 on #1281): whichever
+    /// of the budget and the row limit stops the read first wins, and the read reads no
+    /// further than that. A group stops at its own `n`-th row, which keeps everything the
+    /// merged prefix needs from it, as `Limit` does.
+    pub fn with_rows(self, n: usize) -> ByteBudget<I, F, RowsLimit> {
+        ByteBudget {
+            budget: self.budget,
+            size: self.size,
+            kept: self.kept,
+            used: self.used,
+            open: self.open,
+            open_at: self.open_at,
+            overflow: self.overflow,
+            limit: RowsLimit {
+                n,
+                kept: 0,
+                full: false,
+            },
+        }
+    }
+}
+
+impl<I, F: Fn(&CellRef<'_>, &I) -> u64, L: RowLimit> ByteBudget<I, F, L> {
+    /// The result: the kept cells, their size, and the first column that didn't fit.
+    pub fn into_page(mut self) -> BytePage<I> {
+        self.cut();
+        BytePage {
+            bytes: self.used,
+            overflow: self.overflow,
+            cut_by_rows: self.limit.full(),
+            items: self.kept.into_iter().map(|k| (k.key, k.item)).collect(),
+        }
+    }
+
+    /// Re-cuts `kept` (in cell order) to the global prefix: columns admitted whole while
+    /// they fit, and nothing at or past a recorded overflow. With a row limit, also no row
+    /// past the limit's last.
+    fn cut(&mut self) {
+        let mut used = 0u64;
+        let mut keep = 0;
+        let mut i = 0;
+        let mut rows = 0usize;
+        let mut hit_rows = false;
+        let mut first_out: Option<Overflow> = None;
+        while i < self.kept.len() {
+            if let Some(o) = &self.overflow
+                && self.kept[i].key >= o.key
+            {
+                break;
+            }
+            // The column starting at `i`.
+            let mut j = i + 1;
+            while j < self.kept.len() && same_column(&self.kept[j].key, &self.kept[i].key) {
+                j += 1;
+            }
+            let row = &self.kept[i].key.row;
+            let new_row = keep == 0 || self.kept[keep - 1].key.row != *row;
+            if L::LIMITED && new_row && rows == self.limit.max_rows() {
+                // The row limit stops first: what follows is past it.
+                hit_rows = true;
+                break;
+            }
+            let own: u64 = self.kept[i..j].iter().map(|k| k.size).sum();
+            let charge = if new_row { row.len() as u64 } else { 0 };
+            if used + own + charge > self.budget {
+                first_out = Some(Overflow {
+                    key: self.kept[i].key.clone(),
+                    bytes: own + row.len() as u64,
+                });
+                break;
+            }
+            used += own + charge;
+            if L::LIMITED {
+                rows += usize::from(new_row);
+            }
+            keep = j;
+            i = j;
+        }
+        let mut overflow = match (self.overflow.take(), first_out) {
+            (Some(a), Some(b)) => Some(if a.key <= b.key { a } else { b }),
+            (a, b) => a.or(b),
+        };
+        if L::LIMITED {
+            // An overflow in a row past the limit's last: the row limit stopped the read
+            // first.
+            let last_row = keep.checked_sub(1).map(|k| self.kept[k].key.row.as_slice());
+            if let Some(o) = &overflow
+                && rows == self.limit.max_rows()
+                && last_row != Some(o.key.row.as_slice())
+            {
+                overflow = None;
+                hit_rows = true;
+            }
+            // The budget stopping first (an overflow within the limit's rows) wins over a
+            // group's own row limit.
+            let full = overflow.is_none() && (hit_rows || self.limit.full());
+            self.limit.set_full(full);
+            self.limit.set_kept_rows(rows);
+        }
+        self.overflow = overflow;
+        self.kept.truncate(keep);
+        self.used = used;
+    }
+}
+
+fn same_column(a: &CellKey, b: &CellKey) -> bool {
+    a.row == b.row && a.family_rank == b.family_rank && a.qualifier == b.qualifier
+}
+
+impl<I: Send, F: Fn(&CellRef<'_>, &I) -> u64 + Send, L: RowLimit> Reducer for ByteBudget<I, F, L> {
+    type Item = I;
+
+    fn push(&mut self, cell: CellRef<'_>, item: I) -> ControlFlow<()> {
+        let same = self.open.as_ref().is_some_and(|o| {
+            o.row == cell.row && o.family_rank == cell.family_rank && o.qualifier == cell.qualifier
+        });
+        let size = (self.size)(&cell, &item);
+        if let Some(o) = &mut self.overflow {
+            // The overflow column's later versions count into its size; the next column ends
+            // the group (B1).
+            if same {
+                o.bytes += size;
+                return ControlFlow::Continue(());
+            }
+            return ControlFlow::Break(());
+        }
+        // At the row limit, a new row ends the group (a cell of the open column never starts
+        // one).
+        if L::LIMITED
+            && !same
+            && self.limit.kept_rows() == self.limit.max_rows()
+            && self.kept.last().is_none_or(|k| k.key.row != cell.row)
+        {
+            self.limit.set_full(true);
+            return ControlFlow::Break(());
+        }
+        if !same {
+            self.open = Some(cell.to_key());
+            self.open_at = self.kept.len();
+        }
+        // A group pushes in cell order, so this appends.
+        debug_assert!(self.kept.last().is_none_or(|k| k.key.as_ref() < cell));
+        let new_row = self.kept.last().is_none_or(|k| k.key.row != cell.row);
+        let charge = if new_row { cell.row.len() as u64 } else { 0 };
+        if self.used + size + charge <= self.budget {
+            self.used += size + charge;
+            if L::LIMITED && new_row {
+                self.limit.set_kept_rows(self.limit.kept_rows() + 1);
+            }
+            self.kept.push(Kept {
+                key: cell.to_key(),
+                item,
+                size,
+            });
+            return ControlFlow::Continue(());
+        }
+        // The column doesn't fit: roll back its earlier versions.
+        let rolled: Vec<Kept<I>> = self.kept.drain(self.open_at..).collect();
+        let own: u64 = rolled.iter().map(|k| k.size).sum::<u64>() + size;
+        let open = self.open.clone().expect("a column is open");
+        self.overflow = Some(Overflow {
+            bytes: own + open.row.len() as u64,
+            key: open,
+        });
+        let kept_row = self.kept.last().map(|k| k.key.row.as_slice());
+        let rolled_charge = rolled
+            .first()
+            .filter(|k| kept_row != Some(k.key.row.as_slice()))
+            .map_or(0, |k| k.key.row.len() as u64);
+        self.used -= rolled.iter().map(|k| k.size).sum::<u64>() + rolled_charge;
+        // A rolled-back column that opened its row takes the row back out of the count.
+        if L::LIMITED
+            && rolled
+                .first()
+                .is_some_and(|k| kept_row != Some(k.key.row.as_slice()))
+        {
+            self.limit.set_kept_rows(self.limit.kept_rows() - 1);
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn merge(&mut self, other: Self) {
+        if L::LIMITED {
+            self.limit.set_full(self.limit.full() || other.limit.full());
+        }
+        let mut all = std::mem::take(&mut self.kept);
+        all.extend(other.kept);
+        all.sort_by(|a, b| a.key.cmp(&b.key));
+        self.kept = all;
+        self.overflow = match (self.overflow.take(), other.overflow) {
+            (Some(a), Some(b)) => Some(if a.key <= b.key { a } else { b }),
+            (a, b) => a.or(b),
+        };
+        self.cut();
+        self.open = None;
+        self.open_at = self.kept.len();
+    }
+
+    fn finish(self) -> Vec<(CellKey, I)> {
+        self.into_page().items
+    }
+}
+
+/// One item per key the item gives (for example a traversal's destination node), the one
+/// from the first cell in scan order when a key repeats; keys in `exclude` are dropped
+/// (#489's visited set). Order-independent, so the same at any fan-out (D296 S2).
+pub struct Dedupe<I, K, F> {
+    key_of: F,
+    exclude: Option<Arc<std::collections::HashSet<K>>>,
+    kept: std::collections::HashMap<K, (CellKey, I)>,
+}
+
+impl<I, K, F> std::fmt::Debug for Dedupe<I, K, F> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Dedupe")
+            .field("kept", &self.kept.len())
+            .finish_non_exhaustive()
+    }
+}
+
+impl<I, K: std::hash::Hash + Eq, F: Fn(&I) -> K> Dedupe<I, K, F> {
+    /// One item per `key_of(item)`, none whose key is in `exclude`.
+    pub fn new(key_of: F, exclude: Option<Arc<std::collections::HashSet<K>>>) -> Self {
+        Self {
+            key_of,
+            exclude,
+            kept: std::collections::HashMap::new(),
+        }
+    }
+
+    fn offer(&mut self, cell: CellRef<'_>, item: I) {
+        let key = (self.key_of)(&item);
+        if self.exclude.as_ref().is_some_and(|x| x.contains(&key)) {
+            return;
+        }
+        match self.kept.entry(key) {
+            std::collections::hash_map::Entry::Occupied(mut o) => {
+                if cell < o.get().0.as_ref() {
+                    o.insert((cell.to_key(), item));
+                }
+            }
+            std::collections::hash_map::Entry::Vacant(v) => {
+                v.insert((cell.to_key(), item));
+            }
+        }
+    }
+}
+
+impl<I, K, F> Reducer for Dedupe<I, K, F>
+where
+    I: Send,
+    K: std::hash::Hash + Eq + Send + Sync,
+    F: Fn(&I) -> K + Send,
+{
+    type Item = I;
+
+    fn push(&mut self, cell: CellRef<'_>, item: I) -> ControlFlow<()> {
+        self.offer(cell, item);
+        ControlFlow::Continue(())
+    }
+
+    fn merge(&mut self, other: Self) {
+        for (_, (cell, item)) in other.kept {
+            self.offer(cell.as_ref(), item);
+        }
+    }
+
+    fn finish(self) -> Vec<(CellKey, I)> {
+        let mut out: Vec<_> = self.kept.into_values().collect();
+        out.sort_by(|a, b| a.0.cmp(&b.0));
+        out
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn cell(row: &'static str, rank: u32, q: &'static str, ts: Timestamp) -> CellRef<'static> {
+        CellRef {
+            row: row.as_bytes(),
+            family_rank: rank,
+            family: FamilyId(rank),
+            qualifier: q.as_bytes(),
+            ts,
+        }
+    }
+
+    fn rows<I>(items: &[(CellKey, I)]) -> Vec<(&[u8], u32, &[u8], Timestamp)> {
+        items
+            .iter()
+            .map(|(k, _)| {
+                (
+                    k.row.as_slice(),
+                    k.family_rank,
+                    k.qualifier.as_slice(),
+                    k.ts,
+                )
+            })
+            .collect()
+    }
+
+    #[test]
+    fn cells_order_as_a_scan_does_family_before_qualifier_newest_version_first() {
+        assert!(
+            cell("r", 0, "z", 1) < cell("r", 1, "a", 1),
+            "family rank before qualifier"
+        );
+        assert!(
+            cell("r", 0, "q", 9) < cell("r", 0, "q", 3),
+            "newest version first"
+        );
+        assert!(cell("a", 9, "z", 0) < cell("b", 0, "a", 9), "row first");
+    }
+
+    #[test]
+    fn top_k_breaks_equal_scores_by_cell_order_in_any_merge_order() {
+        // Equal scores across "groups": the first cells in scan order among the ties win,
+        // including two versions of one column and the same qualifier in two families.
+        let cells = [
+            (cell("r3", 0, "a", 1), 1u32),
+            (cell("r1", 1, "a", 1), 1),
+            (cell("r1", 0, "b", 1), 1),
+            (cell("r1", 0, "b", 5), 1),
+            (cell("r2", 0, "a", 1), 1),
+            (cell("r0", 0, "z", 1), 2),
+        ];
+        let mut seq = TopK::new(3);
+        for (c, s) in cells {
+            let _ = seq.push(c, s);
+        }
+        let want = seq.finish();
+        assert_eq!(
+            rows(&want),
+            vec![
+                (&b"r1"[..], 0, &b"b"[..], 5),
+                (b"r1", 0, b"b", 1),
+                (b"r1", 1, b"a", 1)
+            ]
+        );
+        for split in 0..=cells.len() {
+            let (a, b) = cells.split_at(split);
+            let (mut x, mut y) = (TopK::new(3), TopK::new(3));
+            for (c, s) in a {
+                let _ = x.push(*c, *s);
+            }
+            for (c, s) in b.iter().rev() {
+                let _ = y.push(*c, *s);
+            }
+            y.merge(x);
+            assert_eq!(y.finish(), want, "split at {split}");
+        }
+    }
+
+    /// #1283 lever 1: `admits` refuses exactly the scores `push` would drop whatever their
+    /// cell (above the worst kept, with `k` kept), and admits a tie, which `push` settles
+    /// by cell. Dropping a refused item before its push keeps the same result.
+    #[test]
+    fn top_k_admits_every_score_that_could_enter() {
+        assert!(!TopK::<u32>::new(0).admits(&0));
+        let mut t = TopK::new(2);
+        assert!(t.admits(&9), "room left");
+        let _ = t.push(cell("b", 0, "q", 1), 5u32);
+        assert!(t.admits(&9), "room left");
+        let _ = t.push(cell("c", 0, "q", 1), 7);
+        assert!(t.admits(&6) && t.admits(&7), "below and tying the worst");
+        assert!(!t.admits(&8), "above the worst");
+        // A tie with a smaller cell enters; it was admitted.
+        let _ = t.push(cell("a", 0, "q", 1), 7);
+        assert_eq!(
+            rows(&t.clone().finish()),
+            vec![(&b"b"[..], 0, &b"q"[..], 1), (b"a", 0, b"q", 1)]
+        );
+        // Every refused score is one `push` drops.
+        let before = t.clone().finish();
+        let _ = t.push(cell("0", 0, "", 0), 8);
+        assert_eq!(t.finish(), before);
+    }
+
+    #[test]
+    fn limit_keeps_the_first_n_cells_in_scan_order_whichever_group_finishes_first() {
+        // Within a row, family 0's "z" comes before family 1's "a" (ctr274's case).
+        let mut early = Limit::new(2);
+        assert!(early.push(cell("a", 0, "z", 1), 1).is_continue());
+        assert!(early.push(cell("a", 1, "a", 1), 2).is_break());
+        let mut late = Limit::new(2);
+        let _ = late.push(cell("m", 0, "a", 1), 3);
+        let _ = late.push(cell("n", 0, "a", 1), 4);
+        late.merge(early);
+        let got: Vec<i32> = late.finish().into_iter().map(|(_, i)| i).collect();
+        assert_eq!(got, vec![1, 2]);
+    }
+
+    #[test]
+    fn dedupe_keeps_the_first_cell_per_key_and_drops_excluded_keys() {
+        fn key(i: &u32) -> u32 {
+            *i
+        }
+        let exclude = Arc::new(std::collections::HashSet::from([9u32]));
+        let mut a = Dedupe::new(key as fn(&u32) -> u32, Some(Arc::clone(&exclude)));
+        let _ = a.push(cell("r2", 0, "q", 1), 7);
+        let _ = a.push(cell("r1", 0, "q", 1), 9);
+        let mut b = Dedupe::new(key as fn(&u32) -> u32, Some(exclude));
+        let _ = b.push(cell("r1", 0, "q", 1), 7);
+        a.merge(b);
+        let got = a.finish();
+        assert_eq!(got.len(), 1);
+        assert_eq!((got[0].0.row.as_slice(), got[0].1), (&b"r1"[..], 7));
+    }
+
+    /// D308: cut a read's cells into groups at row boundaries, push each group into its own
+    /// budget with its local stop, merge in a random order: the result (cells, size and
+    /// overflow) equals one budget over the whole read, and the hand-cut prefix.
+    /// The whole read's page, and the page of its groups (cut at `cuts`) merged in `plan`'s
+    /// order: each step takes a remaining group by index, merged into the accumulator or the
+    /// accumulator into it.
+    fn pages<F, L>(
+        make: impl Fn() -> ByteBudget<u64, F, L>,
+        cells: &[(CellKey, u64)],
+        cuts: &[usize],
+        plan: &[(usize, bool)],
+    ) -> (BytePage<u64>, BytePage<u64>)
+    where
+        F: Fn(&CellRef<'_>, &u64) -> u64 + Send,
+        L: RowLimit,
+    {
+        let run = |part: &[(CellKey, u64)]| {
+            let mut b = make();
+            for (k, n) in part {
+                if b.push(k.as_ref(), *n).is_break() {
+                    break;
+                }
+            }
+            b
+        };
+        let whole = run(cells).into_page();
+        let mut parts: Vec<_> = cuts.windows(2).map(|w| run(&cells[w[0]..w[1]])).collect();
+        let mut steps = plan.iter();
+        let (first, _) = steps.next().expect("a group at least");
+        let mut acc = parts.swap_remove(*first);
+        for (pick, into_acc) in steps {
+            let next = parts.swap_remove(*pick);
+            if *into_acc {
+                acc.merge(next);
+            } else {
+                let mut other = next;
+                other.merge(acc);
+                acc = other;
+            }
+        }
+        (whole, acc.into_page())
+    }
+
+    #[test]
+    fn a_byte_budget_merges_to_the_sequential_result_in_any_order() {
+        struct Rng(u64);
+        impl Rng {
+            fn below(&mut self, n: u64) -> u64 {
+                self.0 ^= self.0 << 13;
+                self.0 ^= self.0 >> 7;
+                self.0 ^= self.0 << 17;
+                self.0 % n
+            }
+        }
+        let seed = 0x5EED_D308;
+        let mut rng = Rng(seed);
+        // An item's own size: the length it carries.
+        let size = |_: &CellRef<'_>, n: &u64| *n;
+        for case in 0..500 {
+            // Cells in cell order: rows, then two families, qualifiers, and 1 to 3 versions.
+            let mut cells: Vec<(CellKey, u64)> = Vec::new();
+            for r in 0..1 + rng.below(8) {
+                let row = vec![b'r'; 1 + r as usize];
+                for rank in 0..2 {
+                    for q in 0..rng.below(3) {
+                        let versions = 1 + rng.below(3);
+                        for v in 0..versions {
+                            cells.push((
+                                CellKey {
+                                    row: row.clone(),
+                                    family_rank: rank,
+                                    family: FamilyId(rank),
+                                    qualifier: vec![b'q', q as u8],
+                                    ts: 10 - v,
+                                },
+                                1 + rng.below(20),
+                            ));
+                        }
+                    }
+                }
+            }
+            let budget = rng.below(120);
+            // A row limit (a scan's `limit`, #1280) in a third of the cases.
+            let row_limit = (rng.below(3) == 0).then(|| rng.below(5) as usize);
+            // Groups at row boundaries, and the order they merge in.
+            let mut cuts = vec![0];
+            for i in 1..cells.len() {
+                if cells[i].0.row != cells[i - 1].0.row && rng.below(2) == 0 {
+                    cuts.push(i);
+                }
+            }
+            cuts.push(cells.len());
+            let mut plan = Vec::new();
+            for left in (1..cuts.len()).rev() {
+                plan.push((rng.below(left as u64) as usize, rng.below(2) == 0));
+            }
+            let (whole, merged) = match row_limit {
+                None => pages(|| ByteBudget::new(budget, size), &cells, &cuts, &plan),
+                Some(n) => pages(
+                    || ByteBudget::new(budget, size).with_rows(n),
+                    &cells,
+                    &cuts,
+                    &plan,
+                ),
+            };
+            let ctx = format!("seed {seed:#x}, case {case}, budget {budget}, rows {row_limit:?}");
+            assert_eq!(rows(&merged.items), rows(&whole.items), "{ctx}");
+            assert_eq!(merged.bytes, whole.bytes, "{ctx}");
+            assert_eq!(merged.overflow, whole.overflow, "{ctx}");
+            assert_eq!(merged.cut_by_rows, whole.cut_by_rows, "{ctx}");
+            assert!(whole.bytes <= budget, "{ctx}");
+            // The hand-cut prefix: whole columns while they fit and the row limit allows,
+            // each row's key once.
+            let (mut used, mut keep, mut i) = (0u64, 0usize, 0usize);
+            let (mut over, mut cut_rows, mut kept_rows) = (None, false, 0usize);
+            while i < cells.len() {
+                let mut j = i + 1;
+                while j < cells.len() && same_column(&cells[j].0, &cells[i].0) {
+                    j += 1;
+                }
+                let own: u64 = cells[i..j].iter().map(|c| c.1).sum();
+                let row = cells[i].0.row.len() as u64;
+                let new_row = keep == 0 || cells[keep - 1].0.row != cells[i].0.row;
+                if new_row && row_limit == Some(kept_rows) {
+                    cut_rows = true;
+                    break;
+                }
+                let charge = if new_row { row } else { 0 };
+                if used + own + charge > budget {
+                    over = Some((cells[i].0.clone(), own + row));
+                    break;
+                }
+                used += own + charge;
+                kept_rows += usize::from(new_row);
+                keep = j;
+                i = j;
+            }
+            let want: Vec<CellKey> = cells[..keep].iter().map(|c| c.0.clone()).collect();
+            assert_eq!(whole.cut_by_rows, cut_rows, "{ctx}");
+            let got: Vec<CellKey> = whole.items.iter().map(|c| c.0.clone()).collect();
+            assert_eq!(got, want, "{ctx}");
+            assert_eq!(whole.bytes, used, "{ctx}");
+            assert_eq!(whole.overflow.map(|o| (o.key, o.bytes)), over, "{ctx}");
+        }
+    }
+}