pigeonhole-db 1.1.0

data/vendor/sst/src/reader.rs

@@ -0,0 +1,814 @@
+//! Opening an SST and reading its blocks through the block cache.
+
+use std::sync::{Arc, OnceLock};
+
+use pigeonhole_cache::{BlockCache, BlockData, BlockHandle, BlockKey, Priority};
+use pigeonhole_format::Error as FormatError;
+use pigeonhole_format::SstId;
+use pigeonhole_format::block::{Block, BlockAddr, BlockKind, TRAILER_LEN, verify};
+use pigeonhole_format::compress::{Compression, decompress};
+use pigeonhole_format::filter::Filter;
+use pigeonhole_format::manifest::SstMeta;
+use pigeonhole_format::sst::{FOOTER_LEN, Footer, Properties};
+use pigeonhole_io::{BufPool, FileRef, IoBuf};
+
+use crate::Result;
+
+pub(crate) fn corrupt(what: &'static str) -> crate::Error {
+    crate::Error::Format(FormatError::Corrupt { what })
+}
+
+/// What a cache-only read missed (`Error::WouldBlock`, ICR 0014): the read of the file it
+/// needs (one range, or the pieces of a blob record that spans extents) and what to do with
+/// the bytes. The caller starts the read ([`Fetch::start`]: the one already in flight, else
+/// a new one), admits the bytes when it completes ([`Fetch::admit`], which verifies them,
+/// keeps them, and returns a pinned handle when there is one to hold), and reads again.
+pub struct Fetch {
+    file: FileRef,
+    /// Absolute file offset of each piece, and where it goes in the assembled bytes.
+    pieces: Vec<(u64, std::ops::Range<usize>)>,
+    len: usize,
+    then: Then,
+    /// The read, when the cache-only read that missed already submitted it (#709): a data
+    /// or index block a cursor could not read in place ([`Blocks::fetch_now`]).
+    submitted: Option<pigeonhole_io::Completion>,
+}
+
+/// A copy describes the same read but does not share the one in flight: it submits its own.
+impl Clone for Fetch {
+    fn clone(&self) -> Self {
+        Self {
+            file: self.file.clone(),
+            pieces: self.pieces.clone(),
+            len: self.len,
+            then: self.then.clone(),
+            submitted: None,
+        }
+    }
+}
+
+#[derive(Clone)]
+enum Then {
+    /// A block of this kind (checksummed and maybe compressed on disk), into the cache.
+    Block {
+        cache: Arc<BlockCache>,
+        key: BlockKey,
+        kind: BlockKind,
+        priority: Priority,
+        /// For a block that does not fill the cache: the reader's pool its buffers come from
+        /// and go back to (#372).
+        pool: Option<Arc<BufPool>>,
+    },
+    /// An SST's footer, cached as its raw bytes for a cache-only open.
+    Footer {
+        cache: Arc<BlockCache>,
+        key: BlockKey,
+        priority: Priority,
+    },
+    /// A blob extent's header, verified and recorded as such in its reader.
+    BlobHeader {
+        reader: Arc<crate::BlobReader>,
+        index: usize,
+    },
+    /// A blob record, verified and cached.
+    BlobRecord {
+        reader: Arc<crate::BlobReader>,
+        ptr: pigeonhole_format::value::BlobPointer,
+    },
+}
+
+impl std::fmt::Debug for Fetch {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let kind = match &self.then {
+            Then::Block { kind, .. } => format!("{kind:?}"),
+            Then::Footer { .. } => "Footer".into(),
+            Then::BlobHeader { index, .. } => format!("BlobHeader({index})"),
+            Then::BlobRecord { .. } => "BlobRecord".into(),
+        };
+        f.debug_struct("Fetch")
+            .field("offset", &self.pieces.first().map(|p| p.0))
+            .field("len", &self.len)
+            .field("pieces", &self.pieces.len())
+            .field("kind", &kind)
+            .field("submitted", &self.submitted.is_some())
+            .finish()
+    }
+}
+
+impl Fetch {
+    fn one(file: FileRef, offset: u64, len: usize, then: Then) -> Self {
+        Self {
+            file,
+            pieces: vec![(offset, 0..len)],
+            len,
+            then,
+            submitted: None,
+        }
+    }
+
+    pub(crate) fn blob_header(
+        file: FileRef,
+        reader: Arc<crate::BlobReader>,
+        index: usize,
+        offset: u64,
+        len: usize,
+    ) -> Self {
+        Self::one(file, offset, len, Then::BlobHeader { reader, index })
+    }
+
+    pub(crate) fn blob_record(
+        file: FileRef,
+        reader: Arc<crate::BlobReader>,
+        ptr: pigeonhole_format::value::BlobPointer,
+        pieces: Vec<(u64, std::ops::Range<usize>)>,
+        len: usize,
+    ) -> Self {
+        Self {
+            file,
+            pieces,
+            len,
+            then: Then::BlobRecord { reader, ptr },
+            submitted: None,
+        }
+    }
+
+    /// One read of `len` bytes at `offset`. On a direct handle (#403) it covers the aligned
+    /// pages around them, and the buffer it resolves to keeps just them, in place.
+    fn submit_piece(&self, offset: u64, len: usize) -> pigeonhole_io::Completion {
+        submit_bytes(&self.file, offset, len)
+    }
+
+    /// The file offset of the read (its first piece's): two fetches of one block of the same
+    /// file share it.
+    pub fn offset(&self) -> u64 {
+        self.pieces.first().map_or(0, |p| p.0)
+    }
+
+    /// Starts the read: the one already in flight when the cache-only read that missed
+    /// submitted it (a data or index block whose read did not complete at once, #709), else a
+    /// new one ([`Fetch::submit`]).
+    pub fn start(&mut self) -> pigeonhole_io::Completion {
+        self.submitted.take().unwrap_or_else(|| self.submit())
+    }
+
+    /// Submits the read through the VFS's asynchronous reads: one completion that wakes its
+    /// poller once every piece has arrived. Submits a new read even if one is in flight
+    /// ([`Fetch::start`] resumes that one).
+    pub fn submit(&self) -> pigeonhole_io::Completion {
+        if let [(offset, _)] = self.pieces.as_slice() {
+            return self.submit_piece(*offset, self.len);
+        }
+        // Several pieces: each read's continuation copies its bytes into place, and the last
+        // one resolves the joined completion (or the first failure does).
+        struct Join {
+            buf: Option<IoBuf>,
+            left: usize,
+            resolver: Option<pigeonhole_io::Resolver<IoBuf>>,
+        }
+        let (done, resolver) = pigeonhole_io::Completion::pair();
+        let join = Arc::new(std::sync::Mutex::new(Join {
+            // Every piece is copied into its range before the join resolves; a failed piece
+            // drops the buffer, so a reused one never shows its old bytes (#709).
+            buf: Some(IoBuf::reused(self.len)),
+            left: self.pieces.len(),
+            resolver: Some(resolver),
+        }));
+        for (offset, range) in &self.pieces {
+            let (join, range) = (Arc::clone(&join), range.clone());
+            let read = self.submit_piece(*offset, range.len());
+            // The continuation runs when the read completes, whether or not the mapped
+            // completion is kept.
+            drop(read.map(move |r| {
+                let mut j = join
+                    .lock()
+                    .unwrap_or_else(std::sync::PoisonError::into_inner);
+                let finished = match r {
+                    Ok(piece) => {
+                        if let Some(buf) = j.buf.as_mut() {
+                            buf[range].copy_from_slice(&piece);
+                        }
+                        j.left -= 1;
+                        (j.left == 0).then(|| j.buf.take().map(Ok)).flatten()
+                    }
+                    Err(e) => {
+                        j.buf = None;
+                        Some(Err(e))
+                    }
+                };
+                if let Some(result) = finished
+                    && let Some(resolver) = j.resolver.take()
+                {
+                    drop(j);
+                    resolver.resolve(result);
+                }
+                Ok(())
+            }));
+        }
+        done
+    }
+
+    /// Whether what was fetched is kept now, so the read that missed will find it: the block
+    /// or record in the cache, or the header recorded as verified. A block is not kept when
+    /// the cache keeps nothing (capacity 0, or a block larger than a cache shard); the caller
+    /// then reads synchronously instead of fetching again.
+    pub fn is_kept(&self) -> bool {
+        match &self.then {
+            Then::Block { cache, key, .. } | Then::Footer { cache, key, .. } => {
+                cache.get(*key).is_some()
+            }
+            Then::BlobHeader { reader, index } => reader.inner.is_verified(*index),
+            Then::BlobRecord { reader, ptr } => reader.inner.cached(ptr).is_some(),
+        }
+    }
+
+    /// A buffer of `len` bytes for this fetch's bytes to be copied into (a piece of a merged
+    /// readahead run, copied whole): from the reader's pool for a block that does not fill
+    /// the cache, as its decoded bytes are (#372), else this thread's reused buffers (#709).
+    pub(crate) fn buffer(&self, len: usize) -> IoBuf {
+        match &self.then {
+            Then::Block {
+                pool: Some(pool), ..
+            } => pool.take(len),
+            _ => IoBuf::reused(len),
+        }
+    }
+
+    /// Verifies and keeps the bytes `submit` read, returning a pinned handle when there is
+    /// one (a block or record; hold it until the read that missed has run again).
+    pub fn admit(&self, buf: IoBuf) -> Result<Option<BlockHandle>> {
+        Ok(match &self.then {
+            Then::Block {
+                cache,
+                key,
+                kind,
+                priority,
+                pool,
+            } => Some(cache.insert(*key, decode_physical(buf, *kind, pool.as_ref())?, *priority)),
+            Then::Footer {
+                cache,
+                key,
+                priority,
+            } => Some(cache.insert(*key, BlockData::from(buf.to_vec()), *priority)),
+            Then::BlobHeader { reader, index } => {
+                reader.inner.check_header(*index, &buf)?;
+                None
+            }
+            Then::BlobRecord { reader, ptr } => Some(reader.inner.admit_record(ptr, buf)?),
+        })
+    }
+}
+
+/// The aligned read around `len` bytes at `offset`: its start, its length, and where the bytes
+/// begin inside it.
+fn aligned(offset: u64, len: usize, align: usize) -> (u64, usize, usize) {
+    let a = align as u64;
+    let start = offset / a * a;
+    let end = (offset + len as u64).div_ceil(a) * a;
+    (start, (end - start) as usize, (offset - start) as usize)
+}
+
+/// Reads `len` bytes at `offset` into a buffer the block cache can keep. On a direct handle
+/// (#403) the read covers the aligned pages around them and the buffer keeps just them, in
+/// place: no copy (a block's offset and length are not aligned on disk).
+pub(crate) fn read_bytes(file: &FileRef, offset: u64, len: usize) -> Result<IoBuf> {
+    match file.direct_align() {
+        None => {
+            // Reused, not zeroed (#709): the read fills all of it or fails.
+            let mut buf = IoBuf::reused(len);
+            file.read_at(&mut buf, offset)?;
+            Ok(buf)
+        }
+        Some(a) => {
+            let (start, alen, head) = aligned(offset, len, a);
+            let mut buf = IoBuf::zeroed(alen);
+            file.read_at(&mut buf, start)?;
+            buf.keep(head..head + len);
+            Ok(buf)
+        }
+    }
+}
+
+/// Submits a read of `len` bytes at `offset`, resolving to a buffer of just them. On a direct
+/// handle (#403) it covers the aligned pages around them, and the buffer keeps just the bytes,
+/// in place (as [`read_bytes`]).
+fn submit_bytes(file: &FileRef, offset: u64, len: usize) -> pigeonhole_io::Completion {
+    match file.direct_align() {
+        // A registered slot where the backend has one (#402): slots are page-aligned, so they
+        // serve a direct handle's aligned read too.
+        None => file.submit_read(file.read_buf(len), offset),
+        Some(a) => {
+            let (start, alen, head) = aligned(offset, len, a);
+            file.submit_read(file.read_buf(alen), start).map(move |r| {
+                r.map(|mut b| {
+                    b.keep(head..head + len);
+                    b
+                })
+            })
+        }
+    }
+}
+
+/// Reads exactly `out.len()` bytes at `offset` into `out`, any slice: on a direct handle
+/// through an aligned bounce buffer (a copy; for small or piecewise reads).
+pub(crate) fn read_into(file: &FileRef, out: &mut [u8], offset: u64) -> Result<()> {
+    if file.direct_align().is_none() {
+        return Ok(file.read_at(out, offset)?);
+    }
+    let buf = read_bytes(file, offset, out.len())?;
+    out.copy_from_slice(&buf);
+    Ok(())
+}
+
+/// Writes `bytes` at `offset`, any range: on a direct handle (#403) through aligned pages,
+/// reading back the pages it only partly covers (the bytes around the range stay as they
+/// are). For writers without a page-sized buffer of their own (the blob writer).
+pub(crate) fn write_any(file: &FileRef, bytes: &[u8], offset: u64) -> Result<()> {
+    let Some(a) = file.direct_align() else {
+        return Ok(file.write_at(bytes, offset)?);
+    };
+    if bytes.is_empty() {
+        return Ok(());
+    }
+    let (start, len, head) = aligned(offset, bytes.len(), a);
+    let mut buf = IoBuf::zeroed(len);
+    if head != 0 {
+        file.read_at(&mut buf[..a], start)?;
+    }
+    let last = len - a;
+    if !(head + bytes.len()).is_multiple_of(a) && !(last == 0 && head != 0) {
+        file.read_at(&mut buf[last..], start + last as u64)?;
+    }
+    buf[head..head + bytes.len()].copy_from_slice(bytes);
+    Ok(file.write_at(&buf, start)?)
+}
+
+/// Writes `bytes` (a whole number of aligned pages, or the last pages of an SST padded with
+/// zeros) at an aligned `offset` from an aligned copy, as a direct handle needs.
+pub(crate) fn write_padded(file: &FileRef, bytes: &[u8], offset: u64, align: usize) -> Result<()> {
+    let mut buf = IoBuf::zeroed(bytes.len().div_ceil(align) * align);
+    buf[..bytes.len()].copy_from_slice(bytes);
+    Ok(file.write_at(&buf, offset)?)
+}
+
+/// Checks a physical block and turns it into what the cache holds: the logical block. An
+/// uncompressed block keeps its I/O buffer (trimmed of the trailer); a compressed one is
+/// decompressed into an exact-size heap buffer, or, for a block that does not fill the cache,
+/// into a buffer from `pool` that goes back to it when the block is dropped (#372).
+pub(crate) fn decode_physical(
+    mut buf: IoBuf,
+    kind: BlockKind,
+    pool: Option<&Arc<BufPool>>,
+) -> Result<BlockData> {
+    match decode_slice(&buf, kind, pool)? {
+        Some(data) => Ok(data),
+        None => {
+            let n = buf.len() - TRAILER_LEN;
+            buf.resize(n);
+            // Kept by the cache: a registered buffer slot goes back to its pool (#402), and a
+            // reused buffer larger than the block needs is copied (#709).
+            Ok(BlockData::Io(buf.detached()))
+        }
+    }
+}
+
+/// [`decode_physical`] over borrowed bytes: the decompressed block, or `None` if the block
+/// is stored uncompressed (its logical bytes are the payload, which the caller keeps).
+fn decode_slice(
+    physical: &[u8],
+    kind: BlockKind,
+    pool: Option<&Arc<BufPool>>,
+) -> Result<Option<BlockData>> {
+    let (trailer, payload) = verify(physical)?;
+    if trailer.kind != kind {
+        return Err(corrupt("block kind"));
+    }
+    let len = trailer.uncompressed_len as usize;
+    match (trailer.compression, pool) {
+        (Compression::None, _) => Ok(None),
+        // A recycled buffer holds a previous block's bytes: `decompress` fails unless the
+        // codec wrote all `len` of them, so none of those reach a reader.
+        (codec, Some(pool)) => {
+            let mut out = pool.take(len);
+            decompress(codec, payload, &mut out)?;
+            Ok(Some(BlockData::Io(out)))
+        }
+        // A block the cache keeps: a new buffer of exactly its length, charged as such.
+        (codec, None) => {
+            let mut out = vec![0; len];
+            decompress(codec, payload, &mut out)?;
+            Ok(Some(BlockData::from(out)))
+        }
+    }
+}
+
+/// Buffers a reader's pool keeps (#372): compaction's readahead of 4 blocks, the block its
+/// cursor is on, and the one being decoded.
+const RECYCLED: usize = 6;
+
+/// The open state of one SST; see [`crate::SstReader`].
+pub(crate) struct Reader {
+    pub(crate) blocks: Blocks,
+    pub(crate) top: BlockHandle,
+    row_filter: Option<Filter<BlockHandle>>,
+    column_filter: Option<Filter<BlockHandle>>,
+    pub(crate) properties: Properties,
+    /// Bytes of the top index and filters, counted in the cache's pinned bytes while this
+    /// reader is open (D215).
+    pinned: usize,
+}
+
+impl Drop for Reader {
+    fn drop(&mut self) {
+        self.blocks.cache.sub_pinned(self.pinned);
+    }
+}
+
+/// Block access for one SST: the file, where the SST sits in it, and the cache.
+pub(crate) struct Blocks {
+    pub(crate) file: FileRef,
+    pub(crate) id: SstId,
+    /// Absolute file offset of the SST.
+    base: u64,
+    /// End of the block area (start of the footer).
+    limit: u64,
+    pub(crate) cache: Arc<BlockCache>,
+    /// Hands out unshared handles for reads that must not fill the cache.
+    uncached: Arc<BlockCache>,
+    /// Buffers for the blocks of reads that do not fill the cache (#372): made on the first
+    /// such read, so a reader that only serves cached reads holds none.
+    recycled: OnceLock<Arc<BufPool>>,
+}
+
+impl std::fmt::Debug for Reader {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("SstReader")
+            .field("id", &self.blocks.id)
+            .field("len", &self.len_bytes())
+            .field("entries", &self.properties.entries)
+            .field("filters", &self.row_filter.is_some())
+            .finish()
+    }
+}
+
+impl Reader {
+    pub(crate) fn open(
+        file: FileRef,
+        meta: &SstMeta,
+        cache: Arc<BlockCache>,
+        priority: Priority,
+        cache_only: bool,
+    ) -> Result<Self> {
+        if meta.len < FOOTER_LEN as u64 || meta.len > meta.extent.len() {
+            return Err(corrupt("sst length"));
+        }
+        let base = meta.extent.offset();
+        let limit = meta.len - FOOTER_LEN as u64;
+        let mut tail = [0; FOOTER_LEN];
+        if cache_only {
+            // The footer goes through the cache only here, so a cache-only open can fetch it
+            // and run again; a sync open reads it directly, as before.
+            let key = BlockKey {
+                file: crate::sst_cache_file(meta.id),
+                offset: limit,
+            };
+            match cache.get(key) {
+                Some(h) if h.len() == FOOTER_LEN => tail.copy_from_slice(&h),
+                _ => {
+                    return Err(crate::Error::WouldBlock(Box::new(Fetch::one(
+                        file,
+                        base + limit,
+                        FOOTER_LEN,
+                        Then::Footer {
+                            cache,
+                            key,
+                            priority,
+                        },
+                    ))));
+                }
+            }
+        } else {
+            read_into(&file, &mut tail, base + limit)?;
+        }
+        let footer = Footer::decode(&tail)?;
+        let blocks = Blocks {
+            file,
+            id: meta.id,
+            base,
+            limit,
+            cache,
+            uncached: Arc::new(BlockCache::disabled()),
+            recycled: OnceLock::new(),
+        };
+        // A cache-only open reads only from the cache, and fills it with what it fetches
+        // (the properties block too), so its retry finds everything.
+        let read = |addr: BlockAddr, kind: BlockKind, fill: bool| {
+            if cache_only {
+                blocks.read_block_cache_only(addr, kind, priority)
+            } else {
+                blocks.read_block(addr, kind, fill, priority)
+            }
+        };
+        let top = read(footer.top_index, BlockKind::TopIndex, true)?;
+        // Checked once here, so every cursor can trust the top index's tables.
+        Block::new(top.clone())?.validate()?;
+        // Each filter, and the bytes it pins.
+        let filter = |addr: BlockAddr| -> Result<(Option<Filter<BlockHandle>>, usize)> {
+            if addr.len == 0 {
+                return Ok((None, 0));
+            }
+            let h = read(addr, BlockKind::Filter, true)?;
+            let charge = h.charge();
+            Ok((Some(Filter::new(h)?), charge))
+        };
+        let (row_filter, row_bytes) = filter(footer.row_filter)?;
+        let (column_filter, column_bytes) = filter(footer.column_filter)?;
+        let props = read(footer.properties, BlockKind::Properties, false)?;
+        let properties = Properties::decode(&props)?;
+        let pinned = top.charge() + row_bytes + column_bytes;
+        blocks.cache.add_pinned(pinned);
+        Ok(Self {
+            blocks,
+            top,
+            row_filter,
+            column_filter,
+            properties,
+            pinned,
+        })
+    }
+
+    /// The SST's length in bytes (`SstMeta::len`).
+    pub(crate) fn len_bytes(&self) -> u64 {
+        self.blocks.limit + FOOTER_LEN as u64
+    }
+
+    pub(crate) fn may_contain_row(&self, hash: u64) -> bool {
+        self.row_filter.as_ref().is_none_or(|f| f.may_contain(hash))
+    }
+
+    pub(crate) fn may_contain_column(&self, hash: u64) -> bool {
+        self.column_filter
+            .as_ref()
+            .is_none_or(|f| f.may_contain(hash))
+    }
+}
+
+impl Blocks {
+    fn check(&self, addr: BlockAddr) -> Result<()> {
+        let end = addr.offset.checked_add(u64::from(addr.len));
+        if (addr.len as usize) < TRAILER_LEN || end.is_none_or(|e| e > self.limit) {
+            return Err(corrupt("block address"));
+        }
+        Ok(())
+    }
+
+    pub(crate) fn key(&self, addr: BlockAddr) -> BlockKey {
+        BlockKey {
+            file: crate::sst_cache_file(self.id),
+            offset: addr.offset,
+        }
+    }
+
+    /// Whether `addr` is cached (pins it if so).
+    pub(crate) fn cached(&self, addr: BlockAddr) -> Option<BlockHandle> {
+        self.cache.get(self.key(addr))
+    }
+
+    /// Reads a block through the cache. Checksums are verified on reads from disk and skipped
+    /// on hits.
+    pub(crate) fn read_block(
+        &self,
+        addr: BlockAddr,
+        kind: BlockKind,
+        fill_cache: bool,
+        priority: Priority,
+    ) -> Result<BlockHandle> {
+        self.check(addr)?;
+        if let Some(h) = self.cached(addr) {
+            return Ok(h);
+        }
+        crate::note_file_read();
+        let buf = read_bytes(&self.file, self.base + addr.offset, addr.len as usize)?;
+        let data = decode_physical(buf, kind, self.pool(fill_cache))?;
+        Ok(self.admit(addr, data, fill_cache, priority))
+    }
+
+    /// The block at `addr` if it is cached (pinned), after the same check as
+    /// [`Blocks::read_block`]: the hit path of a read, for a cursor that handles its misses
+    /// itself ([`Blocks::read_uncached`], or `Error::WouldBlock` in cache-only mode).
+    #[inline]
+    pub(crate) fn lookup(&self, addr: BlockAddr) -> Result<Option<BlockHandle>> {
+        self.check(addr)?;
+        Ok(self.cached(addr))
+    }
+
+    /// Reads a block that [`Blocks::lookup`] did not find, from the file.
+    pub(crate) fn read_uncached(
+        &self,
+        addr: BlockAddr,
+        kind: BlockKind,
+        fill_cache: bool,
+        priority: Priority,
+    ) -> Result<BlockHandle> {
+        crate::note_file_read();
+        let buf = read_bytes(&self.file, self.base + addr.offset, addr.len as usize)?;
+        let data = decode_physical(buf, kind, self.pool(fill_cache))?;
+        Ok(self.admit(addr, data, fill_cache, priority))
+    }
+
+    /// [`Blocks::read_block`] from the cache only: a miss fails with `Error::WouldBlock`
+    /// naming the block instead of reading it (a cache-only open, ICR 0014).
+    pub(crate) fn read_block_cache_only(
+        &self,
+        addr: BlockAddr,
+        kind: BlockKind,
+        priority: Priority,
+    ) -> Result<BlockHandle> {
+        match self.lookup(addr)? {
+            Some(h) => Ok(h),
+            None => Err(self.would_block(addr, kind, priority)),
+        }
+    }
+
+    /// The `WouldBlock` error of a cache-only read that missed `addr`.
+    #[cold]
+    pub(crate) fn would_block(
+        &self,
+        addr: BlockAddr,
+        kind: BlockKind,
+        priority: Priority,
+    ) -> crate::Error {
+        crate::Error::WouldBlock(Box::new(self.fetch(addr, kind, priority)))
+    }
+
+    /// A cache-only read's miss of the block at `addr` (#709): the block's read, submitted
+    /// at once. A read that is done when `submit` returns (a page-cache hit copied on this
+    /// thread, D292, under its breaker, size cap and refusal of direct handles; or a backend
+    /// that completes reads at submit) is admitted and returned, so the cursor goes on as a
+    /// synchronous read would, without unwinding the read to fetch and run it again. Any
+    /// other read fails with `Error::WouldBlock` carrying the read in flight, which the
+    /// caller resumes ([`Fetch::start`]) instead of submitting it again.
+    #[cold]
+    #[inline(never)]
+    pub(crate) fn fetch_now(
+        &self,
+        addr: BlockAddr,
+        kind: BlockKind,
+        priority: Priority,
+    ) -> Result<BlockHandle> {
+        let mut fetch = self.fetch(addr, kind, priority);
+        let done = fetch.submit();
+        if done.is_ready() {
+            crate::note_file_read();
+            // Done: `wait` takes the result without blocking.
+            if let Some(h) = fetch.admit(done.wait()?)? {
+                return Ok(h);
+            }
+            unreachable!("a block fetch admits a handle");
+        }
+        fetch.submitted = Some(done);
+        Err(crate::Error::WouldBlock(Box::new(fetch)))
+    }
+
+    /// The fetch of the block at `addr`, admitted to the cache.
+    pub(crate) fn fetch(&self, addr: BlockAddr, kind: BlockKind, priority: Priority) -> Fetch {
+        self.fetch_with(addr, kind, priority, true)
+    }
+
+    /// The fetch of the block at `addr`, admitted to the cache if `fill_cache`, else handed
+    /// out unshared (a read that must not fill the cache, as [`Blocks::admit`]).
+    pub(crate) fn fetch_with(
+        &self,
+        addr: BlockAddr,
+        kind: BlockKind,
+        priority: Priority,
+        fill_cache: bool,
+    ) -> Fetch {
+        let cache = if fill_cache {
+            &self.cache
+        } else {
+            &self.uncached
+        };
+        Fetch::one(
+            self.file.clone(),
+            self.base + addr.offset,
+            addr.len as usize,
+            Then::Block {
+                cache: Arc::clone(cache),
+                key: self.key(addr),
+                kind,
+                priority,
+                pool: self.pool(fill_cache).cloned(),
+            },
+        )
+    }
+
+    /// Submits reads of `(offset, len)` byte ranges at block offsets, the blocks (or runs of
+    /// adjacent ones) a readahead predicts, as one batch nothing waits on yet
+    /// ([`pigeonhole_io::File::submit_reads_ahead`], ICR 0077). Each resolves to a buffer of
+    /// just its bytes, as [`submit_bytes`] does.
+    pub(crate) fn submit_ahead(&self, ranges: &[(u64, usize)]) -> Vec<pigeonhole_io::Completion> {
+        let file = &self.file;
+        match file.direct_align() {
+            None => file.submit_reads_ahead(
+                ranges
+                    .iter()
+                    .map(|&(offset, len)| (file.read_buf(len), self.base + offset))
+                    .collect(),
+            ),
+            Some(a) => {
+                let spans: Vec<_> = ranges
+                    .iter()
+                    .map(|&(offset, len)| (aligned(self.base + offset, len, a), len))
+                    .collect();
+                let reads = spans
+                    .iter()
+                    .map(|&((start, alen, _), _)| (file.read_buf(alen), start))
+                    .collect();
+                file.submit_reads_ahead(reads)
+                    .into_iter()
+                    .zip(spans)
+                    .map(|(done, ((_, _, head), len))| {
+                        done.map(move |r| {
+                            r.map(|mut b| {
+                                b.keep(head..head + len);
+                                b
+                            })
+                        })
+                    })
+                    .collect()
+            }
+        }
+    }
+
+    /// The fetch of `addr` (as [`Blocks::fetch_with`]) if the cache does not hold it (and
+    /// its address is valid).
+    pub(crate) fn fetch_if_missing(
+        &self,
+        addr: BlockAddr,
+        kind: BlockKind,
+        priority: Priority,
+        fill_cache: bool,
+    ) -> Option<Fetch> {
+        match self.lookup(addr) {
+            Ok(None) => Some(self.fetch_with(addr, kind, priority, fill_cache)),
+            _ => None,
+        }
+    }
+
+    /// The pool of buffers for a read that does not fill the cache (`None` for one that does:
+    /// a cached block keeps an exact-size buffer, as the cache charges capacity).
+    fn pool(&self, fill_cache: bool) -> Option<&Arc<BufPool>> {
+        (!fill_cache).then(|| self.recycled.get_or_init(|| BufPool::new(RECYCLED)))
+    }
+
+    /// Hands out a decoded block, inserting it into the cache if asked.
+    pub(crate) fn admit(
+        &self,
+        addr: BlockAddr,
+        data: BlockData,
+        fill_cache: bool,
+        priority: Priority,
+    ) -> BlockHandle {
+        let cache = if fill_cache {
+            &*self.cache
+        } else {
+            &*self.uncached
+        };
+        cache.insert(self.key(addr), data, priority)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use pigeonhole_cache::BlockKey;
+    use pigeonhole_format::block::seal;
+
+    /// The charge of a block decoded from `physical` read into a reused buffer of `cap`
+    /// bytes, once the cache keeps it.
+    fn charged(physical: &[u8], cap: usize) -> usize {
+        // A thread-pool buffer of `cap` bytes, reused for this read (#709).
+        drop(IoBuf::reused(cap));
+        let mut buf = IoBuf::reused(physical.len());
+        buf.copy_from_slice(physical);
+        let data = decode_physical(buf, BlockKind::Data, None).unwrap();
+        let cache = BlockCache::new(1 << 20, 1);
+        let key = BlockKey { file: 1, offset: 0 };
+        cache.insert(key, data, Priority::Normal).charge()
+    }
+
+    #[test]
+    fn a_cached_block_is_charged_as_before_reused_read_buffers() {
+        // A compressed block: decompressed into a new buffer of exactly its length.
+        let logical: Vec<u8> = (0..16_500u32).map(|i| (i % 251) as u8).collect();
+        let mut lz4 = Vec::new();
+        seal(BlockKind::Data, Compression::Lz4, &logical, &mut lz4).unwrap();
+        assert_eq!(charged(&lz4, 40 << 10), logical.len());
+        // An uncompressed block keeps its read buffer: a reused one larger than the block's
+        // pages is copied, so it is charged its own pages, as a new buffer was.
+        let mut raw = Vec::new();
+        seal(BlockKind::Data, Compression::None, &logical, &mut raw).unwrap();
+        let pages = raw.len().next_multiple_of(IoBuf::ALIGN);
+        assert_eq!(charged(&raw, 2 * pages - IoBuf::ALIGN), pages);
+    }
+}