pigeonhole-db 1.1.0

data/vendor/format/src/checksum.rs

@@ -0,0 +1,132 @@
+//! Checksums: CRC32C for the WAL, xxh3-64 for everything in the main file.
+//!
+//! ```
+//! use pigeonhole_format::checksum::{crc32c, crc32c_append};
+//!
+//! assert_eq!(crc32c_append(crc32c(b"hello "), b"world"), crc32c(b"hello world"));
+//! ```
+
+/// CRC32C (Castagnoli) of `bytes`. Used by WAL frames and WAL segment headers.
+pub fn crc32c(bytes: &[u8]) -> u32 {
+    crc32c_append(0, bytes)
+}
+
+/// Extends a running CRC32C with `bytes`.
+pub fn crc32c_append(crc: u32, bytes: &[u8]) -> u32 {
+    // The SIMD paths are intrinsics Miri cannot run.
+    if cfg!(miri) {
+        crc32c_soft(crc, bytes)
+    } else if crc == 0 {
+        // The common case (every frame, header and record): crc-fast's whole-buffer entry,
+        // which skips the digest's set-up (#1206 lever 4).
+        crc_fast::checksum(crc_fast::CrcAlgorithm::Crc32Iscsi, bytes) as u32
+    } else {
+        // Continuing a finished CRC: the digest's state is the register before the final
+        // inversion, so it starts at `!crc`.
+        let mut d = crc_fast::Digest::new_with_init_state(
+            crc_fast::CrcAlgorithm::Crc32Iscsi,
+            u64::from(!crc),
+        );
+        d.update(bytes);
+        d.finalize() as u32
+    }
+}
+
+/// Table-driven CRC32C (reflected polynomial `0x82F63B78`), used under Miri and checked
+/// against the crate in tests.
+fn crc32c_soft(crc: u32, bytes: &[u8]) -> u32 {
+    let mut crc = !crc;
+    for &b in bytes {
+        crc = CRC32C_TABLE[((crc ^ u32::from(b)) & 0xFF) as usize] ^ (crc >> 8);
+    }
+    !crc
+}
+
+/// CRC32C of each byte value, computed at compile time.
+const CRC32C_TABLE: [u32; 256] = {
+    let mut table = [0u32; 256];
+    let mut i = 0;
+    while i < 256 {
+        let mut crc = i as u32;
+        let mut bit = 0;
+        while bit < 8 {
+            crc = if crc & 1 != 0 {
+                (crc >> 1) ^ 0x82F6_3B78
+            } else {
+                crc >> 1
+            };
+            bit += 1;
+        }
+        table[i] = crc;
+        i += 1;
+    }
+    table
+};
+
+/// xxh3-64 (seed 0) of `bytes`. Used by blocks, the SST footer, superblocks, manifest blocks,
+/// blob records and filter hashing.
+pub fn xxh3_64(bytes: &[u8]) -> u64 {
+    twox_hash::XxHash3_64::oneshot(bytes)
+}
+
+/// xxh3-64 (seed 0) of the concatenation of `parts`, without copying them together.
+pub(crate) fn xxh3_64_parts(parts: &[&[u8]]) -> u64 {
+    use std::hash::Hasher;
+    use twox_hash::xxhash3_64::{RawHasher, SecretBuffer};
+    // The default seed and secret, borrowed: `XxHash3_64::with_seed` boxes a copy of the
+    // secret per hash (#499).
+    let mut h = RawHasher::new(SecretBuffer::default());
+    for p in parts {
+        h.write(p);
+    }
+    h.finish()
+}
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn xxh3_of_parts_is_xxh3_of_their_concatenation() {
+        let data: Vec<u8> = (0..5000u32).map(|i| (i * 31 + i / 7) as u8).collect();
+        for len in [0, 1, 16, 240, 241, 1024, 5000] {
+            let whole = super::xxh3_64(&data[..len]);
+            for split in [0, len / 3, len / 2, len] {
+                let (a, b) = data[..len].split_at(split);
+                assert_eq!(
+                    super::xxh3_64_parts(&[a, b]),
+                    whole,
+                    "len {len} split {split}"
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn soft_crc_matches() {
+        let data: Vec<u8> = (0..1000u32).map(|i| (i * 7 + i / 3) as u8).collect();
+        // The standard CRC32C check value, on both paths.
+        assert_eq!(super::crc32c_soft(0, b"123456789"), 0xE306_9283);
+        assert_eq!(super::crc32c(b"123456789"), 0xE306_9283);
+        assert_eq!(super::crc32c(b""), 0);
+        // Every length a frame takes (the small-input path below 16 bytes and around the
+        // SIMD block sizes), and continuing a finished CRC at any split.
+        for len in [
+            0, 1, 7, 8, 15, 16, 17, 63, 64, 65, 127, 128, 129, 255, 256, 257, 1000,
+        ] {
+            let whole = super::crc32c(&data[..len]);
+            assert_eq!(whole, super::crc32c_soft(0, &data[..len]), "len {len}");
+            for split in [0, 1, len / 2, len.saturating_sub(1), len] {
+                let (a, b) = data[..len].split_at(split.min(len));
+                assert_eq!(
+                    super::crc32c_soft(super::crc32c_soft(0, a), b),
+                    whole,
+                    "soft len {len} split {split}"
+                );
+                assert_eq!(
+                    super::crc32c_append(super::crc32c(a), b),
+                    whole,
+                    "len {len} split {split}"
+                );
+            }
+        }
+    }
+}

data/vendor/format/src/compress.rs

@@ -0,0 +1,196 @@
+//! Block compression codecs. The codec byte is stored in every block trailer.
+//!
+//! ```
+//! use pigeonhole_format::compress::{Compression, compress, decompress};
+//!
+//! let input = vec![7u8; 4096];
+//! let mut packed = Vec::new();
+//! assert_eq!(compress(Compression::Lz4, &input, &mut packed).unwrap(), Compression::Lz4);
+//! let mut back = vec![0; input.len()];
+//! decompress(Compression::Lz4, &packed, &mut back).unwrap();
+//! assert_eq!(back, input);
+//! ```
+
+use crate::Error;
+
+/// A block compression codec. Numbers are frozen.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
+#[repr(u8)]
+pub enum Compression {
+    /// Stored as is.
+    None = 0,
+    /// LZ4 block format (no frame), the default.
+    #[default]
+    Lz4 = 1,
+    /// zstd frames (libzstd) at the family's level. Trained dictionaries are not used yet.
+    Zstd = 2,
+}
+
+impl Compression {
+    /// Parses a codec byte.
+    pub fn from_u8(b: u8) -> crate::Result<Self> {
+        match b {
+            0 => Ok(Self::None),
+            1 => Ok(Self::Lz4),
+            2 => Ok(Self::Zstd),
+            _ => Err(Error::UnsupportedCompression(b)),
+        }
+    }
+}
+
+/// The zstd level [`compress`] uses (libzstd's default).
+pub const DEFAULT_ZSTD_LEVEL: i8 = 3;
+
+/// Compresses `input` with `codec`, appending to `out`. Returns the codec actually used:
+/// [`Compression::None`] when compression would not save at least 1/8 of the size. zstd
+/// runs at [`DEFAULT_ZSTD_LEVEL`]; see [`compress_with_level`].
+pub fn compress(codec: Compression, input: &[u8], out: &mut Vec<u8>) -> crate::Result<Compression> {
+    compress_with_level(codec, DEFAULT_ZSTD_LEVEL, input, out)
+}
+
+/// [`compress`] with a zstd `level` (libzstd's: negative is faster, up to 22 smaller; 0 is
+/// its default; out-of-range levels are clamped). Other codecs ignore it.
+///
+/// ```
+/// use pigeonhole_format::compress::{Compression, compress_with_level, decompress};
+///
+/// let input: Vec<u8> = (0..8192u32).map(|i| (i % 251) as u8).collect();
+/// let mut packed = Vec::new();
+/// let used = compress_with_level(Compression::Zstd, 19, &input, &mut packed).unwrap();
+/// assert_eq!(used, Compression::Zstd);
+/// let mut back = vec![0; input.len()];
+/// decompress(Compression::Zstd, &packed, &mut back).unwrap();
+/// assert_eq!(back, input);
+/// ```
+pub fn compress_with_level(
+    codec: Compression,
+    level: i8,
+    input: &[u8],
+    out: &mut Vec<u8>,
+) -> crate::Result<Compression> {
+    Compressor::default().compress(codec, level, input, out)
+}
+
+/// Compression state kept across blocks, so a writer compressing many blocks allocates and
+/// zero-fills once rather than per block: the LZ4 hash table, the zstd context, and a scratch
+/// buffer the codec writes into (sized to the largest output yet, zeroed only when it
+/// grows), from which only the compressed bytes are copied out. The results are the same as
+/// [`compress_with_level`]'s.
+///
+/// ```
+/// use pigeonhole_format::compress::{Compression, Compressor, decompress};
+///
+/// let mut c = Compressor::default();
+/// let mut packed = Vec::new();
+/// for block in [vec![7u8; 4096], vec![9u8; 4096]] {
+///     packed.clear();
+///     assert_eq!(c.compress(Compression::Lz4, 3, &block, &mut packed).unwrap(), Compression::Lz4);
+///     let mut back = vec![0; block.len()];
+///     decompress(Compression::Lz4, &packed, &mut back).unwrap();
+///     assert_eq!(back, block);
+/// }
+/// ```
+#[derive(Default)]
+pub struct Compressor {
+    lz4: Option<lz4_flex::block::CompressTable>,
+    zstd: Option<(i8, zstd::bulk::Compressor<'static>)>,
+    scratch: Vec<u8>,
+}
+
+impl std::fmt::Debug for Compressor {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Compressor")
+            .field("scratch", &self.scratch.len())
+            .finish_non_exhaustive()
+    }
+}
+
+impl Compressor {
+    /// Compresses `input` with `codec` (zstd at `level`), appending to `out`. Returns the codec
+    /// actually used: [`Compression::None`] when compression would not save at least 1/8 of
+    /// the size (`input` is then appended as is).
+    pub fn compress(
+        &mut self,
+        codec: Compression,
+        level: i8,
+        input: &[u8],
+        out: &mut Vec<u8>,
+    ) -> crate::Result<Compression> {
+        let n = match codec {
+            Compression::None => None,
+            Compression::Lz4 => {
+                let max = lz4_flex::block::get_maximum_output_size(input.len());
+                let table = self.lz4.get_or_insert_with(Default::default);
+                let scratch = grow(&mut self.scratch, max);
+                let n = lz4_flex::block::compress_into_with_table(input, scratch, table).map_err(
+                    |_| Error::Corrupt {
+                        what: "lz4 compress",
+                    },
+                )?;
+                Some(n)
+            }
+            Compression::Zstd => {
+                let bound = zstd::zstd_safe::compress_bound(input.len());
+                if self.zstd.as_ref().is_none_or(|(l, _)| *l != level) {
+                    let c = zstd::bulk::Compressor::new(i32::from(level)).map_err(|_| {
+                        Error::Corrupt {
+                            what: "zstd compress",
+                        }
+                    })?;
+                    self.zstd = Some((level, c));
+                }
+                let scratch = grow(&mut self.scratch, bound);
+                let (_, c) = self.zstd.as_mut().expect("set above");
+                let n = c
+                    .compress_to_buffer(input, scratch)
+                    .map_err(|_| Error::Corrupt {
+                        what: "zstd compress",
+                    })?;
+                Some(n)
+            }
+        };
+        // Keep the compressed form only if it saves at least 1/8 of the input.
+        if let Some(n) = n
+            && n < input.len()
+            && (input.len() - n) * 8 >= input.len()
+        {
+            out.extend_from_slice(&self.scratch[..n]);
+            return Ok(codec);
+        }
+        out.extend_from_slice(input);
+        Ok(Compression::None)
+    }
+}
+
+/// The first `len` bytes of `scratch`, grown (and zeroed) only past its largest size yet.
+fn grow(scratch: &mut Vec<u8>, len: usize) -> &mut [u8] {
+    if scratch.len() < len {
+        scratch.resize(len, 0);
+    }
+    &mut scratch[..len]
+}
+
+/// Decompresses `input` into `out`, which must be exactly `uncompressed_len` bytes long.
+pub fn decompress(codec: Compression, input: &[u8], out: &mut [u8]) -> crate::Result<()> {
+    match codec {
+        Compression::None => {
+            if input.len() != out.len() {
+                return Err(Error::Corrupt {
+                    what: "uncompressed block length",
+                });
+            }
+            out.copy_from_slice(input);
+            Ok(())
+        }
+        Compression::Lz4 => match lz4_flex::block::decompress_into(input, out) {
+            Ok(n) if n == out.len() => Ok(()),
+            _ => Err(Error::Corrupt { what: "lz4 block" }),
+        },
+        // Into a buffer of exactly the block's length: a frame that would decode to more
+        // fails instead of growing it.
+        Compression::Zstd => match zstd::bulk::decompress_to_buffer(input, out) {
+            Ok(n) if n == out.len() => Ok(()),
+            _ => Err(Error::Corrupt { what: "zstd block" }),
+        },
+    }
+}

data/vendor/format/src/cursor.rs

@@ -0,0 +1,110 @@
+//! The ordered cursor every sorted source implements (memtables, blocks, SSTs, merges).
+
+/// A positioned, forward-only cursor over `(internal key, stored value)` pairs in key order.
+///
+/// Implemented by `MemIter`, `SstIter` and the merging cursors above them. Static dispatch
+/// only: generic code takes `C: Cursor`, and the engine wraps heterogeneous sources in an enum,
+/// so the per-entry path has no virtual calls. Keys and values borrow from the cursor and are
+/// valid until it next moves.
+pub trait Cursor {
+    /// Error produced while moving (I/O, corruption).
+    type Error;
+
+    /// Whether the cursor is on an entry. `key` and `value` may only be called when true.
+    fn valid(&self) -> bool;
+
+    /// The current internal key.
+    fn key(&self) -> &[u8];
+
+    /// The current stored value (empty for deletes).
+    fn value(&self) -> &[u8];
+
+    /// Positions on the first entry.
+    fn seek_to_first(&mut self) -> Result<(), Self::Error>;
+
+    /// Positions on the first entry whose key is `>= target`.
+    fn seek(&mut self, target: &[u8]) -> Result<(), Self::Error>;
+
+    /// Advances one entry.
+    fn next(&mut self) -> Result<(), Self::Error>;
+
+    /// Positions on the first entry whose key is `>= target`, where `target` is at or past
+    /// the current key (a forward seek). The default seeks; a merging cursor overrides it to
+    /// move only its sources still behind `target`.
+    ///
+    /// On a cursor that is not valid (unpositioned, or exhausted by an earlier move) the
+    /// result depends on the implementation: the default and a memtable cursor position as
+    /// [`Cursor::seek`] does, but a merging cursor stays exhausted, even for a `target`
+    /// before the key it was exhausted from. Use `seek` to position such a cursor.
+    fn seek_forward(&mut self, target: &[u8]) -> Result<(), Self::Error> {
+        self.seek(target)
+    }
+
+    /// Moves past the current entry and possibly more entries of `column` (an internal key
+    /// without its suffix), never past an entry outside it, and returns `true`; or returns
+    /// `false` without moving, when the cursor has no faster way than [`Cursor::next`]. The
+    /// cursor must be valid and on an entry of `column`. A memtable with a stale-tail index
+    /// overrides it to pass a column's superseded versions in one jump (D194, ICR 0013); the
+    /// default returns `false`.
+    fn skip_column(&mut self, column: &[u8]) -> Result<bool, Self::Error> {
+        let _ = column;
+        Ok(false)
+    }
+
+    /// Whether the current entry is one the cursor yields, or only a lower bound on the next
+    /// entry it yields (D288). Only a cursor in a lazy mode ([`Cursor::may_park`]) is ever
+    /// unsettled, and only a merging cursor that knows it may be reads one: it settles a
+    /// source with [`Cursor::settle_until`] when the source reaches its top. The default is
+    /// always settled.
+    #[inline]
+    fn settled(&self) -> bool {
+        true
+    }
+
+    /// Moves an unsettled cursor forward to the next entry it yields, but parks (stays
+    /// unsettled) on the first entry whose key is strictly greater than `bound`, by
+    /// [`crate::key::compare`]; `None` settles fully. An entry equal to `bound` is checked, not
+    /// parked on, so a merge that settles its top up to the next source's key always either
+    /// settles it or passes that key. The default does nothing (always settled).
+    fn settle_until(&mut self, bound: Option<&[u8]>) -> Result<(), Self::Error> {
+        let _ = bound;
+        Ok(())
+    }
+
+    /// Whether the cursor may be unsettled after a move (a lazy filtered source, D288): a
+    /// merging cursor over it must settle its top. The default never is.
+    #[inline]
+    fn may_park(&self) -> bool {
+        false
+    }
+
+    /// Advances past every remaining entry of the current row. Sources with a row-start table
+    /// override this to skip without decoding; the default steps with [`Cursor::next`].
+    fn skip_row(&mut self) -> Result<(), Self::Error> {
+        if !self.valid() {
+            return Ok(());
+        }
+        let Ok(n) = crate::key::row_prefix_len(self.key()) else {
+            return self.next();
+        };
+        // The row prefix ends with a terminator that never occurs inside an escaped row, so
+        // a key starting with it belongs to the same row. Copied to the stack, which holds
+        // most row prefixes: a cursor that skips rows (a column limit, a filter's `SkipRow`)
+        // allocated one per skip.
+        let mut inline = [0u8; 64];
+        let heap;
+        let row: &[u8] = if let Some(buf) = inline.get_mut(..n) {
+            buf.copy_from_slice(&self.key()[..n]);
+            buf
+        } else {
+            heap = self.key()[..n].to_vec();
+            &heap
+        };
+        loop {
+            self.next()?;
+            if !self.valid() || !self.key().starts_with(row) {
+                return Ok(());
+            }
+        }
+    }
+}