pigeonhole-db 1.1.0

data/vendor/io/src/file.rs

@@ -0,0 +1,206 @@
+use std::fmt::Debug;
+use std::sync::Arc;
+
+use crate::{Completion, FileIdentity, IoBuf, Result};
+
+/// How to open a file.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+#[non_exhaustive]
+pub struct OpenOptions {
+    /// Open for writing as well as reading.
+    pub write: bool,
+    /// Create the file if it does not exist.
+    pub create: bool,
+    /// Fail if the file exists (implies `create`).
+    pub create_new: bool,
+    /// Bypass the OS page cache (#403): `O_DIRECT` on Linux, `F_NOCACHE` on macOS,
+    /// `FILE_FLAG_NO_BUFFERING` on Windows. Every read and write on such a handle must then
+    /// be aligned ([`File::direct_align`]): its offset, its length and its buffer's address
+    /// (an [`IoBuf`] is). A file system that refuses direct I/O fails the open with
+    /// [`ErrorKind::Unsupported`](crate::ErrorKind::Unsupported); open without it instead.
+    pub direct: bool,
+}
+
+impl OpenOptions {
+    /// Read-only.
+    pub fn read() -> Self {
+        Self::default()
+    }
+
+    /// Read-write, creating if missing.
+    pub fn read_write_create() -> Self {
+        Self {
+            write: true,
+            create: true,
+            ..Self::default()
+        }
+    }
+}
+
+/// Mode of a byte-range lock.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum LockMode {
+    /// Any number of holders.
+    Shared,
+    /// One holder.
+    Exclusive,
+}
+
+/// An open file. Shared across threads; all methods take `&self` and use positional I/O.
+///
+/// Locks are per handle (OFD locks on Linux, `fcntl` behind a per-process registry on macOS
+/// and BSD so closing one handle never drops another's lock, `LockFileEx` on Windows).
+pub trait File: Send + Sync + Debug {
+    /// Reads exactly `buf.len()` bytes at `offset`; fails with `UnexpectedEof` if short.
+    fn read_at(&self, buf: &mut [u8], offset: u64) -> Result<()>;
+
+    /// Writes all of `buf` at `offset`.
+    fn write_at(&self, buf: &[u8], offset: u64) -> Result<()>;
+
+    /// For a handle opened with [`OpenOptions::direct`], the alignment its I/O needs: offsets,
+    /// lengths and buffer addresses a multiple of it, else
+    /// [`ErrorKind::Misaligned`](crate::ErrorKind::Misaligned). `None` for a buffered
+    /// handle, whose I/O may start and end anywhere (the default).
+    fn direct_align(&self) -> Option<usize> {
+        None
+    }
+
+    /// A buffer of `len` bytes for a submitted read ([`File::submit_read`]): on a backend
+    /// with registered buffers (io_uring, #402), a slot of the pool the read will use, so the
+    /// kernel skips pinning its pages. Its contents are unspecified (old bytes of an earlier
+    /// read may be there): fill it fully before anyone reads it, as an exact read does, or
+    /// drop it. Keep a slot-backed buffer only for its I/O: one stored for longer goes through
+    /// [`IoBuf::detached`]. The default is [`IoBuf::reused`].
+    fn read_buf(&self, len: usize) -> IoBuf {
+        IoBuf::reused(len)
+    }
+
+    /// Submits a read of `buf.len()` bytes at `offset` and returns at once.
+    fn submit_read(&self, buf: IoBuf, offset: u64) -> Completion;
+
+    /// Submits reads that nothing waits on yet (a scan's readahead) as one batch, each
+    /// resolving as [`File::submit_read`] would, in the order given. A backend may hand the
+    /// whole batch to the kernel at once and run each read off the submitting thread (io_uring:
+    /// one enter; on the shared ring every read with `IOSQE_ASYNC`, ICR 0077). The default
+    /// submits each in turn.
+    ///
+    /// ```
+    /// use std::path::Path;
+    /// use pigeonhole_io::{File, OpenOptions, Vfs};
+    /// use pigeonhole_io::sim::SimVfs;
+    ///
+    /// # fn main() -> pigeonhole_io::Result<()> {
+    /// let vfs = SimVfs::new(42);
+    /// let f = vfs.open(Path::new("/db/f"), OpenOptions::read_write_create())?;
+    /// f.write_at(&[7; 8192], 0)?;
+    /// let reads = vec![(f.read_buf(4096), 0), (f.read_buf(4096), 4096)];
+    /// for done in f.submit_reads_ahead(reads) {
+    ///     assert!(done.wait()?.iter().all(|&b| b == 7));
+    /// }
+    /// # Ok(())
+    /// # }
+    /// ```
+    fn submit_reads_ahead(&self, reads: Vec<(IoBuf, u64)>) -> Vec<Completion> {
+        reads
+            .into_iter()
+            .map(|(buf, offset)| self.submit_read(buf, offset))
+            .collect()
+    }
+
+    /// Submits a write of `buf` at `offset` and returns at once.
+    fn submit_write(&self, buf: IoBuf, offset: u64) -> Completion;
+
+    /// Makes written data durable (fdatasync / F_FULLFSYNC / FlushFileBuffers).
+    ///
+    /// Callers may not rely on it to make a length change durable: after `set_len`,
+    /// `allocate` or a write past the end, call [`File::sync_all`] before depending on the
+    /// new length surviving a power loss. `SimVfs` enforces this.
+    fn sync_data(&self) -> Result<()>;
+
+    /// Submits [`File::sync_data`] and returns at once; the backend runs it off the caller's
+    /// thread (pread pool, io_uring in Phase 3) so a shard keeps working while it runs.
+    fn submit_sync_data(&self) -> Completion<()>;
+
+    /// Makes data and metadata (size) durable.
+    fn sync_all(&self) -> Result<()>;
+
+    /// Submits [`File::sync_all`] and returns at once. The pread backend runs it on its pool,
+    /// so syncs of several files overlap; the default runs it inline (as `SimVfs` does, which
+    /// keeps simulated runs deterministic).
+    fn submit_sync_all(&self) -> Completion<()> {
+        Completion::ready(self.sync_all())
+    }
+
+    /// Current length.
+    fn len(&self) -> Result<u64>;
+
+    /// Whether the file is empty.
+    fn is_empty(&self) -> Result<bool> {
+        Ok(self.len()? == 0)
+    }
+
+    /// Truncates or extends (with zeros).
+    fn set_len(&self, len: u64) -> Result<()>;
+
+    /// Preallocates `[offset, offset + len)` so later writes there need no metadata update.
+    fn allocate(&self, offset: u64, len: u64) -> Result<()>;
+
+    /// Takes a byte-range lock on the single byte at `byte`. Never blocks: fails with
+    /// `ErrorKind::Locked` if a conflicting lock is held. Taking `Exclusive` while holding
+    /// `Shared` is an upgrade attempt. `Exclusive` needs a handle opened for writing (POSIX
+    /// refuses write locks on read-only descriptors); otherwise it fails with `Unsupported`.
+    fn lock(&self, byte: u64, mode: LockMode) -> Result<()>;
+
+    /// Releases this handle's lock on `byte` (a no-op if none is held).
+    fn unlock(&self, byte: u64) -> Result<()>;
+
+    /// The lock this handle holds on `byte`, if any. Locks never conflict within one handle
+    /// (a second [`File::lock`] of a byte the handle holds succeeds), so a caller that would
+    /// release a byte after a failed step checks this first, to never drop a lock it held
+    /// before (reader slots, #693). The default reports none: a wrapper that forwards
+    /// [`File::lock`] must forward this too.
+    fn holds(&self, byte: u64) -> Option<LockMode> {
+        let _ = byte;
+        None
+    }
+
+    /// Device and inode (file index on Windows), used to name the shared-memory region.
+    fn identity(&self) -> Result<FileIdentity>;
+
+    /// Whether the file lives on a local filesystem. Network filesystems are refused at open.
+    fn is_local(&self) -> Result<bool>;
+
+    /// What kind of filesystem the file lives on: [`Locality::Fuse`] apart from other
+    /// non-local ones, so a caller can accept FUSE on request (decision D173, #299). The
+    /// default derives it from [`File::is_local`] (never `Fuse`).
+    fn locality(&self) -> Result<Locality> {
+        Ok(if self.is_local()? {
+            Locality::Local
+        } else {
+            Locality::Network
+        })
+    }
+}
+
+/// The kind of filesystem a file lives on ([`File::locality`]).
+///
+/// ```
+/// use pigeonhole_io::Locality;
+///
+/// assert_ne!(Locality::Fuse, Locality::Local);
+/// ```
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+#[non_exhaustive]
+pub enum Locality {
+    /// A local filesystem.
+    Local,
+    /// A FUSE filesystem. It may be local (ntfs-3g, gocryptfs) or network-backed (sshfs,
+    /// s3fs, gcsfuse): its byte-range locks may be local to one host, and a sync may not
+    /// reach stable storage.
+    Fuse,
+    /// A network or cluster filesystem (NFS, SMB, Ceph, GPFS, ...).
+    Network,
+}
+
+/// A shared handle to an open file.
+pub type FileRef = Arc<dyn File>;

data/vendor/io/src/lib.rs

@@ -0,0 +1,53 @@
+//! Vfs abstraction for Pigeonhole: files, aligned buffers, completions, pread pool and
+//! fault-injecting simulated backend.
+//!
+//! Every byte any other crate reads or writes goes through a [`Vfs`], so the simulator can
+//! inject faults and control time. Backends:
+//!
+//! - [`pread::PreadVfs`]: real files, a `pread`/`pwrite` thread pool for submitted I/O. The
+//!   safe reference backend on every platform.
+//! - [`sim::SimVfs`]: in-memory, deterministic from a seed, with torn writes, reordered
+//!   fsyncs, ENOSPC and crashes. With [`sim::FaultPlan::none`] it is the plain in-memory mock
+//!   other crates test against.
+//! - `uring::UringVfs` (Linux only, so not linked here): real files with submitted I/O on
+//!   io_uring (#402).
+//!
+//! This crate also owns the other OS-facing `unsafe` the engine needs: shared-memory mappings
+//! ([`SharedRegion`]), byte-range locks, thread pinning and NUMA binding ([`sys`]).
+//!
+//! Part of [Pigeonhole](https://github.com/CodingAnarchy/pigeonhole). See the crate README.
+// `unsafe` is permitted in this crate; every block carries a `// SAFETY:` argument.
+#![deny(unsafe_op_in_unsafe_fn)]
+
+mod async_poll;
+mod buf;
+mod completion;
+mod error;
+mod file;
+mod nowait;
+mod os;
+mod own;
+pub mod pread;
+#[cfg(any(target_os = "linux", io_loom))]
+mod ring_wake;
+mod shared;
+pub mod sim;
+pub mod sys;
+#[cfg(target_os = "linux")]
+pub mod uring;
+mod vfs;
+
+#[doc(hidden)]
+pub use async_poll::{AsyncPollScope, async_poll_scope, in_async_poll};
+pub use buf::{BufPool, IoBuf};
+pub use completion::{Completion, Resolver};
+pub use error::{Error, ErrorKind, Result};
+pub use file::{File, FileRef, Locality, LockMode, OpenOptions};
+#[doc(hidden)]
+pub use nowait::{MISS_BUCKETS, NowaitReads};
+pub use own::{
+    OwnIoWaker, in_inline_io, inline_io, own_io_enters, own_io_fd, own_io_in_flight, own_io_waker,
+    reap_orphan_io, reap_own_io,
+};
+pub use shared::{SharedOpen, SharedRegion};
+pub use vfs::{FileIdentity, ProcessId, RingStats, Vfs, VfsRef};

data/vendor/io/src/nowait.rs

@@ -0,0 +1,256 @@
+//! D292 (#749): a read the page cache holds is copied on the thread that submits it, with
+//! `preadv2(RWF_NOWAIT)`, rather than handed to a pool thread or copied inside an io_uring
+//! enter under the kernel's `uring_lock` that every submitter queues on. Linux only; a miss
+//! (or a short read) is submitted as before.
+//!
+//! Each backend has its own switch and counters ([`Vfs::set_nowait_reads`],
+//! [`Vfs::nowait_reads`], ICR 0083), and each file its own breaker.
+//!
+//! [`Vfs::set_nowait_reads`]: crate::Vfs::set_nowait_reads
+//! [`Vfs::nowait_reads`]: crate::Vfs::nowait_reads
+
+use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering::Relaxed};
+
+/// Longest read tried inline. Shard and compaction threads submit reads too: a large blob
+/// extent copied inline would be a long synchronous copy inside a turn (D289), so it is
+/// submitted as before (perf287's S2).
+#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
+pub(crate) const MAX_LEN: usize = 256 << 10;
+
+/// Misses in a row, with no hit between them, that open a file's breaker (perf287's S1): a
+/// file system that takes the flag but never finds anything cached (some FUSE and network
+/// mounts) would otherwise cost a system call on every read.
+#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
+const BREAK_AFTER: u32 = 64;
+
+/// Reads an open breaker submits without trying, before the file tries again. A file read
+/// cold at first (after a restart) trips it and recovers; one that never hits tries
+/// `BREAK_AFTER` reads in every `BREAK_AFTER + SKIP`.
+#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
+const SKIP: u32 = 4096;
+
+/// Upper bounds of the miss-latency buckets in microseconds: `[0, 1)`, `[1, 2)`, ...
+/// `[512, 1024)`, then everything at or over 1024 µs.
+pub const MISS_BUCKETS: usize = 12;
+
+/// Hits are counted on stripes, one cache line each, by thread: a single counter would be a
+/// line every client's reads share.
+const STRIPES: usize = 16;
+
+#[repr(align(128))]
+#[derive(Default)]
+struct Stripe(AtomicU64);
+
+/// A backend's switch and counters.
+pub(crate) struct Backend {
+    on: AtomicBool,
+    hits: [Stripe; STRIPES],
+    misses: AtomicU64,
+    miss_nanos: AtomicU64,
+    miss_buckets: [AtomicU64; MISS_BUCKETS],
+    refused: AtomicU64,
+    breaks: AtomicU64,
+}
+
+impl Default for Backend {
+    fn default() -> Self {
+        Self {
+            on: AtomicBool::new(true),
+            hits: Default::default(),
+            misses: AtomicU64::new(0),
+            miss_nanos: AtomicU64::new(0),
+            miss_buckets: Default::default(),
+            refused: AtomicU64::new(0),
+            breaks: AtomicU64::new(0),
+        }
+    }
+}
+
+impl Backend {
+    pub(crate) fn set(&self, on: bool) {
+        self.on.store(on, Relaxed);
+    }
+
+    pub(crate) fn stats(&self) -> NowaitReads {
+        NowaitReads {
+            on: self.on.load(Relaxed),
+            hits: self.hits.iter().map(|s| s.0.load(Relaxed)).sum(),
+            misses: self.misses.load(Relaxed),
+            miss_nanos: self.miss_nanos.load(Relaxed),
+            miss_buckets: std::array::from_fn(|i| self.miss_buckets[i].load(Relaxed)),
+            refused: self.refused.load(Relaxed),
+            breaks: self.breaks.load(Relaxed),
+        }
+    }
+
+    #[cfg_attr(not(target_os = "linux"), allow(dead_code))]
+    #[inline]
+    pub(crate) fn on(&self) -> bool {
+        self.on.load(Relaxed)
+    }
+
+    #[cfg_attr(not(target_os = "linux"), allow(dead_code))]
+    #[inline]
+    fn hit(&self) {
+        thread_local! {
+            static STRIPE: usize = {
+                static NEXT: AtomicUsize = AtomicUsize::new(0);
+                NEXT.fetch_add(1, Relaxed) % STRIPES
+            };
+        }
+        let i = STRIPE.with(|s| *s);
+        self.hits[i].0.fetch_add(1, Relaxed);
+    }
+
+    #[cfg_attr(not(target_os = "linux"), allow(dead_code))]
+    #[cold]
+    fn miss(&self, nanos: u64) {
+        self.misses.fetch_add(1, Relaxed);
+        self.miss_nanos.fetch_add(nanos, Relaxed);
+        let micros = nanos / 1000;
+        let bucket = (u64::BITS - micros.leading_zeros()) as usize;
+        self.miss_buckets[bucket.min(MISS_BUCKETS - 1)].fetch_add(1, Relaxed);
+    }
+}
+
+/// A file's state: whether its file system refused the flag, and its breaker.
+#[derive(Default)]
+pub(crate) struct FileState {
+    refused: AtomicBool,
+    /// Misses since the last hit.
+    streak: AtomicU32,
+    /// Reads the open breaker still submits without trying.
+    skip: AtomicU32,
+}
+
+/// What one attempt found ([`crate::os::read_cached_at`]).
+#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum Cached {
+    /// The page cache held all of the buffer, which now has the bytes.
+    Hit,
+    /// It did not (`EAGAIN`), the read came up short, or anything else went wrong (`EINTR`
+    /// included: no retry in the fast path); the buffer holds nothing to rely on.
+    Miss,
+    /// The file system or kernel refuses `RWF_NOWAIT` (overlay, virtiofs and NFS say
+    /// `EOPNOTSUPP`, a kernel without the flag `EINVAL`): it never will for this file.
+    Refused,
+}
+
+impl FileState {
+    /// Whether to try this read inline: the backend's switch is on, the file system took
+    /// the flag, and the breaker is closed.
+    #[cfg_attr(not(target_os = "linux"), allow(dead_code))]
+    #[inline]
+    pub(crate) fn try_now(&self, backend: &Backend) -> bool {
+        if !backend.on() || self.refused.load(Relaxed) {
+            return false;
+        }
+        let mut skip = self.skip.load(Relaxed);
+        if skip == 0 {
+            return true;
+        }
+        // Open: count one skipped read, saturating at 0. Readers racing on the last skips
+        // must not take it below 0, which would wrap it and open the breaker for good
+        // (perf287's review of #1018). A loop of `compare_exchange` only while it is open.
+        while let Some(next) = skip.checked_sub(1) {
+            match self
+                .skip
+                .compare_exchange_weak(skip, next, Relaxed, Relaxed)
+            {
+                Ok(_) => break,
+                Err(now) => skip = now,
+            }
+        }
+        false
+    }
+
+    /// Records what an attempt found, `nanos` after it started.
+    #[cfg_attr(not(target_os = "linux"), allow(dead_code))]
+    #[inline]
+    pub(crate) fn record(&self, backend: &Backend, found: Cached, nanos: u64) {
+        match found {
+            Cached::Hit => {
+                backend.hit();
+                if self.streak.load(Relaxed) != 0 {
+                    self.streak.store(0, Relaxed);
+                }
+            }
+            Cached::Miss => {
+                backend.miss(nanos);
+                if self.streak.fetch_add(1, Relaxed) + 1 >= BREAK_AFTER {
+                    self.streak.store(0, Relaxed);
+                    self.skip.store(SKIP, Relaxed);
+                    backend.breaks.fetch_add(1, Relaxed);
+                }
+            }
+            Cached::Refused => {
+                self.refused.store(true, Relaxed);
+                backend.refused.fetch_add(1, Relaxed);
+            }
+        }
+    }
+}
+
+/// A backend's inline page-cache reads since it started ([`crate::Vfs::nowait_reads`]):
+/// for benches and tests, not part of the API.
+#[doc(hidden)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+#[non_exhaustive]
+pub struct NowaitReads {
+    /// Whether the backend tries them now.
+    pub on: bool,
+    /// Reads copied inline.
+    pub hits: u64,
+    /// Attempts that found the range not all cached (or failed): submitted instead.
+    pub misses: u64,
+    /// Time the missed attempts took, in nanoseconds.
+    pub miss_nanos: u64,
+    /// Missed attempts by time: bucket `i` under `2^i` µs (bucket 0 under 1 µs), the last
+    /// one everything slower.
+    pub miss_buckets: [u64; MISS_BUCKETS],
+    /// Files whose file system refused `RWF_NOWAIT`.
+    pub refused: u64,
+    /// Times a file's breaker opened after misses with no hit.
+    pub breaks: u64,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::sync::Barrier;
+
+    #[test]
+    fn readers_racing_on_an_open_breakers_last_skips_never_wrap_it() {
+        // perf287's review of #1018: two readers that both saw one skip left took the
+        // count from 1 to 0 and then to u32::MAX, opening the breaker for good.
+        let backend = Backend::default();
+        // Miri runs threads interleaved one step at a time: a few rounds there (the
+        // thousands are for real threads racing).
+        let rounds = if cfg!(miri) { 4 } else { 2000 };
+        for round in 0..rounds {
+            let file = FileState::default();
+            file.skip.store(2, Relaxed);
+            let start = Barrier::new(8);
+            std::thread::scope(|s| {
+                for _ in 0..8 {
+                    s.spawn(|| {
+                        start.wait();
+                        for _ in 0..4 {
+                            let _ = file.try_now(&backend);
+                        }
+                    });
+                }
+            });
+            assert_eq!(
+                file.skip.load(Relaxed),
+                0,
+                "round {round}: the skip count wrapped"
+            );
+            assert!(
+                file.try_now(&backend),
+                "round {round}: a closed breaker tries"
+            );
+        }
+    }
+}

data/vendor/io/src/os/mod.rs

@@ -0,0 +1,149 @@
+//! Platform glue: every syscall the crate makes lives under this module.
+
+#[cfg(unix)]
+mod unix;
+#[cfg(unix)]
+pub(crate) use unix::*;
+
+#[cfg(windows)]
+mod windows;
+#[cfg(windows)]
+pub(crate) use windows::*;
+
+/// What the OS says about a process id (unix: `current_start_time` reads its own).
+#[cfg(unix)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum Liveness {
+    /// No such process (or a zombie).
+    Dead,
+    /// Running, with its start time if the platform reports one.
+    Alive(Option<u64>),
+}
+
+/// Validates a shared-memory name: non-empty, no path separators or NULs.
+pub(crate) fn check_shared_name(name: &str) -> crate::Result<()> {
+    if name.is_empty() || name.contains(['/', '\\', '\0']) {
+        return Err(crate::Error::new(
+            crate::ErrorKind::Other,
+            "invalid shared-memory name",
+        ));
+    }
+    Ok(())
+}
+
+/// The `i`-th CPU to pin a thread to (modulo the CPUs), as an index into the allowed set:
+/// the first CPU of every physical core in order, then the remaining CPUs (SMT siblings) in
+/// order (D220). `first_of_core[k]` says whether allowed CPU `k` is the first of its core.
+#[cfg_attr(not(any(target_os = "linux", windows, test)), allow(dead_code))]
+pub(crate) fn spread(first_of_core: &[bool], i: usize) -> usize {
+    if first_of_core.is_empty() {
+        return i;
+    }
+    let i = i % first_of_core.len();
+    let cores = first_of_core.iter().filter(|&&first| first).count();
+    let (want, nth) = if i < cores {
+        (true, i)
+    } else {
+        (false, i - cores)
+    };
+    first_of_core
+        .iter()
+        .enumerate()
+        .filter(|&(_, &first)| first == want)
+        .nth(nth)
+        .map_or(i, |(k, _)| k)
+}
+
+/// The first CPU of a sysfs CPU list ("0,16", "0-1", "3", "8-9,24-25\n"): the name of the
+/// core whose sibling list it is.
+#[cfg_attr(not(any(target_os = "linux", test)), allow(dead_code))]
+pub(crate) fn first_cpu_of_list(list: &str) -> Option<usize> {
+    let list = list.trim();
+    let end = list
+        .find(|c: char| !c.is_ascii_digit())
+        .unwrap_or(list.len());
+    list[..end].parse().ok()
+}
+
+/// For each CPU's core name (in the allowed set's order), whether it is the first CPU of its
+/// core.
+#[cfg_attr(not(any(target_os = "linux", test)), allow(dead_code))]
+pub(crate) fn first_of_core(cores: &[usize]) -> Vec<bool> {
+    cores
+        .iter()
+        .enumerate()
+        .map(|(k, core)| !cores[..k].contains(core))
+        .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{first_cpu_of_list, first_of_core, spread};
+
+    /// Each CPU's sibling list, read as the Linux topology reader reads it.
+    fn cores(lists: &[&str]) -> Vec<bool> {
+        let names: Vec<usize> = lists
+            .iter()
+            .map(|l| first_cpu_of_list(l).unwrap())
+            .collect();
+        first_of_core(&names)
+    }
+
+    #[test]
+    fn sibling_lists_name_their_core() {
+        assert_eq!(first_cpu_of_list("3\n"), Some(3));
+        assert_eq!(first_cpu_of_list("0,16\n"), Some(0));
+        assert_eq!(first_cpu_of_list("14-15\n"), Some(14));
+        assert_eq!(first_cpu_of_list("8-9,24-25\n"), Some(8));
+        assert_eq!(first_cpu_of_list(""), None);
+        assert_eq!(first_cpu_of_list("-1\n"), None);
+    }
+
+    #[test]
+    fn smt_siblings_count_once() {
+        // 4 cores, 8 threads, siblings numbered after every core (AMD, most Intel).
+        let split = ["0,4", "1,5", "2,6", "3,7", "0,4", "1,5", "2,6", "3,7"];
+        assert_eq!(
+            cores(&split),
+            [true, true, true, true, false, false, false, false]
+        );
+        // Siblings numbered next to each other, written as ranges.
+        let paired = ["0-1", "0-1", "2-3", "2-3"];
+        assert_eq!(cores(&paired), [true, false, true, false]);
+        // Two packages: core_id 0 exists on both, the sibling lists stay distinct.
+        let packages = ["0,2", "1,3", "0,2", "1,3"];
+        assert_eq!(cores(&packages), [true, true, false, false]);
+        // No SMT.
+        assert_eq!(cores(&["0", "1", "2"]), [true, true, true]);
+        // An affinity mask with only the siblings of cores 0 and 1 (CPUs 4 and 5).
+        assert_eq!(cores(&["0,4", "1,5"]), [true, true]);
+    }
+
+    #[test]
+    fn spread_over_a_two_package_machine() {
+        // Package 0: CPUs 0-3 = cores 0,1 with siblings; package 1: CPUs 4-7 likewise.
+        let lists = ["0-1", "0-1", "2-3", "2-3", "4-5", "4-5", "6-7", "6-7"];
+        let first = cores(&lists);
+        let order: Vec<usize> = (0..8).map(|i| spread(&first, i)).collect();
+        assert_eq!(order, [0, 2, 4, 6, 1, 3, 5, 7]);
+    }
+
+    #[test]
+    fn threads_spread_over_cores_before_their_siblings() {
+        // Siblings numbered after every core (0-3 are cores, 4-7 their siblings).
+        let split = [true, true, true, true, false, false, false, false];
+        assert_eq!(
+            (0..8).map(|i| spread(&split, i)).collect::<Vec<_>>(),
+            [0, 1, 2, 3, 4, 5, 6, 7]
+        );
+        // Siblings numbered next to each other (0-1 one core, 2-3 the next).
+        let paired = [true, false, true, false, true, false];
+        assert_eq!(
+            (0..7).map(|i| spread(&paired, i)).collect::<Vec<_>>(),
+            [0, 2, 4, 1, 3, 5, 0]
+        );
+        // No SMT, and no topology.
+        assert_eq!(spread(&[true, true], 3), 1);
+        assert_eq!(spread(&[], 5), 5);
+    }
+}