pigeonhole-db 1.1.0

data/vendor/memtable/src/tests.rs

@@ -0,0 +1,1061 @@
+//! Unit tests: layout, allocation, validation and offset-only linking. Concurrency is covered
+//! by `loom_tests` (model checked) and `tests/concurrent.rs` (real threads).
+
+use super::*;
+use pigeonhole_format::{Kind, encode_key};
+
+fn key(row: &[u8], seqno: u64) -> Vec<u8> {
+    let mut k = Vec::new();
+    encode_key(&mut k, row, b"q", 0, seqno, Kind::Put).unwrap();
+    k
+}
+
+fn small() -> ShardArena {
+    ShardArena::new(ArenaRegion::heap(64 * 1024), 1024)
+}
+
+fn scan(reader: &MemtableReader) -> Vec<(Vec<u8>, Vec<u8>)> {
+    let mut it = reader.iter();
+    it.seek_to_first().unwrap();
+    let mut out = Vec::new();
+    while it.valid() {
+        out.push((it.key().to_vec(), it.value().to_vec()));
+        it.next().unwrap();
+    }
+    out
+}
+
+#[test]
+fn empty_memtable() {
+    let mut arena = small();
+    let mt = Memtable::create(&mut arena).unwrap();
+    assert_eq!(mt.len(), 0);
+    assert!(mt.is_empty());
+    assert_eq!(mt.seqno_range(), None);
+    assert!(!mt.is_frozen());
+    assert_eq!(
+        mt.root(),
+        64,
+        "first memtable sits after the reserved prefix"
+    );
+    let reader = mt.reader();
+    assert!(reader.is_empty());
+    assert_eq!(reader.region().len(), 64 * 1024);
+    let mut it = reader.iter();
+    assert!(!it.valid());
+    it.seek_to_first().unwrap();
+    assert!(!it.valid());
+    it.seek(&key(b"x", 1)).unwrap();
+    assert!(!it.valid());
+    it.next().unwrap();
+    assert!(!it.valid());
+}
+
+#[test]
+fn insert_and_scan_in_key_order() {
+    let mut arena = small();
+    let mut mt = Memtable::create(&mut arena).unwrap();
+    let rows: [&[u8]; 5] = [b"m", b"c", b"zz", b"a", b"c\x00"];
+    let mut expected = Vec::new();
+    for (i, row) in rows.iter().enumerate() {
+        let k = key(row, i as u64 + 1);
+        let v = format!("\x00{i}").into_bytes();
+        mt.insert(&mut arena, &k, &v).unwrap();
+        expected.push((k, v));
+    }
+    expected.sort();
+    assert_eq!(mt.len(), 5);
+    assert_eq!(mt.seqno_range(), Some((1, 5)));
+    let reader = mt.reader();
+    assert_eq!(reader.len(), 5);
+    assert_eq!(scan(&reader), expected);
+
+    let mut it = reader.iter();
+    for (k, v) in &expected {
+        it.seek(k).unwrap();
+        assert!(it.valid());
+        assert_eq!(it.key(), &k[..]);
+        assert_eq!(it.value(), &v[..]);
+        assert_eq!(&*it.value_slice(), &v[..]);
+    }
+    // Between two keys: lands on the successor.
+    it.seek(&key(b"b", 1)).unwrap();
+    assert_eq!(it.key(), &expected[1].0[..]);
+    // Before everything and after everything.
+    it.seek(b"").unwrap();
+    assert_eq!(it.key(), &expected[0].0[..]);
+    it.seek(&key(b"zzz", 1)).unwrap();
+    assert!(!it.valid());
+    // `skip_row` (the default implementation) works over internal keys.
+    it.seek_to_first().unwrap();
+    it.skip_row().unwrap();
+    assert_eq!(it.key(), &expected[1].0[..]);
+}
+
+/// #1426: `skip_row` lands where stepping to the row's end lands, from every entry: rows
+/// narrower than, as wide as and wider than its steps before it seeks, rows with an escaped
+/// `0x00` and one with a row prefix longer than its stack buffer, and the last row.
+#[test]
+fn skip_row_lands_where_stepping_does() {
+    let mut arena = ShardArena::new(ArenaRegion::heap(4 << 20), 64 * 1024);
+    let mut mt = Memtable::create(&mut arena).unwrap();
+    let long = vec![b'L'; 80];
+    let rows: Vec<(Vec<u8>, usize)> = vec![
+        (b"a".to_vec(), 1),
+        (b"b".to_vec(), ROW_SKIP_STEPS),
+        (b"c".to_vec(), ROW_SKIP_STEPS + 1),
+        (b"d\x00e".to_vec(), 3),
+        (b"d".to_vec(), 40),
+        (long, 20),
+        (b"w".to_vec(), 300),
+        (b"z".to_vec(), 12),
+    ];
+    let mut seqno = 0;
+    for (row, width) in &rows {
+        for q in 0..*width {
+            seqno += 1;
+            let mut k = Vec::new();
+            encode_key(
+                &mut k,
+                row,
+                format!("q{q:04}").as_bytes(),
+                0,
+                seqno,
+                Kind::Put,
+            )
+            .unwrap();
+            mt.insert(&mut arena, &k, b"\x00v").unwrap();
+        }
+    }
+    let reader = mt.reader();
+    let keys: Vec<Vec<u8>> = scan(&reader).into_iter().map(|(k, _)| k).collect();
+    let row_of = |k: &[u8]| k[..pigeonhole_format::key::row_prefix_len(k).unwrap()].to_vec();
+    let mut it = reader.iter();
+    for (i, k) in keys.iter().enumerate() {
+        let want = keys[i + 1..].iter().find(|n| row_of(n) != row_of(k));
+        it.seek(k).unwrap();
+        it.skip_row().unwrap();
+        assert_eq!(
+            it.valid().then(|| it.key().to_vec()).as_ref(),
+            want,
+            "from entry {i}"
+        );
+        // And after stepping to it, as a filter's `SkipRow` does mid-row (a finger search).
+        if i > 0 {
+            it.seek(&keys[i - 1]).unwrap();
+            it.next().unwrap();
+            it.skip_row().unwrap();
+            assert_eq!(
+                it.valid().then(|| it.key().to_vec()).as_ref(),
+                want,
+                "stepped to {i}"
+            );
+        }
+    }
+}
+
+#[test]
+fn header_follows_format() {
+    let mut arena = small();
+    let mut mt = Memtable::create(&mut arena).unwrap();
+    let region = arena.region().clone();
+    let mem = &region.mem;
+    let root = mt.root() as usize;
+    assert_eq!(mem.read_u32(root + layout::H_MAGIC), MEMTABLE_MAGIC);
+    assert_eq!(mem.copy(root + layout::H_MAGIC, 4), b"MEMT");
+    assert_eq!(mem.read_u8(root + layout::H_VERSION), 2);
+    assert_eq!(mem.read_u8(root + layout::H_VERSION + 1), 0);
+    assert_eq!(mem.read_u8(root + layout::H_FLAGS), 0);
+    let head = mem.read_u32(root + layout::H_HEAD) as usize;
+    assert_eq!(head, root + layout::HEADER_LEN);
+    assert_eq!(mem.read_u8(head + layout::N_HEIGHT), 16);
+    assert_eq!(mem.load_u32(root + layout::H_COUNT, Ordering::Acquire), 0);
+    assert_eq!(
+        mem.load_u64(root + layout::H_BYTES, Ordering::Acquire),
+        (layout::HEADER_LEN + HEAD_LEN) as u64
+    );
+
+    mt.insert(&mut arena, &key(b"r", 7), b"\x00v").unwrap();
+    mt.insert(&mut arena, &key(b"s", 3), b"\x00w").unwrap();
+    assert_eq!(mem.load_u32(root + layout::H_COUNT, Ordering::Acquire), 2);
+    assert_eq!(
+        mem.load_u64(root + layout::H_MIN_SEQNO, Ordering::Acquire),
+        3
+    );
+    assert_eq!(
+        mem.load_u64(root + layout::H_MAX_SEQNO, Ordering::Acquire),
+        7
+    );
+    assert!(mem.load_u64(root + layout::H_BYTES, Ordering::Acquire) > 140);
+    assert_eq!(mt.seqno_range(), Some((3, 7)));
+
+    mt.freeze();
+    assert!(mt.is_frozen());
+    assert_eq!(mem.load_u8(root + layout::H_FLAGS, Ordering::Acquire), 1);
+}
+
+#[test]
+fn node_layout_follows_format() {
+    let mut arena = small();
+    let mut mt = Memtable::create(&mut arena).unwrap();
+    let k = key(b"row", 1);
+    mt.insert(&mut arena, &k, b"\x00value").unwrap();
+    let mem = &arena.region().mem;
+    let head = mt.head as usize;
+    let node = mem.load_u32(head + layout::N_TOWER, Ordering::Acquire) as usize;
+    assert_eq!(node % 4, 0);
+    assert_eq!(mem.read_u32(node + layout::N_KEY_LEN) as usize, k.len());
+    assert_eq!(mem.read_u32(node + layout::N_VALUE_LEN), 6);
+    let height = mem.read_u8(node + layout::N_HEIGHT) as usize;
+    assert!((1..=16).contains(&height));
+    assert_eq!(mem.copy(node + layout::N_HEIGHT + 1, 3), [0, 0, 0]);
+    for level in 0..height {
+        assert_eq!(
+            mem.load_u32(tower(node as u32, level), Ordering::Acquire),
+            0
+        );
+    }
+    let key_off = node + layout::N_TOWER + 4 * height;
+    assert_eq!(mem.copy(key_off, k.len()), k);
+    assert_eq!(mem.copy(key_off + k.len(), 6), b"\x00value");
+}
+
+#[test]
+fn chunks_spill_fill_and_reclaim() {
+    let mut arena = small();
+    assert_eq!(arena.free_bytes(), 64 * 1024);
+    let mut mt = Memtable::create(&mut arena).unwrap();
+    assert_eq!(mt.allocated_bytes(), 1024);
+    assert_eq!(arena.free_bytes(), 63 * 1024);
+
+    let value = vec![0u8; 300];
+    let mut n = 0u64;
+    let err = loop {
+        n += 1;
+        match mt.insert(&mut arena, &key(&n.to_be_bytes(), n), &value) {
+            Ok(()) => {}
+            Err(e) => break e,
+        }
+    };
+    assert_eq!(err, Error::ArenaFull);
+    // Each node is 346..406 bytes, so two fit a 1 KiB chunk (chunk 0 included: 140 bytes of
+    // header and head plus two nodes stay under its 960 usable bytes): 64 chunks, 128 entries.
+    assert_eq!(mt.len(), 128);
+    assert_eq!(mt.allocated_bytes(), 64 * 1024);
+    assert_eq!(arena.free_bytes(), 0);
+    assert_eq!(scan(&mt.reader()).len(), mt.len());
+
+    // Reclaim, then a new memtable reuses the dirty chunks and starts clean.
+    let old_root = mt.root();
+    arena.reclaim(mt.retire());
+    assert_eq!(arena.free_bytes(), 64 * 1024);
+    let mut mt2 = Memtable::create(&mut arena).unwrap();
+    assert_eq!(mt2.root(), old_root);
+    assert!(scan(&mt2.reader()).is_empty());
+    mt2.insert(&mut arena, &key(b"k", 1), b"").unwrap();
+    assert_eq!(scan(&mt2.reader()).len(), 1);
+}
+
+#[test]
+fn large_entries_take_contiguous_runs() {
+    let mut arena = small();
+    let mut mt = Memtable::create(&mut arena).unwrap();
+    // Larger than a chunk: a run of three.
+    let big = vec![9u8; 2500];
+    mt.insert(&mut arena, &key(b"big", 1), &big).unwrap();
+    assert_eq!(mt.allocated_bytes(), 4 * 1024);
+    assert_eq!(mt.runs.len(), 2);
+    assert_eq!(mt.runs[1].count, 3);
+    // Larger than the arena can ever hold.
+    assert_eq!(
+        mt.insert(&mut arena, &key(b"huge", 2), &vec![0; 64 * 1024]),
+        Err(Error::EntryTooLarge)
+    );
+    assert_eq!(
+        mt.insert(&mut arena, &key(b"huge", 2), &vec![0; 64 * 1024 - 100]),
+        Err(Error::EntryTooLarge),
+        "the reserved prefix and the header count against the arena"
+    );
+    let mut it = mt.reader().iter();
+    it.seek_to_first().unwrap();
+    assert_eq!(it.value(), &big[..]);
+    let slice = it.value_slice();
+    drop(it);
+    assert_eq!(slice.len(), 2500);
+}
+
+#[test]
+fn first_fit_skips_allocated_chunks() {
+    let mut arena = ShardArena::new(ArenaRegion::heap(8 * 1024), 1024);
+    let a = Memtable::create(&mut arena).unwrap();
+    let b = Memtable::create(&mut arena).unwrap();
+    let c = Memtable::create(&mut arena).unwrap();
+    assert_eq!((a.root(), b.root(), c.root()), (64, 1024, 2048));
+    arena.reclaim(b.retire());
+    let d = Memtable::create(&mut arena).unwrap();
+    assert_eq!(d.root(), 1024, "the freed middle chunk is reused first");
+    // A run of two cannot use the single free chunk between a and c... it is taken by d
+    // now, so the run comes after c.
+    let mut d = d;
+    d.insert(&mut arena, &key(b"x", 1), &vec![0; 1500]).unwrap();
+    assert_eq!(d.runs[1], Run { first: 3, count: 2 });
+    assert_eq!(arena.free_bytes(), 3 * 1024);
+}
+
+#[test]
+#[should_panic(expected = "frozen")]
+fn insert_into_frozen_panics() {
+    let mut arena = small();
+    let mut mt = Memtable::create(&mut arena).unwrap();
+    mt.freeze();
+    let _ = mt.insert(&mut arena, &key(b"a", 1), b"");
+}
+
+#[test]
+#[should_panic(expected = "chunk size")]
+fn bad_chunk_size_panics() {
+    ShardArena::new(ArenaRegion::heap(4096), 100);
+}
+
+#[test]
+fn malformed_key_rejected() {
+    let mut arena = small();
+    let mut mt = Memtable::create(&mut arena).unwrap();
+    assert_eq!(
+        mt.insert(&mut arena, b"short", b""),
+        Err(Error::Corrupt("insert key is not an internal key"))
+    );
+    assert!(mt.is_empty());
+}
+
+#[test]
+fn duplicate_keys_are_both_visible() {
+    let mut arena = small();
+    let mut mt = Memtable::create(&mut arena).unwrap();
+    let k = key(b"dup", 1);
+    mt.insert(&mut arena, &k, b"\x00a").unwrap();
+    mt.insert(&mut arena, &k, b"\x00b").unwrap();
+    let entries = scan(&mt.reader());
+    assert_eq!(entries.len(), 2);
+    assert!(entries.iter().all(|(key, _)| key == &k));
+}
+
+#[test]
+fn create_fails_when_arena_is_full() {
+    let mut arena = ShardArena::new(ArenaRegion::heap(2048), 1024);
+    let _a = Memtable::create(&mut arena).unwrap();
+    let _b = Memtable::create(&mut arena).unwrap();
+    assert_eq!(Memtable::create(&mut arena).unwrap_err(), Error::ArenaFull);
+    let mut tiny = ShardArena::new(ArenaRegion::heap(512), 1024);
+    assert_eq!(
+        Memtable::create(&mut tiny).unwrap_err(),
+        Error::EntryTooLarge
+    );
+}
+
+#[test]
+fn arena_region_validation() {
+    let region = SharedRegion::heap(8192);
+    let err = |r: Result<ArenaRegion>| r.unwrap_err();
+    assert_eq!(
+        err(ArenaRegion::new(region.clone(), 32, 64)),
+        Error::Corrupt("arena offset is not 64-byte aligned")
+    );
+    assert_eq!(
+        err(ArenaRegion::new(region.clone(), 64, 8192)),
+        Error::Corrupt("arena extends past its region")
+    );
+    assert_eq!(
+        err(ArenaRegion::new(region.clone(), 0, MAX_ARENA_LEN + 1)),
+        Error::Corrupt("arena longer than 4 GiB")
+    );
+    let arena = ArenaRegion::new(region.clone(), 4096, 4096).unwrap();
+    assert_eq!(arena.len(), 4096);
+    assert!(ArenaRegion::new(region, 8192, 0).unwrap().is_empty());
+
+    // A memtable in a sub-range of a region is readable through the same region bytes.
+    let mut shard = ShardArena::new(arena, 1024);
+    let mut mt = Memtable::create(&mut shard).unwrap();
+    mt.insert(&mut shard, &key(b"k", 1), b"\x00v").unwrap();
+    assert_eq!(scan(&mt.reader()).len(), 1);
+}
+
+#[test]
+fn open_validates_header() {
+    let mut arena = small();
+    let mut mt = Memtable::create(&mut arena).unwrap();
+    mt.insert(&mut arena, &key(b"k", 1), b"\x00v").unwrap();
+    let region = arena.region().clone();
+    let root = mt.root();
+    assert!(MemtableReader::open(region.clone(), root).is_ok());
+
+    let corrupt = |root: u32| MemtableReader::open(region.clone(), root).unwrap_err();
+    assert_eq!(corrupt(0), Error::Corrupt("memtable root is misaligned"));
+    assert_eq!(
+        corrupt(root + 4),
+        Error::Corrupt("memtable root is misaligned")
+    );
+    assert_eq!(
+        corrupt(64 * 1024 - 32),
+        Error::Corrupt("memtable header outside arena")
+    );
+    assert_eq!(corrupt(root + 128), Error::Corrupt("memtable header magic"));
+
+    let mem = &region.mem;
+    let saved = mem.copy(root as usize, layout::HEADER_LEN);
+    // A version this build does not know, and version 1 (nodes without a stale-tail field,
+    // before D277): both refused.
+    mem.write(root as usize + layout::H_VERSION, &[3, 0]);
+    assert_eq!(corrupt(root), Error::Corrupt("memtable header version"));
+    mem.write(root as usize + layout::H_VERSION, &[1, 0]);
+    assert_eq!(corrupt(root), Error::Corrupt("memtable header version"));
+    mem.write(root as usize, &saved);
+
+    mem.write_u32(root as usize + layout::H_HEAD, 64 * 1024 - 4);
+    assert_eq!(corrupt(root), Error::Corrupt("node header outside arena"));
+    mem.write_u32(root as usize + layout::H_HEAD, 3);
+    assert_eq!(corrupt(root), Error::Corrupt("node offset"));
+    // Point the head at the data node: wrong height for a head.
+    let node = mem.load_u32(tower(mt.head, 0), Ordering::Acquire);
+    mem.write_u32(root as usize + layout::H_HEAD, node);
+    assert_eq!(corrupt(root), Error::Corrupt("memtable head node"));
+    mem.write(root as usize, &saved);
+    assert!(MemtableReader::open(region.clone(), root).is_ok());
+}
+
+#[test]
+fn reader_reports_corrupt_nodes_instead_of_faulting() {
+    let mut arena = small();
+    let mut mt = Memtable::create(&mut arena).unwrap();
+    for i in 1..=3 {
+        mt.insert(&mut arena, &key(&[i as u8], i), b"\x00v")
+            .unwrap();
+    }
+    let reader = mt.reader();
+    let mem = &arena.region().mem;
+    let first = mem.load_u32(tower(mt.head, 0), Ordering::Acquire);
+    let second = mem.load_u32(tower(first, 0), Ordering::Acquire);
+    let saved = mem.copy(second as usize, layout::N_TOWER);
+
+    let step = || {
+        let mut it = reader.iter();
+        it.seek_to_first()?;
+        it.next()
+    };
+    mem.write(second as usize + layout::N_HEIGHT, &[0]);
+    assert_eq!(step(), Err(Error::Corrupt("node height")));
+    mem.write(second as usize + layout::N_HEIGHT, &[17]);
+    assert_eq!(step(), Err(Error::Corrupt("node height")));
+    mem.write(second as usize, &saved);
+    mem.write_u32(second as usize + layout::N_KEY_LEN, u32::MAX);
+    assert_eq!(step(), Err(Error::Corrupt("node outside arena")));
+    mem.write(second as usize, &saved);
+    mem.write_u32(second as usize + layout::N_VALUE_LEN, 64 * 1024);
+    assert_eq!(step(), Err(Error::Corrupt("node outside arena")));
+    mem.write(second as usize, &saved);
+
+    // Bad links: misaligned, past the arena, and a seek that crosses them.
+    mem.store_u32(tower(first, 0), 64 * 1024 - 2, Ordering::Release);
+    assert_eq!(step(), Err(Error::Corrupt("node offset")));
+    mem.store_u32(tower(first, 0), 64 * 1024 - 8, Ordering::Release);
+    assert_eq!(step(), Err(Error::Corrupt("node header outside arena")));
+    let mut it = reader.iter();
+    assert_eq!(
+        it.seek(&key(&[9], 9)),
+        Err(Error::Corrupt("node header outside arena"))
+    );
+    mem.store_u32(tower(first, 0), second, Ordering::Release);
+    assert_eq!(step(), Ok(()));
+    assert_eq!(scan(&reader).len(), 3);
+}
+
+#[test]
+fn no_pointers_in_the_arena() {
+    // Scaled down under Miri (the byte scan is interpreted).
+    let (len, entries) = if cfg!(miri) {
+        (16 * 1024, 80u64)
+    } else {
+        (256 * 1024, 1000)
+    };
+    let mut arena = ShardArena::new(ArenaRegion::heap(len), 4096);
+    let mut mt = Memtable::create(&mut arena).unwrap();
+    for i in 1..=entries {
+        mt.insert(&mut arena, &key(&i.to_le_bytes(), i), &i.to_le_bytes())
+            .unwrap();
+    }
+    let mem = &arena.region().mem;
+    let bytes = mem.copy(0, mem.len());
+    let base = mem.base_addr();
+    let needles = [
+        base.to_ne_bytes(),
+        (base + mt.root() as usize).to_ne_bytes(),
+    ];
+    for needle in needles {
+        assert!(
+            !bytes.windows(needle.len()).any(|w| w == needle),
+            "found an absolute address in the arena"
+        );
+    }
+}
+
+#[test]
+fn arena_bytes_relocate_to_another_address() {
+    let (len, entries) = if cfg!(miri) {
+        (32 * 1024, 60u64)
+    } else {
+        (256 * 1024, 500)
+    };
+    let mut arena = ShardArena::new(ArenaRegion::heap(len), 4096);
+    let mut mt = Memtable::create(&mut arena).unwrap();
+    let mut expected = Vec::new();
+    for i in 1..=entries {
+        let k = key(&(i * 7919 % 1000).to_be_bytes(), i);
+        let v = i.to_le_bytes().to_vec();
+        mt.insert(&mut arena, &k, &v).unwrap();
+        expected.push((k, v));
+    }
+    expected.sort();
+    mt.freeze();
+
+    // Copy the raw arena to a different allocation: offsets must still resolve.
+    let copy = ArenaRegion::heap(len);
+    assert_ne!(copy.mem.base_addr(), arena.region().mem.base_addr());
+    let bytes = arena.region().mem.copy(0, len);
+    copy.mem.write(0, &bytes);
+    let reader = MemtableReader::open(copy, mt.root()).unwrap();
+    assert_eq!(reader.len(), entries as usize);
+    assert_eq!(scan(&reader), expected);
+    let mut it = reader.iter();
+    for (k, v) in expected.iter().step_by(37) {
+        it.seek(k).unwrap();
+        assert_eq!((it.key(), it.value()), (&k[..], &v[..]));
+    }
+}
+
+#[test]
+fn heights_are_geometric_and_bounded() {
+    let mut arena = small();
+    let mut mt = Memtable::create(&mut arena).unwrap();
+    let draws = if cfg!(miri) { 1000 } else { 10_000 };
+    let mut tall = 0;
+    for _ in 0..draws {
+        let h = mt.random_height();
+        assert!((1..=MAX_HEIGHT).contains(&h));
+        if h >= 2 {
+            tall += 1;
+        }
+    }
+    // One in four nodes reaches level 2 (binomial, within about seven standard deviations).
+    assert!(
+        (draws * 22 / 100..draws * 28 / 100).contains(&tall),
+        "{tall} tall nodes"
+    );
+}
+
+#[test]
+fn error_display() {
+    assert_eq!(Error::ArenaFull.to_string(), "shard arena is full");
+    assert_eq!(
+        Error::EntryTooLarge.to_string(),
+        "entry is larger than the arena can hold"
+    );
+    assert_eq!(Error::Corrupt("x").to_string(), "corrupt memtable: x");
+}
+
+#[test]
+fn reader_and_slices_are_send_and_sync() {
+    fn assert_send_sync<T: Send + Sync>() {}
+    assert_send_sync::<MemtableReader>();
+    assert_send_sync::<MemIter>();
+    assert_send_sync::<ArenaSlice>();
+    assert_send_sync::<ArenaRegion>();
+    fn assert_send<T: Send>() {}
+    assert_send::<Memtable>();
+    assert_send::<ShardArena>();
+    assert_send::<Retired>();
+}
+
+#[test]
+fn reclaim_waits_for_in_process_readers() {
+    let mut arena = small();
+    let mut mt = Memtable::create(&mut arena).unwrap();
+    let value = vec![0x5a; 100];
+    mt.insert(&mut arena, &key(b"k", 1), &value).unwrap();
+    let root = mt.root();
+    let reader = mt.reader();
+    let mut it = reader.iter();
+    it.seek_to_first().unwrap();
+    let slice = it.value_slice();
+
+    arena.reclaim(mt.retire());
+    assert_eq!(
+        arena.free_bytes(),
+        63 * 1024,
+        "still borrowed by the cursor"
+    );
+    // A new memtable does not reuse the borrowed chunk; the borrowed bytes stay intact.
+    let mut other = Memtable::create(&mut arena).unwrap();
+    assert_ne!(other.root(), root);
+    other.insert(&mut arena, &key(b"x", 2), &[1; 100]).unwrap();
+    assert_eq!(it.value(), &value[..]);
+    assert_eq!(&*slice, &value[..]);
+    assert_eq!(reader.len(), 1);
+
+    // The last handle to go releases the chunk, at the next reclaim or allocation.
+    drop(it);
+    drop(reader);
+    assert_eq!(arena.free_bytes(), 62 * 1024, "the slice still pins");
+    drop(slice);
+    assert_eq!(
+        arena.free_bytes(),
+        62 * 1024,
+        "nothing has run the release yet"
+    );
+    let third = Memtable::create(&mut arena).unwrap();
+    assert_eq!(third.root(), root, "the released chunk is reused first");
+    assert_eq!(arena.free_bytes(), 62 * 1024);
+    arena.reclaim(third.retire());
+    arena.reclaim(other.retire());
+    assert_eq!(arena.free_bytes(), 64 * 1024);
+}
+
+#[test]
+fn reader_opened_by_root_pins_in_the_writer_process() {
+    let mut arena = small();
+    let mt = Memtable::create(&mut arena).unwrap();
+    let root = mt.root();
+    let opened = MemtableReader::open(arena.region().clone(), root).unwrap();
+    arena.reclaim(mt.retire());
+    assert_eq!(arena.free_bytes(), 63 * 1024);
+    assert_ne!(Memtable::create(&mut arena).unwrap().root(), root);
+    drop(opened);
+    let probe = Memtable::create(&mut arena).unwrap();
+    assert_eq!(
+        arena.free_bytes(),
+        62 * 1024,
+        "the old chunk was released on allocation, then one was taken"
+    );
+    arena.reclaim(probe.retire());
+    assert_eq!(arena.free_bytes(), 63 * 1024);
+    assert_eq!(Memtable::create(&mut arena).unwrap().root(), root);
+
+    // A reader opened on a different `ArenaRegion` value (another process's mapping, or
+    // raw bytes copied elsewhere) pins nothing here, as it cannot.
+    let copy = ArenaRegion::heap(64 * 1024);
+    copy.mem.write(0, &arena.region().mem.copy(0, 64 * 1024));
+    let _foreign = MemtableReader::open(copy, root).unwrap();
+}
+
+#[test]
+fn pinned_retired_chunks_count_as_allocated() {
+    let mut arena = ShardArena::new(ArenaRegion::heap(2048), 1024);
+    let a = Memtable::create(&mut arena).unwrap();
+    let reader = a.reader();
+    arena.reclaim(a.retire());
+    let _b = Memtable::create(&mut arena).unwrap();
+    assert_eq!(Memtable::create(&mut arena).unwrap_err(), Error::ArenaFull);
+    drop(reader);
+    assert_eq!(Memtable::create(&mut arena).unwrap().root(), 64);
+}
+
+#[test]
+#[should_panic(expected = "different shard arena")]
+fn insert_through_another_arena_panics() {
+    let mut arena = small();
+    let mut other = small();
+    let mut mt = Memtable::create(&mut arena).unwrap();
+    let _ = mt.insert(&mut other, &key(b"k", 1), b"");
+}
+
+#[test]
+fn reader_rejects_a_node_linked_above_its_height() {
+    let mut arena = ShardArena::new(ArenaRegion::heap(64 * 1024), 4096);
+    let mut mt = Memtable::create(&mut arena).unwrap();
+    for i in 1..=if cfg!(miri) { 40u64 } else { 200 } {
+        mt.insert(&mut arena, &key(&i.to_be_bytes(), i), b"")
+            .unwrap();
+    }
+    let region = arena.region().clone();
+    let mem = &region.mem;
+    let tall = mem.load_u32(tower(mt.head, 1), Ordering::Acquire);
+    assert_ne!(tall, 0, "some node reaches level 1");
+    assert!(mem.read_u8(tall as usize + layout::N_HEIGHT) >= 2);
+    mem.write(tall as usize + layout::N_HEIGHT, &[1]);
+    let mut it = mt.reader().iter();
+    assert_eq!(
+        it.seek(b""),
+        Err(Error::Corrupt("node linked above its height"))
+    );
+    // Level 0 never follows a link above a height of 1, so a scan still works.
+    it.seek_to_first().unwrap();
+    assert!(it.valid());
+}
+
+#[test]
+fn reader_reports_link_cycles() {
+    let mut arena = small();
+    let mut mt = Memtable::create(&mut arena).unwrap();
+    for i in 1..=3u8 {
+        mt.insert(&mut arena, &key(&[i], i as u64), b"").unwrap();
+    }
+    let region = arena.region().clone();
+    let mem = &region.mem;
+    let first = mem.load_u32(tower(mt.head, 0), Ordering::Acquire);
+    let second = mem.load_u32(tower(first, 0), Ordering::Acquire);
+    // second -> first: a level-0 cycle.
+    mem.store_u32(tower(second, 0), first, Ordering::Release);
+
+    let reader = mt.reader();
+    let mut it = reader.iter();
+    it.seek_to_first().unwrap();
+    let result = loop {
+        match it.next() {
+            Ok(()) if it.valid() => {}
+            other => break other,
+        }
+    };
+    assert_eq!(result, Err(Error::Corrupt("link cycle")));
+    assert_eq!(it.seek(&key(&[9], 9)), Err(Error::Corrupt("link cycle")));
+}
+
+#[test]
+fn largest_free_run_is_the_largest_contiguous_free_space() {
+    // 8 chunks of 1 KiB; the first loses the reserved 64-byte prefix.
+    let mut arena = ShardArena::new(ArenaRegion::heap(8 * 1024), 1024);
+    assert_eq!(arena.largest_free_run(), 8 * 1024 - 64);
+    let a = Memtable::create(&mut arena).unwrap(); // chunk 0
+    let mut b = Memtable::create(&mut arena).unwrap(); // chunk 1
+    // An entry of two chunks takes chunks 2-3.
+    b.insert(&mut arena, &key(b"r", 1), &[0u8; 1500]).unwrap();
+    assert_eq!(arena.largest_free_run(), 4 * 1024);
+    // Freeing chunk 0 leaves runs of 1 and 4 chunks.
+    arena.reclaim(a.retire());
+    assert_eq!(arena.free_bytes(), 5 * 1024);
+    assert_eq!(arena.largest_free_run(), 4 * 1024);
+    // Runs of 1 and 4 chunks: five 1-chunk blocks, two of 2, one of 3 or 4, none of 5.
+    assert_eq!(
+        (1..=5).map(|k| arena.blocks_left(k, 0)).collect::<Vec<_>>(),
+        [5, 2, 1, 1, 0]
+    );
+    // Blocks of 3: the 4-run holds one with a spare chunk, so taking two chunks from its
+    // start breaks it, and one does not.
+    assert_eq!(arena.blocks_left(3, 1), 1);
+    assert_eq!(arena.blocks_left(3, 2), 0);
+    // Blocks of 2: breaking the 4-run's first block costs 1, its second 2 more.
+    assert_eq!(
+        (0..5).map(|r| arena.blocks_left(2, r)).collect::<Vec<_>>(),
+        [2, 1, 1, 0, 0]
+    );
+}
+
+/// ICR 0020: the writer knows whether a memtable holds a delete marker, and so does every
+/// handle sharing its pin.
+#[test]
+fn may_have_markers_is_known_only_to_the_writer() {
+    let mut arena = ShardArena::new(ArenaRegion::heap(1 << 20), 64 * 1024);
+    let mut mt = Memtable::create(&mut arena).unwrap();
+    let reader = mt.reader();
+    assert!(!reader.may_have_markers());
+    // Opened by root while the writer lives: the same pin, the same knowledge.
+    let by_root = MemtableReader::open(arena.region().clone(), mt.root()).unwrap();
+    assert!(!by_root.may_have_markers());
+    mt.note_marker();
+    let mut k = Vec::new();
+    pigeonhole_format::encode_key(&mut k, b"r", b"", 1, 1, pigeonhole_format::Kind::Put).unwrap();
+    mt.insert(&mut arena, &k, b"").unwrap();
+    assert!(reader.may_have_markers() && by_root.may_have_markers());
+    // A handle without the writer's pin (a reader process's, opened from a published view)
+    // always answers "maybe": `Pin::foreign`.
+    drop((reader, by_root));
+    let retired = mt.retire();
+    arena.reclaim(retired);
+}
+
+/// D194: wherever `skip_column` jumps, it lands on the first entry after the current one
+/// that is outside the current column, in a memtable built in random order (versions
+/// inserted newest last, oldest last, and interleaved, so tails go stale). Prints the seed
+/// on failure.
+#[test]
+fn skip_column_lands_on_the_next_column() {
+    // Miri checks the index's unsafe code on a few seeds; the rest are for the logic.
+    let seeds = if cfg!(miri) { 2 } else { 40u64 };
+    for seed in 1..=seeds {
+        let mut x = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15);
+        let mut rand = move |n: u64| {
+            x ^= x << 13;
+            x ^= x >> 7;
+            x ^= x << 17;
+            x % n
+        };
+        let mut arena = ShardArena::new(ArenaRegion::heap(4 << 20), 64 * 1024);
+        let mut mt = Memtable::create(&mut arena).unwrap().with_tail_index();
+        // Opened by root in the writer's process: it shares the writer's pin, and with it the
+        // index. (A reader process's registry has no pin for the root, so it has none.)
+        let by_root = MemtableReader::open(arena.region().clone(), mt.root()).unwrap();
+        let mut seqno = 0;
+        for _ in 0..if cfg!(miri) { 120 } else { 600 } {
+            let row = [b'r', b'0' + rand(4) as u8];
+            let qual = [b'q', b'0' + rand(5) as u8];
+            // Mostly newer timestamps (the overwrite pattern), sometimes older or equal.
+            let ts = match rand(10) {
+                0 => rand(50),
+                _ => 50 + seqno,
+            };
+            seqno += 1;
+            let kind = match rand(20) {
+                0 => Kind::CellDelete,
+                1 => Kind::Merge,
+                _ => Kind::Put,
+            };
+            let mut k = Vec::new();
+            encode_key(&mut k, &row, &qual, ts, seqno, kind).unwrap();
+            mt.insert(&mut arena, &k, b"\x00v").unwrap();
+        }
+        let all: Vec<Vec<u8>> = scan(&mt.reader()).into_iter().map(|(k, _)| k).collect();
+        let column = |k: &[u8]| k[..k.len() - SUFFIX_LEN].to_vec();
+        let mut jumps = 0;
+        for (i, k) in all.iter().enumerate() {
+            let col = column(k);
+            let want = all[i + 1..].iter().find(|n| column(n) != col);
+            let mut it = mt.reader().iter();
+            it.seek(k).unwrap();
+            if it.skip_column(&col).unwrap() {
+                jumps += 1;
+                assert_eq!(
+                    it.valid().then(|| it.key().to_vec()).as_ref(),
+                    want,
+                    "seed {seed}, entry {i}"
+                );
+            } else {
+                assert_eq!(it.key(), &k[..], "seed {seed}: a refused skip moved");
+            }
+            // A reader opened by root lands where the writer's own does.
+            let mut other = by_root.iter();
+            other.seek(k).unwrap();
+            let mut own = mt.reader().iter();
+            own.seek(k).unwrap();
+            assert_eq!(
+                other.skip_column(&col).unwrap(),
+                own.skip_column(&col).unwrap(),
+                "seed {seed}"
+            );
+            assert_eq!(
+                other.valid().then(|| other.key().to_vec()),
+                own.valid().then(|| own.key().to_vec()),
+                "seed {seed}"
+            );
+        }
+        assert!(
+            jumps > if cfg!(miri) { 10 } else { 100 },
+            "seed {seed}: only {jumps} jumps"
+        );
+    }
+}
+
+/// D277 (perf287's M1 and P1): a stale tail is an offset read from the arena, untrusted in a
+/// reader process. A bad one (past the arena, misaligned, inside the header, at a node of
+/// height 0, pointing backward, or at a valid node of another column) surfaces as
+/// `Corrupt` from `skip_column`, never as a panic, a fault or a silent skip.
+#[test]
+fn a_bad_stale_tail_is_corrupt_never_a_fault() {
+    let mut arena = small();
+    let mut mt = Memtable::create(&mut arena).unwrap().with_tail_index();
+    let ckey = |q: &[u8], ts| {
+        let mut k = Vec::new();
+        encode_key(&mut k, b"row", q, ts, ts, Kind::Put).unwrap();
+        k
+    };
+    for ts in 1..=5 {
+        mt.insert(&mut arena, &ckey(b"a", ts), b"\x00v").unwrap();
+    }
+    mt.insert(&mut arena, &ckey(b"b", 1), b"\x00v").unwrap();
+    let column = ckey(b"a", 0)[..ckey(b"a", 0).len() - SUFFIX_LEN].to_vec();
+    let region = arena.region().clone();
+    let mem = &region.mem;
+    // The nodes in key order: a@5, a@4, ..., a@1, b@1.
+    let a5 = mem.load_u32(tower(mt.head, 0), Ordering::Acquire);
+    let a4 = mem.load_u32(tower(a5, 0), Ordering::Acquire);
+    let a3 = mem.load_u32(tower(a4, 0), Ordering::Acquire);
+    let mut b1 = a4;
+    while b1 != NULL
+        && mem.read_u32(b1 as usize + layout::N_KEY_LEN) as usize == column.len() + SUFFIX_LEN
+        && mem.cmp(
+            b1 as usize
+                + layout::N_TOWER
+                + 4 * mem.read_u8(b1 as usize + layout::N_HEIGHT) as usize,
+            column.len(),
+            &column,
+        ) == Cmp::Equal
+    {
+        b1 = mem.load_u32(tower(b1, 0), Ordering::Acquire);
+    }
+    let good = mem.read_u32(a4 as usize + layout::N_TAIL);
+    assert_ne!(good, NULL, "a@4 has a stale tail");
+    // Through a reader opened by root, as a reader process opens it.
+    let reader = MemtableReader::open(region.clone(), mt.root()).unwrap();
+    let skip = || {
+        let mut it = reader.iter();
+        it.seek_to_first()?;
+        it.next()?; // on a@4
+        it.skip_column(&column)
+    };
+    assert_eq!(skip(), Ok(true), "the good tail jumps");
+
+    // A node of height 0, written in unused arena space near its end.
+    let zero = (64 * 1024 - 128) as u32;
+    mem.write_u32(zero as usize + layout::N_KEY_LEN, 0);
+    mem.write_u32(zero as usize + layout::N_VALUE_LEN, 0);
+    mem.write(zero as usize + layout::N_HEIGHT, &[0, 0, 0, 0]);
+    // Wherever a bad tail lands, the reader reports corruption (which check fires depends
+    // on the bytes there); a tail on a valid node of another column names itself (P1).
+    let cases: [(&str, u32, Option<&str>); 6] = [
+        ("past the arena", (64 * 1024 - 4) as u32, None),
+        ("misaligned", a3 + 1, Some("node offset")),
+        ("inside the header", mt.root() + 4, None),
+        ("at a node of height 0", zero, Some("node height")),
+        ("another column", b1, Some("stale tail outside its column")),
+        (
+            "backward, to the head",
+            mt.head,
+            Some("stale tail outside its column"),
+        ),
+    ];
+    for (what, bad, expect) in cases {
+        mem.write_u32(a4 as usize + layout::N_TAIL, bad);
+        match (skip(), expect) {
+            (Err(Error::Corrupt(got)), Some(want)) => assert_eq!(got, want, "{what}"),
+            (Err(Error::Corrupt(_)), None) => {}
+            (other, _) => panic!("{what}: {other:?}"),
+        }
+    }
+    mem.write_u32(a4 as usize + layout::N_TAIL, good);
+    assert_eq!(skip(), Ok(true));
+}
+
+/// D277 (perf287's option 2 on #934): the writer's own `reader()` follows the tails it wrote
+/// without validating them, and lands where a reader opened by root, which validates them,
+/// does.
+#[test]
+fn the_writers_reader_jumps_by_its_own_tails_as_a_checked_reader_does() {
+    let (arena, mt, column, _) = tail_fixture();
+    let checked = MemtableReader::open(arena.region().clone(), mt.root()).unwrap();
+    let land = |reader: &MemtableReader| {
+        let mut it = reader.iter();
+        it.seek_to_first().unwrap();
+        it.next().unwrap(); // on a@4
+        assert_eq!(it.skip_column(&column), Ok(true), "the tail jumps");
+        it.key().to_vec()
+    };
+    let landed = land(&mt.reader());
+    assert!(landed.starts_with(b"row") && !landed.starts_with(&column));
+    assert_eq!(landed, land(&checked));
+}
+
+/// D277 (perf287's condition 3 on #934): on the trusted path M1 and P1 stay as debug
+/// assertions, so a `record_tail` bug still fails the tests in debug builds.
+#[cfg(debug_assertions)]
+#[test]
+#[should_panic(expected = "outside its column")]
+fn a_bad_tail_on_the_writers_reader_fails_a_debug_assertion() {
+    let (arena, mt, column, b1) = tail_fixture();
+    let a5 = arena
+        .region()
+        .mem
+        .load_u32(tower(mt.head, 0), Ordering::Acquire);
+    let a4 = arena.region().mem.load_u32(tower(a5, 0), Ordering::Acquire);
+    arena
+        .region()
+        .mem
+        .write_u32(a4 as usize + layout::N_TAIL, b1);
+    let reader = mt.reader();
+    let mut it = reader.iter();
+    it.seek_to_first().unwrap();
+    it.next().unwrap();
+    let _ = it.skip_column(&column);
+}
+
+/// Column `a` at timestamps 5 down to 1, then `b@1`, with tails recorded: the arena, the
+/// memtable, column `a`, and `b@1`'s node.
+fn tail_fixture() -> (ShardArena, Memtable, Vec<u8>, u32) {
+    let mut arena = small();
+    let mut mt = Memtable::create(&mut arena).unwrap().with_tail_index();
+    let ckey = |q: &[u8], ts| {
+        let mut k = Vec::new();
+        encode_key(&mut k, b"row", q, ts, ts, Kind::Put).unwrap();
+        k
+    };
+    for ts in 1..=5 {
+        mt.insert(&mut arena, &ckey(b"a", ts), b"\x00v").unwrap();
+    }
+    mt.insert(&mut arena, &ckey(b"b", 1), b"\x00v").unwrap();
+    let column = ckey(b"a", 0)[..ckey(b"a", 0).len() - SUFFIX_LEN].to_vec();
+    let mut b1 = mt.head;
+    for _ in 0..6 {
+        b1 = arena.region().mem.load_u32(tower(b1, 0), Ordering::Acquire);
+    }
+    (arena, mt, column, b1)
+}
+
+/// D277 (perf287's P2): chunks are reused without zeroing, so every insert writes its node's
+/// stale-tail field before the node is linked, whether or not the memtable records tails.
+/// A memtable built over chunks whose previous memtable recorded tails everywhere shows a
+/// null tail on every node that has none, with tails recorded or not.
+#[test]
+fn reused_chunks_never_show_a_previous_memtables_tails() {
+    for tails in [false, true] {
+        let mut arena = small();
+        let ckey = |q: &[u8], ts| {
+            let mut k = Vec::new();
+            encode_key(&mut k, b"row", q, ts, ts, Kind::Put).unwrap();
+            k
+        };
+        // Overwrites, oldest first: every version after the first records a tail.
+        let mut old = Memtable::create(&mut arena).unwrap().with_tail_index();
+        for q in 0..20u8 {
+            for ts in 1..=6 {
+                old.insert(&mut arena, &ckey(&[q], ts), b"\x00old").unwrap();
+            }
+        }
+        arena.reclaim(old.retire());
+        // Key order only, distinct columns: no node has a tail.
+        let mut new = Memtable::create(&mut arena).unwrap();
+        if tails {
+            new = new.with_tail_index();
+        }
+        for q in 0..120u8 {
+            new.insert(&mut arena, &ckey(&[q], 1), b"\x00new").unwrap();
+        }
+        let mem = &arena.region().mem;
+        let mut at = mem.load_u32(tower(new.head, 0), Ordering::Acquire);
+        let mut seen = 0;
+        while at != NULL {
+            assert_eq!(
+                mem.read_u32(at as usize + layout::N_TAIL),
+                NULL,
+                "tails {tails}: node {at} shows a stale tail"
+            );
+            seen += 1;
+            at = mem.load_u32(tower(at, 0), Ordering::Acquire);
+        }
+        assert_eq!(seen, 120);
+    }
+}
+
+#[test]
+fn a_reader_reads_its_own_memtable_only() {
+    // #1228: the engine's publish cache reuses a slot's readers while they still read the
+    // slot's memtables, compared by identity, never by root.
+    let mut arena = small();
+    let a = Memtable::create(&mut arena).unwrap();
+    let b = Memtable::create(&mut arena).unwrap();
+    let ra = a.reader();
+    assert!(ra.reads(&a) && ra.clone().reads(&a));
+    assert!(!ra.reads(&b) && b.reader().reads(&b));
+    // A memtable that reuses a reclaimed one's root is another memtable.
+    drop(ra);
+    let old_root = b.root();
+    arena.reclaim(b.retire());
+    let c = Memtable::create(&mut arena).unwrap();
+    assert_eq!(c.root(), old_root);
+    assert!(!a.reader().reads(&c) && c.reader().reads(&c));
+}