pigeonhole-db 1.1.0

data/vendor/memory/src/hooks.rs

@@ -0,0 +1,109 @@
+//! Hold and fault seams for tests (`__unstable-hooks` only, ICR 0085's convention). They
+//! pause or fail; they never run another path.
+
+use std::sync::{Arc, Condvar, Mutex, PoisonError};
+
+/// Where a test can hold a store operation.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[non_exhaustive]
+pub enum Hold {
+    /// A `remember` with near dedup, after its near-duplicate check and before its
+    /// transaction (simp724's F1 on #1386).
+    RememberAfterNear,
+}
+
+type Gate = Arc<(Mutex<State>, Condvar)>;
+
+#[derive(Debug, Default)]
+struct State {
+    reached: bool,
+    released: bool,
+}
+
+/// An armed hold: [`wait`](Self::wait) until an operation reaches it, then
+/// [`release`](Self::release) it. Dropping it releases too.
+#[derive(Debug)]
+pub struct Armed {
+    gate: Gate,
+}
+
+static ARMED: Mutex<Option<(Hold, Gate)>> = Mutex::new(None);
+
+/// Holds the next operation that reaches `at` (one at a time, process-wide).
+pub fn arm(at: Hold) -> Armed {
+    let gate = Arc::new((Mutex::new(State::default()), Condvar::new()));
+    *ARMED.lock().unwrap_or_else(PoisonError::into_inner) = Some((at, Arc::clone(&gate)));
+    Armed { gate }
+}
+
+impl Armed {
+    /// Waits until an operation is held here.
+    pub fn wait(&self) {
+        let (m, cv) = &*self.gate;
+        let mut s = m.lock().unwrap_or_else(PoisonError::into_inner);
+        while !s.reached {
+            s = cv.wait(s).unwrap_or_else(PoisonError::into_inner);
+        }
+    }
+
+    /// Lets the held operation go on.
+    pub fn release(&self) {
+        let (m, cv) = &*self.gate;
+        m.lock().unwrap_or_else(PoisonError::into_inner).released = true;
+        cv.notify_all();
+    }
+}
+
+impl Drop for Armed {
+    fn drop(&mut self) {
+        self.release();
+    }
+}
+
+pub(crate) fn hold(at: Hold) {
+    let gate = {
+        let mut armed = ARMED.lock().unwrap_or_else(PoisonError::into_inner);
+        match armed.as_ref() {
+            Some((h, _)) if *h == at => armed.take().map(|(_, g)| g),
+            _ => None,
+        }
+    };
+    let Some(gate) = gate else {
+        return;
+    };
+    let (m, cv) = &*gate;
+    let mut s = m.lock().unwrap_or_else(PoisonError::into_inner);
+    s.reached = true;
+    cv.notify_all();
+    while !s.released {
+        s = cv.wait(s).unwrap_or_else(PoisonError::into_inner);
+    }
+}
+
+static FAIL_LOOP_SAVE: Mutex<Option<String>> = Mutex::new(None);
+
+/// Fails the next save of the consolidation loop named `name` at its commit, with an error
+/// that isn't a conflict (simp724's L1 on #1432: a failed save mustn't fence the loop out).
+/// By name, so tests running in parallel don't take each other's.
+pub fn fail_next_loop_save(name: &str) {
+    *FAIL_LOOP_SAVE
+        .lock()
+        .unwrap_or_else(PoisonError::into_inner) = Some(name.to_owned());
+}
+
+pub(crate) fn take_fail_loop_save(name: &str) -> bool {
+    let mut armed = FAIL_LOOP_SAVE
+        .lock()
+        .unwrap_or_else(PoisonError::into_inner);
+    if armed.as_deref() == Some(name) {
+        *armed = None;
+        return true;
+    }
+    false
+}
+
+/// The IVF write records a store has waiting to settle (D311 T8): an inspection helper for
+/// the bounded-growth test (simp724's R1 on #1424).
+pub fn ivf_pending(store: &crate::Store) -> usize {
+    store.ivf_pending_len()
+}

data/vendor/memory/src/ivf.rs

@@ -0,0 +1,204 @@
+//! IVF recall for large namespaces (D311 T8, #1342, ICR 0107): a store created with
+//! [`StoreOptions::ivf`](crate::StoreOptions::ivf) keeps a `pigeonhole-vector` index of its
+//! memories' embeddings beside `emb:v`, keyed by the memory's row key, and recall searches it
+//! for a namespace of at least [`IvfOptions::min_namespace_len`] memories with an embedding,
+//! filtered to the namespace by its row-key prefix.
+
+use pigeonhole::{Snapshot, Transaction};
+use pigeonhole_vector::{Index as VIndex, ReadIndex as VReadIndex};
+
+use crate::error::{Error, Result, corrupt};
+use crate::layout::{EMBEDDED, STATS, count_row};
+use crate::recall::Rows;
+
+/// How a store's IVF recall works (D311 T8): set at create with
+/// [`StoreOptions::ivf`](crate::StoreOptions::ivf), fixed after.
+///
+/// ```
+/// use pigeonhole_memory::IvfOptions;
+///
+/// let o = IvfOptions::new().min_namespace_len(50_000).nprobe(32);
+/// assert_eq!(o.get_min_namespace_len(), 50_000);
+/// assert_eq!(o.get_nprobe(), Some(32));
+/// ```
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[non_exhaustive]
+pub struct IvfOptions {
+    min_namespace_len: u64,
+    nprobe: Option<u32>,
+    target_list_len: Option<u32>,
+}
+
+impl Default for IvfOptions {
+    fn default() -> Self {
+        IvfOptions {
+            min_namespace_len: 10_000,
+            nprobe: None,
+            target_list_len: None,
+        }
+    }
+}
+
+impl IvfOptions {
+    /// The defaults: the index for namespaces of 10,000 memories with an embedding or more,
+    /// and the vector index's own `nprobe` and list length.
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// A namespace with fewer memories with an embedding than `n` is recalled exactly,
+    /// as in a store without the index (T5: exact search stays under 15 ms at p99 up to a
+    /// few thousand per namespace). At least 1.
+    pub fn min_namespace_len(mut self, n: u64) -> Self {
+        self.min_namespace_len = n;
+        self
+    }
+
+    /// The lists a recall probes first (`pigeonhole_vector::IndexOptions::nprobe`), unless a
+    /// recall sets its own ([`Recall::nprobe`](crate::Recall::nprobe)).
+    pub fn nprobe(mut self, n: u32) -> Self {
+        self.nprobe = Some(n);
+        self
+    }
+
+    /// The index's target list length (`pigeonhole_vector::IndexOptions::target_list_len`).
+    pub fn target_list_len(mut self, n: u32) -> Self {
+        self.target_list_len = Some(n);
+        self
+    }
+
+    /// See [`min_namespace_len`](Self::min_namespace_len).
+    pub fn get_min_namespace_len(&self) -> u64 {
+        self.min_namespace_len
+    }
+
+    /// See [`nprobe`](Self::nprobe): `None` for the vector index's default.
+    pub fn get_nprobe(&self) -> Option<u32> {
+        self.nprobe
+    }
+
+    /// See [`target_list_len`](Self::target_list_len): `None` for the vector index's
+    /// default.
+    pub fn get_target_list_len(&self) -> Option<u32> {
+        self.target_list_len
+    }
+
+    pub(crate) fn check(&self) -> Result<()> {
+        if self.min_namespace_len == 0 {
+            return Err(Error::InvalidArgument(
+                "an IVF min_namespace_len is at least 1".into(),
+            ));
+        }
+        if self.nprobe == Some(0) || self.target_list_len == Some(0) {
+            return Err(Error::InvalidArgument(
+                "an IVF nprobe or target_list_len is at least 1".into(),
+            ));
+        }
+        Ok(())
+    }
+
+    /// `G:meta:ivf`: a version byte (1), `min_namespace_len` (8 bytes), then `nprobe` and
+    /// `target_list_len` (4 bytes each, 0 for the index's default), little endian.
+    pub(crate) fn encode(&self) -> Vec<u8> {
+        let mut b = vec![1u8];
+        b.extend_from_slice(&self.min_namespace_len.to_le_bytes());
+        b.extend_from_slice(&self.nprobe.unwrap_or(0).to_le_bytes());
+        b.extend_from_slice(&self.target_list_len.unwrap_or(0).to_le_bytes());
+        b
+    }
+
+    pub(crate) fn decode(b: &[u8]) -> Result<IvfOptions> {
+        let bad = || corrupt("the store's IVF options");
+        if b.len() != 17 || b[0] != 1 {
+            return Err(bad());
+        }
+        let min = u64::from_le_bytes(b[1..9].try_into().map_err(|_| bad())?);
+        let n = u32::from_le_bytes(b[9..13].try_into().map_err(|_| bad())?);
+        let t = u32::from_le_bytes(b[13..17].try_into().map_err(|_| bad())?);
+        Ok(IvfOptions {
+            min_namespace_len: min,
+            nprobe: (n != 0).then_some(n),
+            target_list_len: (t != 0).then_some(t),
+        })
+    }
+}
+
+/// The store's IVF index's name.
+pub(crate) fn ivf_name(store: &str) -> String {
+    format!("mem_{store}_ivf")
+}
+
+/// A seek into a probed list costs about this many scored cells: an estimate, not fitted
+/// (perf287 on #1342 and #1424). R1 showed a prefixed search seeks, and the T8 run's cap
+/// never bit. A prefixed search widens to at most `max(nprobe, count / SEEK_CELLS)` lists
+/// before recall searches the namespace exactly instead.
+pub(crate) const SEEK_CELLS: u64 = 64;
+
+/// The index, in the writer process or a reader's.
+#[derive(Clone)]
+pub(crate) enum Ivf {
+    W(VIndex, IvfOptions),
+    R(VReadIndex, IvfOptions),
+}
+
+impl Ivf {
+    pub(crate) fn options(&self) -> IvfOptions {
+        match self {
+            Ivf::W(_, o) | Ivf::R(_, o) => *o,
+        }
+    }
+
+    /// The `n` nearest memory rows under `prefix` (`{ns}\0m`) at `s`, or `None` when the
+    /// widening hit its cap first: the namespace is sparse among the probed lists, and an
+    /// exact search of it is cheaper (perf287's per-list fallback on #1342).
+    pub(crate) fn nearest(
+        &self,
+        s: &Snapshot,
+        prefix: &[u8],
+        q: &[f32],
+        n: usize,
+        count: u64,
+        nprobe: Option<u32>,
+    ) -> Result<Option<Vec<Vec<u8>>>> {
+        let o = self.options();
+        let first = nprobe.or(o.nprobe);
+        let run = |search: pigeonhole_vector::Search<'_>| -> Result<_> {
+            let search = match first {
+                Some(p) => search.nprobe(p),
+                None => search,
+            };
+            // Unset, the index's own `nprobe` is the first probe. `max_probe` never goes below
+            // the first probe, so the floor of 1 here means "the index's `nprobe`" (perf287's
+            // nit on #1424; `IndexOptions` doesn't expose `nprobe`).
+            let first = first.unwrap_or(1);
+            let cap = u32::try_from(count / SEEK_CELLS)
+                .unwrap_or(u32::MAX)
+                .max(first);
+            Ok(search
+                .id_prefix(prefix)
+                .max_probe(cap)
+                .snapshot(s)
+                .run_with_stats()?)
+        };
+        let (hits, stats) = match self {
+            Ivf::W(i, _) => run(i.search_f32(q, n))?,
+            Ivf::R(i, _) => run(i.search_f32(q, n))?,
+        };
+        if hits.len() < n && stats.lists_probed < stats.lists {
+            return Ok(None);
+        }
+        Ok(Some(hits.into_iter().map(|h| h.id).collect()))
+    }
+}
+
+/// The namespace counter at `s`: memories with an embedding (0 when absent).
+pub(crate) fn count(rows: &Rows<'_>, s: &Snapshot, ns: &str) -> Result<u64> {
+    let n = rows.counter_at(s, &count_row(ns), STATS, EMBEDDED)?;
+    Ok(n.unwrap_or(0).max(0) as u64)
+}
+
+/// The counter's blind increment in a write (no read: concurrent remembers don't conflict
+/// through it, simp724 on #1342).
+pub(crate) fn counted(txn: &mut Transaction, t: &pigeonhole::Table, ns: &str, by: i64) {
+    txn.incr(t, &count_row(ns), STATS, EMBEDDED, by);
+}

data/vendor/memory/src/layout.rs

@@ -0,0 +1,340 @@
+//! The store's layout (D311 §2, ICR 0107): row keys, families, the `MemoryId` forms, the
+//! dedup normalisation and hash.
+
+use std::fmt;
+use std::str::FromStr;
+
+use crate::error::{Error, Result};
+
+/// The memory rows' tag after the namespace: `{ns}\0m{rid}`.
+pub(crate) const MEMORY_TAG: &[u8] = b"\0m";
+/// The dedup rows' tag: `{ns}\0h{hash}`.
+pub(crate) const DEDUP_TAG: &[u8] = b"\0h";
+/// A namespace's counter row's tag: `{ns}\0c` (D311 T8), `stats:embedded` counting the
+/// namespace's memories with an embedding, so recall can choose the IVF index for a large
+/// one. A blind increment in each write: advisory, as drift only changes the path.
+pub(crate) const COUNT_TAG: &[u8] = b"\0c";
+/// The store's own row: the layout version, the embedding model, the text index's name.
+pub(crate) const GLOBAL: &[u8] = b"G";
+
+/// `content:text`, the text index's source column.
+pub(crate) const CONTENT: &str = "content";
+pub(crate) const TEXT: &[u8] = b"text";
+/// `emb:v`, the embedding.
+pub(crate) const EMB: &str = "emb";
+pub(crate) const V: &[u8] = b"v";
+/// `meta`: kind, session, pinned, consolidated, superseded_by, valid_to; and `rid` in a
+/// dedup row; and `G`'s columns.
+pub(crate) const META: &str = "meta";
+pub(crate) const KIND: &[u8] = b"kind";
+pub(crate) const SESSION: &[u8] = b"session";
+pub(crate) const PINNED: &[u8] = b"pinned";
+/// Set once a consolidation took the memory in (T3).
+pub(crate) const CONSOLIDATED: &[u8] = b"consolidated";
+/// `meta:summary` (`1`): a memory a consolidator returned (D311 T9), so watchers and the
+/// feed-driven loop can tell a pass's own writes from new memories.
+pub(crate) const SUMMARY: &[u8] = b"summary";
+pub(crate) const SUPERSEDED_BY: &[u8] = b"superseded_by";
+pub(crate) const VALID_TO: &[u8] = b"valid_to";
+pub(crate) const RID: &[u8] = b"rid";
+/// `sources`: one column per provenance entry, the value its first-seen time (µs).
+pub(crate) const SOURCES: &str = "sources";
+/// `stats` (a counter): `uses`.
+pub(crate) const STATS: &str = "stats";
+pub(crate) const USES: &[u8] = b"uses";
+/// `{ns}\0c:stats:embedded`, the namespace counter (D311 T8).
+pub(crate) const EMBEDDED: &[u8] = b"embedded";
+/// `peak` (a `max` aggregate): `importance` (parts per million) and `last_used` (µs).
+pub(crate) const PEAK: &str = "peak";
+pub(crate) const IMPORTANCE: &[u8] = b"importance";
+pub(crate) const LAST_USED: &[u8] = b"last_used";
+/// `links` and `backlinks`: one column per link, the qualifier the other end's row key, the
+/// value `(type u8, weight f32 LE)`.
+pub(crate) const LINKS: &str = "links";
+pub(crate) const BACKLINKS: &str = "backlinks";
+
+/// `G`'s columns.
+pub(crate) const LAYOUT: &[u8] = b"layout";
+pub(crate) const MODEL_ID: &[u8] = b"model_id";
+pub(crate) const MODEL_DIM: &[u8] = b"model_dim";
+pub(crate) const MODEL_ELEMENT: &[u8] = b"model_element";
+pub(crate) const TEXT_INDEX: &[u8] = b"text_index";
+/// `G`'s forget policy (T2), when the store has one.
+pub(crate) const FORGET: &[u8] = b"forget";
+/// `G`: the near-duplicate dedup setting (T3).
+pub(crate) const NEAR_DEDUP: &[u8] = b"near_dedup";
+/// `G:meta:ivf`: the store's IVF recall options (D311 T8), when it has the index.
+pub(crate) const IVF: &[u8] = b"ivf";
+
+pub(crate) const LAYOUT_VERSION: u64 = 1;
+
+/// Importance is stored in parts per million of 0–1, so the `max` aggregate is exact.
+pub(crate) const PPM: f64 = 1_000_000.0;
+
+/// A namespace: non-empty, without `\0` (D311 §5, exact and flat).
+pub(crate) fn check_namespace(ns: &str) -> Result<()> {
+    if ns.is_empty() {
+        return Err(Error::InvalidArgument("a namespace can't be empty".into()));
+    }
+    if ns.as_bytes().contains(&0) {
+        return Err(Error::InvalidArgument(format!(
+            "a namespace can't contain \\0: {ns:?}"
+        )));
+    }
+    if ns.len() > u16::MAX as usize {
+        return Err(Error::InvalidArgument(format!(
+            "a namespace is at most {} bytes, and this one is {}",
+            u16::MAX,
+            ns.len()
+        )));
+    }
+    Ok(())
+}
+
+/// A memory's row key: `{ns}\0m{rid}`.
+pub(crate) fn memory_row(ns: &str, rid: &[u8; 16]) -> Vec<u8> {
+    [ns.as_bytes(), MEMORY_TAG, rid].concat()
+}
+
+/// A namespace's counter row: `{ns}\0c` (D311 T8).
+pub(crate) fn count_row(ns: &str) -> Vec<u8> {
+    [ns.as_bytes(), COUNT_TAG].concat()
+}
+
+/// A dedup row's key: `{ns}\0h{hash}`.
+pub(crate) fn dedup_row(ns: &str, hash: &[u8; 16]) -> Vec<u8> {
+    [ns.as_bytes(), DEDUP_TAG, hash].concat()
+}
+
+/// A time-ordered `rid` (D311 §2): `(u64::MAX − created_µs)` big-endian, then 8 random
+/// bytes. A forward scan of a namespace's memory rows is newest first.
+pub(crate) fn new_rid(created_us: u64) -> Result<[u8; 16]> {
+    let mut rid = [0u8; 16];
+    rid[..8].copy_from_slice(&(u64::MAX - created_us).to_be_bytes());
+    getrandom::fill(&mut rid[8..])
+        .map_err(|e| Error::InvalidArgument(format!("no randomness for a memory id: {e}")))?;
+    Ok(rid)
+}
+
+/// The wall clock, in µs since the Unix epoch.
+pub(crate) fn now_us() -> u64 {
+    std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .map_or(0, |d| d.as_micros() as u64)
+}
+
+/// Dedup's normalisation (D311 §4, cli692's 7, frozen in ICR 0107): trim, then collapse
+/// every run of Unicode whitespace to one space. No case folding, no Unicode normalisation.
+pub fn normalize(text: &str) -> String {
+    let mut out = String::with_capacity(text.len());
+    for word in text.split_whitespace() {
+        if !out.is_empty() {
+            out.push(' ');
+        }
+        out.push_str(word);
+    }
+    out
+}
+
+/// `xxh3_128` of the normalised UTF-8, big-endian.
+pub(crate) fn content_hash(normalized: &str) -> [u8; 16] {
+    twox_hash::XxHash3_128::oneshot(normalized.as_bytes()).to_be_bytes()
+}
+
+/// A memory's id: its namespace and its `rid` (D311 §2, frozen in ICR 0107).
+///
+/// - **The byte form** (`to_bytes`/`from_bytes`): the namespace's length (`u16`, little
+///   endian), its bytes, then the 16-byte `rid`.
+/// - **The text form** (`Display`/`FromStr`): RFC 4648 base32 of the byte form, lowercase,
+///   without padding. That's what MCP tools and Python pass.
+///
+/// Both round-trip, and both refuse anything malformed (`InvalidArgument`).
+#[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub struct MemoryId {
+    ns: String,
+    rid: [u8; 16],
+}
+
+impl MemoryId {
+    pub(crate) fn new(ns: &str, rid: [u8; 16]) -> Self {
+        MemoryId {
+            ns: ns.to_owned(),
+            rid,
+        }
+    }
+
+    /// The memory's id from its row key, if the key is a memory row's.
+    pub(crate) fn from_row(key: &[u8]) -> Option<Self> {
+        let at = key.windows(2).position(|w| w == MEMORY_TAG)?;
+        let rid: [u8; 16] = key.get(at + 2..)?.try_into().ok()?;
+        let ns = std::str::from_utf8(&key[..at]).ok()?;
+        Some(MemoryId::new(ns, rid))
+    }
+
+    /// The namespace.
+    pub fn namespace(&self) -> &str {
+        &self.ns
+    }
+
+    /// When the memory was created, in µs since the Unix epoch: the `rid`'s time.
+    pub fn created_us(&self) -> u64 {
+        let t = u64::from_be_bytes(self.rid[..8].try_into().expect("8 bytes"));
+        u64::MAX - t
+    }
+
+    /// The memory's row key.
+    pub(crate) fn row(&self) -> Vec<u8> {
+        memory_row(&self.ns, &self.rid)
+    }
+
+    /// The byte form.
+    pub fn to_bytes(&self) -> Vec<u8> {
+        let mut out = Vec::with_capacity(2 + self.ns.len() + 16);
+        out.extend_from_slice(&(self.ns.len() as u16).to_le_bytes());
+        out.extend_from_slice(self.ns.as_bytes());
+        out.extend_from_slice(&self.rid);
+        out
+    }
+
+    /// The id the byte form names; anything else is `InvalidArgument`.
+    pub fn from_bytes(bytes: &[u8]) -> Result<Self> {
+        let bad = || Error::InvalidArgument("not a memory id".into());
+        let len = u16::from_le_bytes(bytes.get(..2).ok_or_else(bad)?.try_into().expect("2"));
+        let len = len as usize;
+        if bytes.len() != 2 + len + 16 {
+            return Err(bad());
+        }
+        let ns = std::str::from_utf8(&bytes[2..2 + len]).map_err(|_| bad())?;
+        check_namespace(ns).map_err(|_| bad())?;
+        let rid: [u8; 16] = bytes[2 + len..].try_into().expect("16");
+        Ok(MemoryId::new(ns, rid))
+    }
+}
+
+impl fmt::Display for MemoryId {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(&base32_encode(&self.to_bytes()))
+    }
+}
+
+impl fmt::Debug for MemoryId {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "MemoryId({:?}, {})", self.ns, self)
+    }
+}
+
+impl FromStr for MemoryId {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Self> {
+        let bytes =
+            base32_decode(s).ok_or_else(|| Error::InvalidArgument("not a memory id".into()))?;
+        MemoryId::from_bytes(&bytes)
+    }
+}
+
+const BASE32: &[u8; 32] = b"abcdefghijklmnopqrstuvwxyz234567";
+
+/// RFC 4648 base32, lowercase, without padding.
+fn base32_encode(bytes: &[u8]) -> String {
+    let mut out = String::with_capacity(bytes.len().div_ceil(5) * 8);
+    let (mut acc, mut bits) = (0u32, 0u32);
+    for &b in bytes {
+        acc = (acc << 8) | u32::from(b);
+        bits += 8;
+        while bits >= 5 {
+            bits -= 5;
+            out.push(BASE32[((acc >> bits) & 31) as usize] as char);
+        }
+    }
+    if bits > 0 {
+        out.push(BASE32[((acc << (5 - bits)) & 31) as usize] as char);
+    }
+    out
+}
+
+/// The bytes `base32_encode` wrote: lowercase, no padding, and zero trailing bits.
+fn base32_decode(s: &str) -> Option<Vec<u8>> {
+    let mut out = Vec::with_capacity(s.len() * 5 / 8);
+    let (mut acc, mut bits) = (0u32, 0u32);
+    for c in s.bytes() {
+        let v = BASE32.iter().position(|&x| x == c)? as u32;
+        acc = (acc << 5) | v;
+        bits += 5;
+        if bits >= 8 {
+            bits -= 8;
+            out.push((acc >> bits) as u8);
+        }
+    }
+    // A canonical encoding leaves fewer than 5 bits, all zero.
+    if bits >= 5 || acc & ((1 << bits) - 1) != 0 {
+        return None;
+    }
+    Some(out)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn base32_matches_rfc_4648_vectors_lowercase_unpadded() {
+        for (raw, enc) in [
+            ("", ""),
+            ("f", "my"),
+            ("fo", "mzxq"),
+            ("foo", "mzxw6"),
+            ("foob", "mzxw6yq"),
+            ("fooba", "mzxw6ytb"),
+            ("foobar", "mzxw6ytboi"),
+        ] {
+            assert_eq!(base32_encode(raw.as_bytes()), enc);
+            assert_eq!(base32_decode(enc).unwrap(), raw.as_bytes());
+        }
+        // Not canonical: uppercase, padding, a stray bit, an impossible length.
+        for bad in ["MY", "my==", "mz", "m"] {
+            assert_eq!(base32_decode(bad), None, "{bad}");
+        }
+    }
+
+    #[test]
+    fn a_memory_id_round_trips_and_refuses_the_malformed() {
+        let id = MemoryId::new("agent:alice", new_rid(1_700_000_000_000_000).unwrap());
+        assert_eq!(MemoryId::from_bytes(&id.to_bytes()).unwrap(), id);
+        assert_eq!(id.to_string().parse::<MemoryId>().unwrap(), id);
+        assert_eq!(id.created_us(), 1_700_000_000_000_000);
+        assert_eq!(MemoryId::from_row(&id.row()).unwrap(), id);
+        let b = id.to_bytes();
+        for bad in [&b[..b.len() - 1], &b[..1], &[0, 0][..]] {
+            assert!(MemoryId::from_bytes(bad).is_err());
+        }
+        assert!("not!base32".parse::<MemoryId>().is_err());
+        // An empty namespace isn't an id.
+        let mut empty = vec![0, 0];
+        empty.extend_from_slice(&[7; 16]);
+        assert!(MemoryId::from_bytes(&empty).is_err());
+    }
+
+    #[test]
+    fn newer_rids_sort_first() {
+        let old = new_rid(1_000).unwrap();
+        let new = new_rid(2_000).unwrap();
+        assert!(new < old);
+    }
+
+    #[test]
+    fn normalisation_trims_and_collapses_whitespace_only() {
+        assert_eq!(normalize("  a \t b\n\nc  "), "a b c");
+        assert_eq!(normalize("Café\u{00a0}au  lait"), "Café au lait");
+        assert_ne!(normalize("A"), normalize("a"));
+        assert_eq!(normalize(" \n "), "");
+        assert_eq!(content_hash("a b"), content_hash(&normalize(" a  b ")));
+    }
+
+    #[test]
+    fn namespaces_refuse_empty_and_nul() {
+        assert!(check_namespace("").is_err());
+        assert!(check_namespace("a\0b").is_err());
+        assert!(check_namespace("agent:alice/x").is_ok());
+    }
+}