pigeonhole-db 1.1.0

data/vendor/text/src/sloppy.rs

@@ -0,0 +1,460 @@
+//! Sloppy phrases (D309 rev 6 §B, #1290): a phrase's terms within `slop` position moves of
+//! their places, with Lucene's sloppy-phrase semantics and frequency.
+//!
+//! Each phrase term `i` at offset `oᵢ` walks its positions as `pᵢ − oᵢ` (its "phrase
+//! position"). A match is one position per term whose phrase positions span at most `slop`
+//! (`max − min`, the match's distance). The walk keeps the terms in a queue ordered by
+//! phrase position, then offset, then term index. It repeatedly takes the leading term and
+//! advances it, shrinking the current match's distance while the lead stays at or before
+//! the next term. When the lead passes the next term, the match it was minimising is done:
+//! it counts `1 / (distance + 1)` if its distance is within `slop`, and the walk continues
+//! from the new lead. An exact match has distance 0 and counts 1.
+//!
+//! A word repeated in the phrase (`"the cat the"`) is a repeat group. Its terms share one
+//! position list but never stand on one position together: the `j`-th repeat starts at its
+//! word's `j`-th position, and when two collide the lesser (by phrase position, then offset)
+//! moves on. A phrase whose repeats can't all be placed doesn't match.
+//!
+//! This follows the algorithm as Lucene's `SloppyPhraseMatcher` documents it, written here
+//! from that description. The tests check it against a brute force of the match rule and
+//! against the exact evaluator at slop 0.
+
+/// One phrase term's walk over its positions.
+struct Walk<'a> {
+    positions: &'a [u32],
+    /// The next position to take.
+    next: usize,
+    offset: i64,
+    /// The current phrase position: the position taken, less the offset.
+    at: i64,
+    /// The repeat group: terms of the same word.
+    group: usize,
+}
+
+impl Walk<'_> {
+    /// Moves to the next position; `false` if there's none.
+    fn advance(&mut self) -> bool {
+        match self.positions.get(self.next) {
+            Some(&p) => {
+                self.at = i64::from(p) - self.offset;
+                self.next += 1;
+                true
+            }
+            None => false,
+        }
+    }
+
+    /// The position in the document (not less the offset).
+    fn position(&self) -> i64 {
+        self.at + self.offset
+    }
+}
+
+/// What the walk tells its caller about each match it counts: nothing when scoring (a
+/// zero-sized [`Count`], so the scoring walk does no extra work), the matched positions when
+/// highlighting ([`Positions`]).
+pub(crate) trait Record {
+    /// Whether the walk keeps the positions of the match it's minimising.
+    const ON: bool;
+    /// A counted match: one document position per phrase term, in the phrase's order.
+    fn matched(&mut self, positions: &[u32]);
+}
+
+/// Counts only.
+pub(crate) struct Count;
+
+impl Record for Count {
+    const ON: bool = false;
+    fn matched(&mut self, _: &[u32]) {}
+}
+
+/// Every counted match's positions, in the order the walk counts them.
+#[derive(Default)]
+pub(crate) struct Positions(pub(crate) Vec<Vec<u32>>);
+
+impl Record for Positions {
+    const ON: bool = true;
+    fn matched(&mut self, positions: &[u32]) {
+        self.0.push(positions.to_vec());
+    }
+}
+
+/// The sloppy frequency of a phrase in one document.
+///
+/// `terms` holds each phrase term's sorted positions in the document, its offset in the
+/// phrase, and its word's index (equal for repeats of one word). Every term must have at
+/// least one position.
+pub(crate) fn sloppy_freq(terms: &[(&[u32], u32, usize)], slop: u16) -> f64 {
+    sloppy(terms, slop, &mut Count)
+}
+
+/// [`sloppy_freq`], telling `rec` about each match it counts: the positions where the
+/// match's distance was smallest.
+pub(crate) fn sloppy<R: Record>(terms: &[(&[u32], u32, usize)], slop: u16, rec: &mut R) -> f64 {
+    let slop = i64::from(slop);
+    let mut w: Vec<Walk<'_>> = terms
+        .iter()
+        .map(|&(positions, offset, group)| Walk {
+            positions,
+            next: 0,
+            offset: i64::from(offset),
+            at: 0,
+            group,
+        })
+        .collect();
+    let repeats = (0..w.len()).any(|i| (0..i).any(|j| w[j].group == w[i].group));
+    for walk in &mut w {
+        if !walk.advance() {
+            return 0.0;
+        }
+    }
+    if repeats {
+        // The j-th repeat of a word starts at the word's j-th position.
+        for i in 0..w.len() {
+            let earlier = (0..i).filter(|&j| w[j].group == w[i].group).count();
+            for _ in 0..earlier {
+                if !w[i].advance() {
+                    return 0.0;
+                }
+            }
+        }
+    }
+    let mut end = w.iter().map(|x| x.at).max().expect("a phrase has terms");
+    // The queue: every walk but the lead, which is held out of it.
+    let mut queued = vec![true; w.len()];
+    let pop = |w: &[Walk<'_>], queued: &mut [bool]| -> usize {
+        let i = (0..w.len())
+            .filter(|&i| queued[i])
+            .min_by_key(|&i| (w[i].at, w[i].offset, i))
+            .expect("a queued walk");
+        queued[i] = false;
+        i
+    };
+    let top = |w: &[Walk<'_>], queued: &[bool]| -> i64 {
+        (0..w.len())
+            .filter(|&i| queued[i])
+            .map(|i| (w[i].at, w[i].offset, i))
+            .min()
+            .expect("a queued walk")
+            .0
+    };
+    let advance = |w: &mut [Walk<'_>], i: usize, end: &mut i64| -> bool {
+        if !w[i].advance() {
+            return false;
+        }
+        *end = (*end).max(w[i].at);
+        true
+    };
+    let weight = |distance: i64| 1.0 / (distance as f64 + 1.0);
+    // The positions where the current match's distance was smallest (only with `R::ON`).
+    let mut best: Vec<u32> = Vec::new();
+    let snap = |w: &[Walk<'_>], best: &mut Vec<u32>| {
+        if R::ON {
+            best.clear();
+            best.extend(w.iter().map(|x| x.position() as u32));
+        }
+    };
+    let mut freq = 0.0;
+    let mut lead = pop(&w, &mut queued);
+    let mut distance = end - w[lead].at;
+    snap(&w, &mut best);
+    let mut next = top(&w, &queued);
+    loop {
+        if !advance(&mut w, lead, &mut end) {
+            break;
+        }
+        if repeats && !resolve_collisions(&mut w, lead, &mut end) {
+            break;
+        }
+        if w[lead].at > next {
+            // The match the old lead was minimising is done.
+            if distance <= slop {
+                freq += weight(distance);
+                rec.matched(&best);
+            }
+            queued[lead] = true;
+            lead = pop(&w, &mut queued);
+            next = top(&w, &queued);
+            distance = end - w[lead].at;
+            snap(&w, &mut best);
+        } else if end - w[lead].at < distance {
+            distance = end - w[lead].at;
+            snap(&w, &mut best);
+        }
+    }
+    if distance <= slop {
+        freq += weight(distance);
+        rec.matched(&best);
+    }
+    freq
+}
+
+/// After walk `moved` advanced: while it shares a document position with another walk of its
+/// word, advances the lesser of the two (by phrase position, then offset). `false` when a
+/// walk runs out.
+fn resolve_collisions(w: &mut [Walk<'_>], moved: usize, end: &mut i64) -> bool {
+    let mut cur = moved;
+    while let Some(other) = (0..w.len())
+        .find(|&j| j != cur && w[j].group == w[cur].group && w[j].position() == w[cur].position())
+    {
+        let lesser = if (w[cur].at, w[cur].offset) < (w[other].at, w[other].offset) {
+            cur
+        } else {
+            other
+        };
+        if !w[lesser].advance() {
+            return false;
+        }
+        *end = (*end).max(w[lesser].at);
+        cur = lesser;
+    }
+    true
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn freq(phrase: &[&str], doc: &str, slop: u16) -> f64 {
+        let words: Vec<&str> = doc.split(' ').collect();
+        let mut groups: Vec<&str> = Vec::new();
+        let lists: Vec<Vec<u32>> = phrase
+            .iter()
+            .map(|t| {
+                (0..words.len() as u32)
+                    .filter(|&p| words[p as usize] == *t)
+                    .collect()
+            })
+            .collect();
+        if lists.iter().any(Vec::is_empty) {
+            return 0.0;
+        }
+        let terms: Vec<(&[u32], u32, usize)> = phrase
+            .iter()
+            .zip(&lists)
+            .zip(0u32..)
+            .map(|((t, l), off)| {
+                let g = groups.iter().position(|x| x == t).unwrap_or_else(|| {
+                    groups.push(t);
+                    groups.len() - 1
+                });
+                (l.as_slice(), off, g)
+            })
+            .collect();
+        sloppy_freq(&terms, slop)
+    }
+
+    /// Whether some placement of the phrase's terms (distinct positions for a repeated word)
+    /// spans at most `slop`: the match rule, by brute force.
+    fn matches(phrase: &[&str], doc: &str, slop: u16) -> bool {
+        let words: Vec<&str> = doc.split(' ').collect();
+        fn go(
+            phrase: &[&str],
+            words: &[&str],
+            i: usize,
+            picked: &mut Vec<usize>,
+            slop: i64,
+        ) -> bool {
+            if i == phrase.len() {
+                let rel: Vec<i64> = picked
+                    .iter()
+                    .zip(0i64..)
+                    .map(|(&p, o)| p as i64 - o)
+                    .collect();
+                return rel.iter().max().unwrap() - rel.iter().min().unwrap() <= slop;
+            }
+            for p in 0..words.len() {
+                if words[p] == phrase[i] && !picked.contains(&p) {
+                    picked.push(p);
+                    if go(phrase, words, i + 1, picked, slop) {
+                        return true;
+                    }
+                    picked.pop();
+                }
+            }
+            false
+        }
+        go(phrase, &words, 0, &mut Vec::new(), i64::from(slop))
+    }
+
+    #[test]
+    fn worked_examples() {
+        // Exact.
+        assert_eq!(freq(&["a", "b"], "a b", 0), 1.0);
+        assert_eq!(freq(&["a", "b"], "a b a b", 0), 2.0);
+        // One word between: distance 1.
+        assert_eq!(freq(&["a", "b"], "a x b", 0), 0.0);
+        assert_eq!(freq(&["a", "b"], "a x b", 1), 0.5);
+        // Reversed: distance 2.
+        assert_eq!(freq(&["a", "b"], "b a", 1), 0.0);
+        assert_eq!(freq(&["a", "b"], "b a", 2), 1.0 / 3.0);
+        // simp724's case on #1247: one match at distance 0 (the lead minimises it to 0
+        // before the next term), so 1.0; MaxScore's bound (Σ maxtf) covers it either way.
+        assert_eq!(freq(&["a", "b"], "a a a b", 2), 1.0);
+        // Repeats never share a position.
+        assert_eq!(freq(&["a", "a"], "a", 5), 0.0);
+        assert_eq!(freq(&["a", "a"], "a a", 0), 1.0);
+        assert!(freq(&["the", "cat", "the"], "the cat the", 0) > 0.0);
+        assert_eq!(freq(&["the", "cat", "the"], "the cat", 3), 0.0);
+    }
+
+    /// Repeated words, traced by hand through the walk (simp724 on #1300).
+    #[test]
+    fn repeated_words_by_hand() {
+        // "a a" on "a x a": the second a starts at its word's second position (2), one match
+        // at distance 1; the lead's next step collides at 2 and the lesser runs out.
+        assert_eq!(freq(&["a", "a"], "a x a", 1), 0.5);
+        assert_eq!(freq(&["a", "a"], "a x a", 0), 0.0);
+        // "a a" on "a a a": two exact matches (0-1 and 1-2), as the exact count gives.
+        assert_eq!(freq(&["a", "a"], "a a a", 0), 2.0);
+        // "the cat the" on "the cat the": one exact match; more slop finds no other.
+        assert_eq!(freq(&["the", "cat", "the"], "the cat the", 0), 1.0);
+        assert_eq!(freq(&["the", "cat", "the"], "the cat the", 2), 1.0);
+        // "a b a" on "a b x a": the second a sits one place late, distance 1.
+        assert_eq!(freq(&["a", "b", "a"], "a b x a", 1), 0.5);
+        assert_eq!(freq(&["a", "b", "a"], "a b x a", 0), 0.0);
+        // simp724's "a b a"~2 on "a b a b a": the lead a advances to 2 and collides with the
+        // second a, which moves to 4; that closes a match at distance 0. The new lead b moves
+        // to 3 and closes another at 0. That's the two exact occurrences (0-2, 2-4), sharing
+        // position 2.
+        assert_eq!(freq(&["a", "b", "a"], "a b a b a", 2), 2.0);
+        assert_eq!(freq(&["a", "b", "a"], "a b a b a", 0), 2.0);
+    }
+
+    /// xorshift: deterministic per seed.
+    fn rng(seed: &mut u64) -> u64 {
+        *seed ^= *seed << 13;
+        *seed ^= *seed >> 7;
+        *seed ^= *seed << 17;
+        *seed
+    }
+
+    #[test]
+    fn a_phrase_without_repeats_matches_exactly_when_the_rule_says() {
+        let mut seed = 0x5_1090_u64;
+        let vocab = ["a", "b", "c", "d"];
+        for case in 0..3000 {
+            let n = 2 + (rng(&mut seed) % 2) as usize;
+            let mut phrase: Vec<&str> = vocab.to_vec();
+            for i in (1..phrase.len()).rev() {
+                phrase.swap(i, (rng(&mut seed) % (i as u64 + 1)) as usize);
+            }
+            phrase.truncate(n);
+            let len = 1 + (rng(&mut seed) % 9) as usize;
+            let doc: Vec<&str> = (0..len)
+                .map(|_| vocab[(rng(&mut seed) % 4) as usize])
+                .collect();
+            let doc = doc.join(" ");
+            let slop = (rng(&mut seed) % 5) as u16;
+            let f = freq(&phrase, &doc, slop);
+            assert_eq!(
+                f > 0.0,
+                matches(&phrase, &doc, slop),
+                "case {case}: {phrase:?} ~{slop} in {doc:?} gave {f}"
+            );
+            // At slop 0, every match is exact and counts 1: the exact phrase's count.
+            let words: Vec<&str> = doc.split(' ').collect();
+            let exact = (0..words.len())
+                .filter(|&i| {
+                    i + n <= words.len() && phrase.iter().zip(&words[i..]).all(|(a, b)| a == b)
+                })
+                .count();
+            assert_eq!(
+                freq(&phrase, &doc, 0),
+                exact as f64,
+                "case {case}: {phrase:?} in {doc:?} at slop 0"
+            );
+        }
+    }
+
+    #[test]
+    fn a_phrase_with_repeats_only_matches_where_the_rule_allows() {
+        let mut seed = 0x5_1091_u64;
+        let vocab = ["a", "b", "c"];
+        for case in 0..3000 {
+            let n = 2 + (rng(&mut seed) % 2) as usize;
+            let phrase: Vec<&str> = (0..n)
+                .map(|_| vocab[(rng(&mut seed) % 3) as usize])
+                .collect();
+            let len = 1 + (rng(&mut seed) % 8) as usize;
+            let doc: Vec<&str> = (0..len)
+                .map(|_| vocab[(rng(&mut seed) % 3) as usize])
+                .collect();
+            let doc = doc.join(" ");
+            let slop = (rng(&mut seed) % 4) as u16;
+            let f = freq(&phrase, &doc, slop);
+            if f > 0.0 {
+                assert!(
+                    matches(&phrase, &doc, slop),
+                    "case {case}: {phrase:?} ~{slop} in {doc:?} gave {f} without a placement"
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn each_counted_match_is_a_placement_and_their_weights_are_the_frequency() {
+        let mut seed = 0x5_1092_u64;
+        let vocab = ["a", "b", "c"];
+        for case in 0..3000 {
+            let n = 2 + (rng(&mut seed) % 2) as usize;
+            let phrase: Vec<&str> = (0..n)
+                .map(|_| vocab[(rng(&mut seed) % 3) as usize])
+                .collect();
+            let len = 1 + (rng(&mut seed) % 9) as usize;
+            let words: Vec<&str> = (0..len)
+                .map(|_| vocab[(rng(&mut seed) % 3) as usize])
+                .collect();
+            let slop = (rng(&mut seed) % 4) as u16;
+            let lists: Vec<Vec<u32>> = phrase
+                .iter()
+                .map(|t| {
+                    (0..len as u32)
+                        .filter(|&p| words[p as usize] == *t)
+                        .collect()
+                })
+                .collect();
+            if lists.iter().any(Vec::is_empty) {
+                continue;
+            }
+            let terms: Vec<(&[u32], u32, usize)> = phrase
+                .iter()
+                .zip(&lists)
+                .zip(0u32..)
+                .map(|((t, l), off)| {
+                    let g = phrase.iter().position(|x| x == t).expect("itself");
+                    (l.as_slice(), off, g)
+                })
+                .collect();
+            let mut rec = Positions::default();
+            let f = sloppy(&terms, slop, &mut rec);
+            assert_eq!(f, sloppy_freq(&terms, slop), "case {case}");
+            let ctx = format!("case {case}: {phrase:?} ~{slop} in {words:?}");
+            let mut sum = 0.0;
+            for ps in &rec.0 {
+                assert_eq!(ps.len(), n, "{ctx}");
+                for (i, &p) in ps.iter().enumerate() {
+                    assert_eq!(words[p as usize], phrase[i], "{ctx}: {ps:?}");
+                    for j in 0..i {
+                        assert!(
+                            phrase[j] != phrase[i] || ps[j] != p,
+                            "{ctx}: {ps:?} reuses {p}"
+                        );
+                    }
+                }
+                let at: Vec<i64> = ps
+                    .iter()
+                    .zip(0i64..)
+                    .map(|(&p, o)| i64::from(p) - o)
+                    .collect();
+                let d = at.iter().max().unwrap() - at.iter().min().unwrap();
+                assert!(d <= i64::from(slop), "{ctx}: {ps:?} spans {d}");
+                sum += 1.0 / (d as f64 + 1.0);
+            }
+            assert!(
+                (sum - f).abs() < 1e-12,
+                "{ctx}: weights {sum} vs frequency {f}"
+            );
+        }
+    }
+}

data/vendor/text/src/tokenizer.rs

@@ -0,0 +1,163 @@
+//! Tokenisers (D309 §5, §6b): the [`Tokenizer`] trait and the built-in one.
+
+use std::ops::Range;
+
+use unicode_segmentation::UnicodeSegmentation;
+
+/// One token of a text, as a [`Tokenizer`] emits it.
+///
+/// Built with [`Token::new`], so it can gain fields later.
+#[derive(Debug, Clone, PartialEq, Eq)]
+#[non_exhaustive]
+pub struct Token<'a> {
+    /// The term as indexed, which may differ from the source text (lowercased, stemmed).
+    pub text: &'a str,
+    /// The token's position. Positions never decrease. A gap is a dropped token (a stop
+    /// word), and two tokens may share a position (synonyms).
+    pub position: u32,
+    /// The token's byte range in the **source text**, for highlighting. Two tokens at one
+    /// position may share a range.
+    pub range: Range<usize>,
+}
+
+impl<'a> Token<'a> {
+    /// A token of `text` at `position`, from `range` of the source text.
+    pub fn new(text: &'a str, position: u32, range: Range<usize>) -> Self {
+        Token {
+            text,
+            position,
+            range,
+        }
+    }
+}
+
+/// Splits text into terms, for documents and queries alike.
+///
+/// An index stores its tokeniser's [`name`](Tokenizer::name) and
+/// [`fingerprint`](Tokenizer::fingerprint), and refuses to open with another. A tokeniser
+/// must be deterministic: the same text always gives the same tokens, or the index's
+/// [`verify`](crate::Index::verify) reports mismatches.
+pub trait Tokenizer: Send + Sync {
+    /// A stable name for the tokeniser.
+    fn name(&self) -> &str;
+    /// A stable hash of everything that changes its output (its options and version).
+    fn fingerprint(&self) -> u64;
+    /// Calls `out` with each token of `text`, in position order. A dropped word (a stop
+    /// word, say) is never emitted, but advances the next token's position.
+    fn tokens(&self, text: &str, out: &mut dyn FnMut(Token<'_>));
+}
+
+/// Lucene's English stop words (its `EnglishAnalyzer` default set).
+const STOP_WORDS: [&str; 33] = [
+    "a", "an", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into", "is", "it",
+    "no", "not", "of", "on", "or", "such", "that", "the", "their", "then", "there", "these",
+    "they", "this", "to", "was", "will", "with",
+];
+
+/// The built-in tokeniser: Unicode word boundaries (UAX #29), then lowercasing.
+/// Optionally it drops English stop words. A dropped word keeps its position.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) struct Standard {
+    pub(crate) stop_words: bool,
+}
+
+impl Standard {
+    pub(crate) const NAME: &'static str = "standard";
+}
+
+impl Tokenizer for Standard {
+    fn name(&self) -> &str {
+        Self::NAME
+    }
+
+    /// Covers the options and the word-boundary rules' Unicode version, since UAX #29
+    /// changes between Unicode versions (cli692 on D309 rev 6): a dependency bump that
+    /// moves it makes `open` refuse with `Mismatch`, and the index needs a `repair`.
+    fn fingerprint(&self) -> u64 {
+        let (major, minor, update) = unicode_segmentation::UNICODE_VERSION;
+        let stop = if self.stop_words { " stop-en33" } else { "" };
+        fnv1a(format!("standard/1 uax29 {major}.{minor}.{update} lowercase{stop}").as_bytes())
+    }
+
+    fn tokens(&self, text: &str, out: &mut dyn FnMut(Token<'_>)) {
+        let mut lower = String::new();
+        for (position, (at, word)) in text.unicode_word_indices().enumerate() {
+            lower.clear();
+            lower.extend(word.chars().flat_map(char::to_lowercase));
+            if self.stop_words && STOP_WORDS.binary_search(&lower.as_str()).is_ok() {
+                continue;
+            }
+            let position = u32::try_from(position).unwrap_or(u32::MAX);
+            out(Token::new(&lower, position, at..at + word.len()));
+        }
+    }
+}
+
+/// FNV-1a, 64-bit: a stable hash for fingerprints (never `std`'s, which may change).
+pub(crate) fn fnv1a(bytes: &[u8]) -> u64 {
+    let mut h = 0xcbf2_9ce4_8422_2325u64;
+    for &b in bytes {
+        h ^= u64::from(b);
+        h = h.wrapping_mul(0x0000_0100_0000_01b3);
+    }
+    h
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn tokens(t: &dyn Tokenizer, text: &str) -> Vec<(String, u32, Range<usize>)> {
+        let mut v = Vec::new();
+        t.tokens(text, &mut |tok| {
+            v.push((tok.text.to_owned(), tok.position, tok.range));
+        });
+        v
+    }
+
+    #[test]
+    fn the_stop_words_are_sorted_for_the_binary_search() {
+        assert!(STOP_WORDS.windows(2).all(|w| w[0] < w[1]));
+    }
+
+    #[test]
+    fn words_are_lowercased_with_their_source_ranges() {
+        let got = tokens(&Standard { stop_words: false }, "Chose Postgres, in MARCH.");
+        assert_eq!(
+            got,
+            [
+                ("chose".into(), 0, 0..5),
+                ("postgres".into(), 1, 6..14),
+                ("in".into(), 2, 16..18),
+                ("march".into(), 3, 19..24),
+            ]
+        );
+    }
+
+    #[test]
+    fn a_stop_word_is_dropped_but_keeps_its_position() {
+        let got = tokens(&Standard { stop_words: true }, "state of the art");
+        assert_eq!(got, [("state".into(), 0, 0..5), ("art".into(), 3, 13..16)]);
+    }
+
+    #[test]
+    fn multi_byte_words_keep_byte_ranges() {
+        let got = tokens(&Standard { stop_words: false }, "Ève à Zürich");
+        assert_eq!(
+            got,
+            [
+                ("ève".into(), 0, 0..4),
+                ("à".into(), 1, 5..7),
+                ("zürich".into(), 2, 8..15),
+            ]
+        );
+    }
+
+    #[test]
+    fn the_fingerprint_follows_the_options() {
+        let a = Standard { stop_words: false }.fingerprint();
+        let b = Standard { stop_words: true }.fingerprint();
+        assert_ne!(a, b);
+        assert_eq!(a, Standard { stop_words: false }.fingerprint());
+    }
+}

data/vendor/vector/Cargo.toml

@@ -0,0 +1,28 @@
+# Generated by `rake gem:source` (crates/ruby/rakelib/source_gem.rake): a vendored copy of crates/vector
+
+[dependencies.futures-core]
+default-features = false
+optional = true
+version = "0.3"
+[dependencies.pigeonhole]
+path = "../pigeonhole"
+version = "1.1.0"
+[features]
+__unstable-hooks = []
+async = ["pigeonhole/async", "dep:futures-core"]
+default = ["async"]
+[lints.rust.unexpected_cfgs]
+check-cfg = ["cfg(loom)", "cfg(engine_loom)", "cfg(io_loom)", "cfg(pigeonhole_sim)"]
+level = "warn"
+[package]
+authors = ["Matt Tanous <matt.tanous@hey.com>"]
+categories = ["database-implementations", "data-structures"]
+description = "An IVF vector index for Pigeonhole, built on its public API."
+edition = "2024"
+keywords = ["database", "wide-column", "embedded", "lsm", "bigtable"]
+license = "MIT"
+name = "pigeonhole-vector"
+readme = "README.md"
+repository = "https://github.com/CodingAnarchy/pigeonhole"
+rust-version = "1.96"
+version = "1.1.0"

data/vendor/vector/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Matt Tanous
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/vendor/vector/README.md

@@ -0,0 +1,9 @@
+# pigeonhole-vector
+
+An IVF (inverted-file) vector index for [Pigeonhole](https://github.com/CodingAnarchy/pigeonhole), built on its public API only (decision D306, #490 D).
+
+- **Layout:** each list is one wide row of a Pigeonhole vector family, with one column per vector, and a query is one multi-get with a top-k nearest search over the `nprobe` lists nearest the query.
+- **Rebalancing:** lists split and merge incrementally, inside the inserts and deletes that cause them. A `Rebalancer` the application drives can take that work instead. Every move is a transaction.
+- **Target:** 95% recall@10 on SIFT1M at 10 ms or less, single-threaded.
+
+**Status:** the index, bulk loading, inline or background rebalancing, and async search (D306). On SIFT1M it reaches 95.8% recall@10 at 3.4 ms per query, single-threaded. See the [vectors guide](https://github.com/CodingAnarchy/pigeonhole/blob/main/docs/guide/vectors.md). It follows the public 1.x version line, with semver from its first release.