pigeonhole-db 1.1.0

data/vendor/text/src/error.rs

@@ -0,0 +1,108 @@
+//! The crate's error.
+
+use std::fmt;
+
+use pigeonhole::ErrorCode;
+
+/// An index operation's error.
+#[derive(Debug)]
+#[non_exhaustive]
+pub enum Error {
+    /// The database refused or failed the operation.
+    Db(pigeonhole::Error),
+    /// The arguments don't fit the index: an option out of range, a document with more
+    /// distinct terms than [`IndexOptions::max_terms`](crate::IndexOptions::max_terms), a
+    /// valid query using a feature the index lacks (a phrase without positions), or a table
+    /// that isn't an index.
+    InvalidArgument(String),
+    /// A query string didn't parse ([`Query::parse`](crate::Query::parse)). `at` is a byte
+    /// offset into the string.
+    #[non_exhaustive]
+    InvalidQuery {
+        /// Where in the query string, in bytes.
+        at: usize,
+        /// What was wrong there.
+        reason: QueryErrorKind,
+    },
+    /// The index is being repaired ([`Index::repair`](crate::Index::repair)): writes through
+    /// it and searches of it fail with this while the repair runs. Retry after it finishes.
+    Busy,
+    /// The index was created with another tokeniser (its name or fingerprint differs).
+    Mismatch(String),
+    /// The index's rows don't parse.
+    Corruption(String),
+}
+
+/// Why a query string didn't parse.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+#[non_exhaustive]
+pub enum QueryErrorKind {
+    /// A `"` with no closing `"`.
+    UnclosedQuote,
+    /// A phrase's slop (`"..."~N`) is missing, above 65,535, or has something glued to it.
+    Slop,
+    /// A `~` inside a phrase: a phrase's words can't be fuzzy.
+    FuzzyInPhrase,
+    /// A fuzzy distance other than `~`, `~1` or `~2`.
+    Distance,
+}
+
+impl fmt::Display for QueryErrorKind {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(match self {
+            QueryErrorKind::UnclosedQuote => "a quote is never closed",
+            QueryErrorKind::Slop => "a phrase's slop is `~N`, with N from 0 to 65,535",
+            QueryErrorKind::FuzzyInPhrase => "a phrase's words can't be fuzzy",
+            QueryErrorKind::Distance => "a fuzzy distance is `~`, `~1` or `~2`",
+        })
+    }
+}
+
+impl Error {
+    /// The closest Pigeonhole error code, for callers that branch on codes.
+    pub fn code(&self) -> ErrorCode {
+        match self {
+            Error::Db(e) => e.code(),
+            Error::InvalidArgument(_) | Error::InvalidQuery { .. } | Error::Mismatch(_) => {
+                ErrorCode::InvalidArgument
+            }
+            Error::Busy => ErrorCode::Busy,
+            Error::Corruption(_) => ErrorCode::Corruption,
+        }
+    }
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Error::Db(e) => e.fmt(f),
+            Error::InvalidArgument(m) => write!(f, "invalid argument: {m}"),
+            Error::InvalidQuery { at, reason } => write!(f, "invalid query at byte {at}: {reason}"),
+            Error::Busy => f.write_str("the index is being repaired"),
+            Error::Mismatch(m) => write!(f, "tokeniser mismatch: {m}"),
+            Error::Corruption(m) => write!(f, "index corruption: {m}"),
+        }
+    }
+}
+
+impl std::error::Error for Error {
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        match self {
+            Error::Db(e) => Some(e),
+            _ => None,
+        }
+    }
+}
+
+impl From<pigeonhole::Error> for Error {
+    fn from(e: pigeonhole::Error) -> Self {
+        Error::Db(e)
+    }
+}
+
+/// An index operation's result.
+pub type Result<T> = std::result::Result<T, Error>;
+
+pub(crate) fn corrupt(what: &str) -> Error {
+    Error::Corruption(what.to_owned())
+}

data/vendor/text/src/fuzzy.rs

@@ -0,0 +1,265 @@
+//! Fuzzy terms (D309 §4.4): a term's expansions, the index's terms within a few edits of it.
+//!
+//! The expansion walks a Levenshtein automaton (Schulz–Mihov, from `levenshtein_automata`,
+//! with an adjacent transposition as one edit) against the sorted `P{term}` rows, reading
+//! only each row's `df` counter. At a row whose key the automaton rejects partway, the walk
+//! seeks to the smallest key above it that could still match, so the cost is the matching
+//! terms plus one seek per dead branch, not the dictionary's size.
+
+use std::ops::Bound;
+use std::sync::OnceLock;
+
+use levenshtein_automata::{DFA, Distance, LevenshteinAutomatonBuilder, SINK_STATE};
+use pigeonhole::Snapshot;
+
+use crate::error::{Result, corrupt};
+use crate::index::Rows;
+use crate::layout::*;
+use crate::query::Fuzziness;
+
+/// One term a fuzzy term matched.
+#[derive(Debug, Clone, PartialEq)]
+pub(crate) struct Expansion {
+    pub(crate) term: String,
+    /// Its edit distance from the fuzzy term.
+    pub(crate) distance: u8,
+    pub(crate) df: i64,
+    /// Lucene's FuzzyQuery boost: `1 − distance / min(len(q), len(t))`, in characters.
+    pub(crate) boost: f64,
+}
+
+/// The edits a fuzzy term allows: [`Fuzziness::Auto`] by its length in characters.
+pub(crate) fn edits(term: &str, fuzziness: Fuzziness) -> u8 {
+    match fuzziness {
+        Fuzziness::One => 1,
+        Fuzziness::Two => 2,
+        Fuzziness::Auto => match term.chars().count() {
+            0..=2 => 0,
+            3..=5 => 1,
+            _ => 2,
+        },
+    }
+}
+
+/// The boost of a match `distance` edits away, or `None` when the edits are as many as the
+/// shorter term's characters: such a "match" replaces a whole term (`a` and `b` are one
+/// edit apart), and its boost would be 0 or below.
+pub(crate) fn boost(q_chars: usize, t_chars: usize, distance: u8) -> Option<f64> {
+    if distance == 0 {
+        return Some(1.0);
+    }
+    let shorter = q_chars.min(t_chars);
+    (usize::from(distance) < shorter).then(|| 1.0 - f64::from(distance) / shorter as f64)
+}
+
+/// The automaton builders, for 1 and 2 edits: a few milliseconds each to build, so built once.
+fn builder(edits: u8) -> &'static LevenshteinAutomatonBuilder {
+    static ONE: OnceLock<LevenshteinAutomatonBuilder> = OnceLock::new();
+    static TWO: OnceLock<LevenshteinAutomatonBuilder> = OnceLock::new();
+    let cell = if edits == 1 { &ONE } else { &TWO };
+    cell.get_or_init(|| LevenshteinAutomatonBuilder::new(edits, true))
+}
+
+/// `term`'s expansions within `edits` (at most 2), whose first `prefix_chars` characters
+/// equal `term`'s: the closest first, then the highest `df`, then the term (bytewise), at
+/// most `max` of them. The flag says whether `max` cut any.
+pub(crate) fn expand(
+    t: &Rows,
+    s: &Snapshot,
+    term: &str,
+    edits: u8,
+    prefix_chars: usize,
+    max: usize,
+) -> Result<(Vec<Expansion>, bool)> {
+    let q_chars = term.chars().count();
+    let mut found = Vec::new();
+    if edits == 0 {
+        let df = df_at(t, s, &post_row(term))?;
+        if df > 0 {
+            found.push(Expansion {
+                term: term.to_owned(),
+                distance: 0,
+                df,
+                boost: 1.0,
+            });
+        }
+    } else {
+        let split = term
+            .char_indices()
+            .nth(prefix_chars)
+            .map_or(term.len(), |(i, _)| i);
+        let (head, tail) = term.split_at(split);
+        let dfa = builder(edits).build_dfa(tail);
+        walk(t, s, &post_row(head), &dfa, |key, distance, df| {
+            let t_term = std::str::from_utf8(key).map_err(|_| corrupt("a term isn't UTF-8"))?;
+            if let Some(boost) = boost(q_chars, t_term.chars().count(), distance) {
+                found.push(Expansion {
+                    term: t_term.to_owned(),
+                    distance,
+                    df,
+                    boost,
+                });
+            }
+            Ok(())
+        })?;
+    }
+    found.sort_by(|a, b| {
+        a.distance
+            .cmp(&b.distance)
+            .then(b.df.cmp(&a.df))
+            .then_with(|| a.term.as_bytes().cmp(b.term.as_bytes()))
+    });
+    let cut = found.len() > max;
+    found.truncate(max);
+    Ok((found, cut))
+}
+
+/// Walks the `P` rows under `base` (the row prefix plus the fixed head) whose rest `dfa`
+/// accepts, calling `hit` with each one's term, distance and `df` (above 0).
+fn walk(
+    t: &Rows,
+    s: &Snapshot,
+    base: &[u8],
+    dfa: &DFA,
+    mut hit: impl FnMut(&[u8], u8, i64) -> Result<()>,
+) -> Result<()> {
+    let end = prefix_end(base);
+    let mut from = base.to_vec();
+    let mut states = Vec::new();
+    'seek: loop {
+        let end_bound = end.as_deref().map_or(Bound::Unbounded, Bound::Excluded);
+        let mut rows = t
+            .scan_bounds(Bound::Included(&from), end_bound)
+            .family(DF)
+            .snapshot(s)
+            .iter()?;
+        while let Some(row) = rows.next_ref()? {
+            let key = row.key();
+            let rest = &key[base.len()..];
+            // `states[i]`: the state before `rest[i]`.
+            states.clear();
+            let mut state = dfa.initial_state();
+            let mut dead = None;
+            for (i, &b) in rest.iter().enumerate() {
+                states.push(state);
+                state = dfa.transition(state, b);
+                if state == SINK_STATE {
+                    dead = Some(i);
+                    break;
+                }
+            }
+            match dead {
+                None => {
+                    if let Distance::Exact(d) = dfa.distance(state) {
+                        let df = row.get(DF, N).and_then(|c| c.as_i64()).unwrap_or(0);
+                        if df > 0 {
+                            let term = &key[POST_PREFIX.len()..];
+                            hit(term, d, df)?;
+                        }
+                    }
+                }
+                Some(j) => {
+                    // Every key from here to the seek target dies where this one did.
+                    match next_live(dfa, rest, &states, j) {
+                        Some(next) => {
+                            from = [base, &next].concat();
+                            continue 'seek;
+                        }
+                        None => return Ok(()),
+                    }
+                }
+            }
+        }
+        return Ok(());
+    }
+}
+
+/// The smallest byte string above `rest` that the automaton hasn't rejected, given that it
+/// rejects `rest[..=dead]`: the deepest `i ≤ dead` where some byte above `rest[i]` keeps it
+/// live, and that byte. `None` when there's none, so nothing after `rest` can match.
+fn next_live(dfa: &DFA, rest: &[u8], states: &[u32], dead: usize) -> Option<Vec<u8>> {
+    for i in (0..=dead).rev() {
+        if rest[i] == u8::MAX {
+            continue;
+        }
+        if let Some(b) =
+            (rest[i] + 1..=u8::MAX).find(|&b| dfa.transition(states[i], b) != SINK_STATE)
+        {
+            let mut next = rest[..i].to_vec();
+            next.push(b);
+            return Some(next);
+        }
+    }
+    None
+}
+
+/// The first key after every key starting with `prefix`, or `None` when there's none.
+fn prefix_end(prefix: &[u8]) -> Option<Vec<u8>> {
+    let mut end = prefix.to_vec();
+    while let Some(last) = end.pop() {
+        if last < u8::MAX {
+            end.push(last + 1);
+            return Some(end);
+        }
+    }
+    None
+}
+
+fn df_at(t: &Rows, s: &Snapshot, row: &[u8]) -> Result<i64> {
+    Ok(t.get_at(s, row, DF, N)?
+        .and_then(|c| c.as_i64())
+        .unwrap_or(0))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn the_automatic_distance_follows_the_length_in_characters() {
+        assert_eq!(edits("ab", Fuzziness::Auto), 0);
+        assert_eq!(edits("abc", Fuzziness::Auto), 1);
+        assert_eq!(edits("ébcde", Fuzziness::Auto), 1);
+        assert_eq!(edits("abcdef", Fuzziness::Auto), 2);
+        assert_eq!(edits("ab", Fuzziness::Two), 2);
+    }
+
+    #[test]
+    fn a_match_that_replaces_a_whole_term_has_no_boost() {
+        assert_eq!(boost(1, 1, 0), Some(1.0));
+        assert_eq!(boost(1, 1, 1), None);
+        assert_eq!(boost(4, 2, 2), None);
+        assert_eq!(boost(4, 5, 1), Some(0.75));
+    }
+
+    #[test]
+    fn next_live_skips_the_dead_branch() {
+        let dfa = builder(1).build_dfa("cat");
+        // `cxy` is two edits into `cat` at `y`. No byte above `y` keeps `cx` live (only `t`
+        // would, below it) but a multi-byte character's lead byte: the UTF-8 automaton
+        // rejects a character at its last byte. So the seek lands on `cx\xC0`, a lower bound
+        // the walk then rejects, and every key it skipped dies where `cxy` did.
+        let rest = b"cxy";
+        let mut states = Vec::new();
+        let mut st = dfa.initial_state();
+        let mut dead = None;
+        for (i, &b) in rest.iter().enumerate() {
+            states.push(st);
+            st = dfa.transition(st, b);
+            if st == SINK_STATE {
+                dead = Some(i);
+                break;
+            }
+        }
+        assert_eq!(dead, Some(2));
+        let next = next_live(&dfa, rest, &states, 2).expect("a later key can match");
+        assert_eq!(next, b"cx\xC0");
+        for b in b'y' + 1..=0xBF {
+            assert_eq!(dfa.transition(states[2], b), SINK_STATE, "{b:#x}");
+        }
+        // At the last byte value, nothing is above.
+        assert_eq!(next_live(&dfa, b"\xff", &[dfa.initial_state()], 0), None);
+        assert_eq!(prefix_end(b"P"), Some(b"Q".to_vec()));
+        assert_eq!(prefix_end(&[0xFF, 0xFF]), None);
+    }
+}

data/vendor/text/src/highlight.rs

@@ -0,0 +1,236 @@
+//! Highlighting (D309 §4.5): each hit's matches, recomputed by re-tokenising its source text,
+//! and [`snippet`], a window of the text around the best of them.
+
+use std::collections::{BTreeMap, BTreeSet};
+use std::ops::Range;
+
+use unicode_segmentation::UnicodeSegmentation;
+
+use crate::sloppy::{Positions, sloppy};
+
+/// One occurrence of a query clause in a hit's text ([`Hit::matches`](crate::Hit::matches)).
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[non_exhaustive]
+pub struct Match {
+    /// Its byte range in the hit's text: a term's token, or an exact phrase from its first
+    /// token to its last. A sloppy phrase match lists one range per phrase word.
+    pub range: Range<usize>,
+    /// The clause it matched: an index into [`Query::clauses`](crate::Query::clauses).
+    pub clause: usize,
+    /// Which of the clause's occurrences in the text it is, from 0 in the order they're
+    /// found. The ranges of one sloppy phrase match share it.
+    pub occurrence: u32,
+}
+
+/// A window of a hit's text ([`snippet`]).
+#[derive(Debug, Clone, PartialEq, Eq)]
+#[non_exhaustive]
+pub struct Snippet {
+    /// The window's byte range in the text.
+    pub range: Range<usize>,
+    /// The matches wholly inside it, by position. Their ranges are still offsets into the
+    /// whole text, not the window, so `&text[m.range.clone()]` works; subtract
+    /// `range.start` to place them in `&text[range]`.
+    pub matches: Vec<Match>,
+}
+
+/// The window of `text`, at most `max_bytes` long, that holds the most distinct clauses of
+/// `matches`, then the most matches, then starts earliest (D309 §4.5).
+///
+/// - **A window starts at the text's start or at a match,** so it shows no context before
+///   its first match (as Lucene's simple fragmenter does).
+/// - **It ends on a word boundary** (Unicode UAX #29), so it never cuts a word or a
+///   character, and it's never longer than `max_bytes`.
+/// - **Only matches wholly inside it are returned.** A match longer than `max_bytes` never
+///   is, and a window starting at it is empty, so another window wins if one holds a match.
+/// - **With no matches,** it's the text's start.
+pub fn snippet(text: &str, matches: &[Match], max_bytes: usize) -> Snippet {
+    let mut bounds: Vec<usize> = text.split_word_bound_indices().map(|(i, _)| i).collect();
+    bounds.push(text.len());
+    let mut starts: Vec<usize> = std::iter::once(0)
+        .chain(matches.iter().map(|m| m.range.start))
+        .filter(|&s| s <= text.len())
+        .collect();
+    starts.sort_unstable();
+    starts.dedup();
+    let mut best: Option<((usize, usize), Range<usize>)> = None;
+    for s in starts {
+        let limit = s.saturating_add(max_bytes);
+        // The last word boundary within the limit; the start itself when none is.
+        let i = bounds.partition_point(|&b| b <= limit);
+        let e = if i == 0 { s } else { bounds[i - 1].max(s) };
+        let inside = matches
+            .iter()
+            .filter(|m| m.range.start >= s && m.range.end <= e);
+        let clauses: BTreeSet<usize> = inside.clone().map(|m| m.clause).collect();
+        let rank = (clauses.len(), inside.count());
+        // Strictly better only, so ties keep the earliest start.
+        if best.as_ref().is_none_or(|(r, _)| rank > *r) {
+            best = Some((rank, s..e));
+        }
+    }
+    let (_, range) = best.expect("the text's start is a candidate");
+    let mut inside: Vec<Match> = matches
+        .iter()
+        .filter(|m| m.range.start >= range.start && m.range.end <= range.end)
+        .cloned()
+        .collect();
+    inside.sort_by_key(|m| (m.range.start, m.range.end, m.clause, m.occurrence));
+    Snippet {
+        range,
+        matches: inside,
+    }
+}
+
+/// What one clause looks for in a hit's text.
+#[derive(Debug, Clone)]
+pub(crate) enum Want {
+    /// Any of these terms (a term, text's terms, a fuzzy term's expansions).
+    Terms(BTreeSet<String>),
+    /// A phrase: each term with its offset from the first, and its slop.
+    Phrase {
+        terms: Vec<(String, u32)>,
+        slop: u16,
+    },
+}
+
+/// A document's tokens as the index holds them: each term, its position and its range.
+pub(crate) type Spans = Vec<(String, u32, Range<usize>)>;
+
+/// Every match of `wants` (clause index, what it wants) in a text's `spans`, by position.
+pub(crate) fn find(spans: &Spans, wants: &[(usize, Want)]) -> Vec<Match> {
+    let mut out = Vec::new();
+    // (term, position) → range: the first token there with that term.
+    let mut at: BTreeMap<(&str, u32), &Range<usize>> = BTreeMap::new();
+    let mut lists: BTreeMap<&str, Vec<u32>> = BTreeMap::new();
+    for (t, p, r) in spans {
+        at.entry((t.as_str(), *p)).or_insert(r);
+        let l = lists.entry(t.as_str()).or_default();
+        if l.last() != Some(p) {
+            l.push(*p);
+        }
+    }
+    for (clause, want) in wants {
+        let clause = *clause;
+        let mut occurrence = 0u32;
+        match want {
+            Want::Terms(terms) => {
+                for (t, _, r) in spans {
+                    if terms.contains(t) {
+                        out.push(Match {
+                            range: r.clone(),
+                            clause,
+                            occurrence,
+                        });
+                        occurrence += 1;
+                    }
+                }
+            }
+            Want::Phrase { terms, slop: 0 } => {
+                let (first, _) = &terms[0];
+                for &p in lists.get(first.as_str()).map_or(&[][..], Vec::as_slice) {
+                    let rs: Option<Vec<&Range<usize>>> = terms
+                        .iter()
+                        .map(|(t, off)| at.get(&(t.as_str(), p + off)).copied())
+                        .collect();
+                    if let Some(rs) = rs {
+                        let start = rs.iter().map(|r| r.start).min().expect("terms");
+                        let end = rs.iter().map(|r| r.end).max().expect("terms");
+                        out.push(Match {
+                            range: start..end,
+                            clause,
+                            occurrence,
+                        });
+                        occurrence += 1;
+                    }
+                }
+            }
+            Want::Phrase { terms, slop } => {
+                let ls: Option<Vec<&[u32]>> = terms
+                    .iter()
+                    .map(|(t, _)| lists.get(t.as_str()).map(Vec::as_slice))
+                    .collect();
+                let Some(ls) = ls else {
+                    continue;
+                };
+                let walks: Vec<(&[u32], u32, usize)> = ls
+                    .iter()
+                    .zip(terms)
+                    .map(|(l, (t, off))| {
+                        let g = terms.iter().position(|(x, _)| x == t).expect("itself");
+                        (*l, *off, g)
+                    })
+                    .collect();
+                let mut rec = Positions::default();
+                sloppy(&walks, *slop, &mut rec);
+                for ps in rec.0 {
+                    for ((t, _), p) in terms.iter().zip(ps) {
+                        out.push(Match {
+                            range: at[&(t.as_str(), p)].clone(),
+                            clause,
+                            occurrence,
+                        });
+                    }
+                    occurrence += 1;
+                }
+            }
+        }
+    }
+    out.sort_by_key(|m| (m.range.start, m.range.end, m.clause, m.occurrence));
+    out
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn m(range: Range<usize>, clause: usize, occurrence: u32) -> Match {
+        Match {
+            range,
+            clause,
+            occurrence,
+        }
+    }
+
+    #[test]
+    fn the_snippet_prefers_distinct_clauses_then_more_matches_then_the_earliest() {
+        //          0         1         2         3         4
+        //          0123456789012345678901234567890123456789012345
+        let text = "alpha beta alpha gamma delta beta alpha epsilon";
+        let ms = [
+            m(0..5, 0, 0),
+            m(6..10, 1, 0),
+            m(11..16, 0, 1),
+            m(29..33, 1, 1),
+            m(34..39, 0, 2),
+        ];
+        // 16 bytes: "alpha beta alpha" holds both clauses and three matches.
+        let s = snippet(text, &ms, 16);
+        assert_eq!(s.range, 0..16);
+        assert_eq!(s.matches.len(), 3);
+        // 10 bytes: "alpha beta" and two "beta alpha"s tie; the earliest wins.
+        let s = snippet(text, &ms, 10);
+        assert_eq!(s.range, 0..10);
+        // A window never cuts a word: 13 bytes from 0 end after "beta ".
+        let s = snippet(text, &ms, 13);
+        assert_eq!(&text[s.range.clone()], "alpha beta ");
+        // No matches: the text's start.
+        let s = snippet(text, &[], 8);
+        assert_eq!((s.range, s.matches.len()), (0..6, 0));
+        // Multi-byte text is cut on a word boundary, so never inside a character: from 0,
+        // 5 bytes is "café" (é is 2), and from the match at 6 the window can't reach past
+        // "crème"'s start, so it holds nothing and the earlier start wins.
+        let s = snippet("café crème", &[m(6..12, 0, 0)], 5);
+        assert_eq!(s.range, 0..5);
+        let s = snippet("café crème", &[m(6..12, 0, 0)], 6);
+        assert_eq!((s.range, s.matches.len()), (6..12, 1));
+        // A match longer than the budget: its own window would be empty, so a window that
+        // holds another match wins, and with none, the text's start does.
+        let text = "supercalifragilistic and short";
+        let ms = [m(0..20, 0, 0), m(25..30, 1, 0)];
+        let s = snippet(text, &ms, 8);
+        assert_eq!((s.range.clone(), s.matches), (25..30, vec![ms[1].clone()]));
+        let s = snippet(text, &ms[..1], 8);
+        assert_eq!((s.range, s.matches.len()), (0..0, 0));
+    }
+}