@pify/search 0.1.0 → 0.3.0

package/native/src/score.rs

@@ -0,0 +1,147 @@
+//! Fuzzy path matching and ranking.
+//!
+//! `find` answers with everything matching a glob, in filesystem order. The
+//! question people actually ask is "the auth route file — you know the one",
+//! which needs two things a glob cannot give: a match that tolerates how
+//! people type, and an order that puts the likely file first.
+//!
+//! The model is fff's — match quality, plus frecency, recency and git state —
+//! and the constants are kept identical to the TypeScript fallback so the two
+//! engines rank the same way. A user who loses the native binary should get a
+//! slower search, not a differently-ordered one.
+
+pub const BONUS_EXACT_FILENAME: i32 = 300;
+pub const BONUS_EXACT_STEM: i32 = 200;
+pub const BONUS_FILENAME_PREFIX: i32 = 120;
+pub const BONUS_IN_FILENAME: i32 = 60;
+pub const BONUS_CONSECUTIVE: i32 = 12;
+pub const BONUS_BOUNDARY: i32 = 18;
+pub const PENALTY_LEADING: i32 = 2;
+pub const PENALTY_TYPO: i32 = 40;
+pub const PENALTY_DEPTH: i32 = 3;
+
+/// One wrong letter in four is a different mistake from one in twenty.
+pub fn max_typos(query: &str) -> usize {
+    (query.chars().count() / 4).clamp(2, 6)
+}
+
+pub fn recency_boost(mtime_ms: i64, now_ms: i64) -> i32 {
+    if mtime_ms <= 0 {
+        return 0;
+    }
+    let age = (now_ms - mtime_ms) / 1000;
+    match age {
+        a if a < 120 => 16,
+        a if a < 900 => 8,
+        a if a < 3_600 => 4,
+        a if a < 86_400 => 2,
+        a if a < 604_800 => 1,
+        _ => 0,
+    }
+}
+
+pub fn git_boost(status: &str) -> i32 {
+    match status {
+        "modified" => 24,
+        "staged" => 20,
+        "untracked" => 12,
+        _ => 0,
+    }
+}
+
+struct Match {
+    positions: Vec<usize>,
+    typos: usize,
+}
+
+/// Subsequence match within a typo budget. A skipped query character is the
+/// typo — a slip or a transposition — and spending more than the budget means
+/// this is simply not the file.
+fn match_positions(haystack: &str, needle: &str, budget: usize) -> Option<Match> {
+    let hay: Vec<char> = haystack.chars().flat_map(|c| c.to_lowercase()).collect();
+    let need: Vec<char> = needle.chars().flat_map(|c| c.to_lowercase()).collect();
+    let mut positions = Vec::with_capacity(need.len());
+    let mut typos = 0usize;
+    let mut at = 0usize;
+
+    for &want in &need {
+        match hay[at.min(hay.len())..].iter().position(|&c| c == want) {
+            Some(offset) => {
+                let found = at + offset;
+                positions.push(found);
+                at = found + 1;
+            }
+            None => {
+                typos += 1;
+                if typos > budget {
+                    return None;
+                }
+            }
+        }
+    }
+    if positions.is_empty() {
+        return None;
+    }
+    Some(Match { positions, typos })
+}
+
+fn is_boundary(chars: &[char], index: usize) -> bool {
+    if index == 0 {
+        return true;
+    }
+    let prev = chars[index - 1];
+    prev == '/' || prev == '_' || prev == '-' || prev == '.' || (prev.is_lowercase() && chars[index].is_uppercase())
+}
+
+pub struct Candidate<'a> {
+    pub path: &'a str,
+    pub frecency: i32,
+    pub git: &'a str,
+    pub mtime_ms: i64,
+}
+
+/// Score one candidate, or `None` when the query does not match it at all.
+pub fn score(candidate: &Candidate, query: &str, now_ms: i64) -> Option<i32> {
+    let base = candidate.frecency + git_boost(candidate.git) + recency_boost(candidate.mtime_ms, now_ms);
+    if query.is_empty() {
+        return Some(base);
+    }
+
+    let m = match_positions(candidate.path, query, max_typos(query))?;
+    let chars: Vec<char> = candidate.path.chars().collect();
+
+    let filename = candidate.path.rsplit('/').next().unwrap_or(candidate.path);
+    let lower_name = filename.to_lowercase();
+    let lower_query = query.to_lowercase();
+    let stem = match lower_name.rfind('.') {
+        Some(dot) if dot > 0 => &lower_name[..dot],
+        _ => lower_name.as_str(),
+    };
+
+    let mut total = 0i32;
+    if lower_name == lower_query {
+        total += BONUS_EXACT_FILENAME;
+    } else if stem == lower_query {
+        total += BONUS_EXACT_STEM;
+    } else if lower_name.starts_with(&lower_query) {
+        total += BONUS_FILENAME_PREFIX;
+    } else if lower_name.contains(&lower_query) {
+        total += BONUS_IN_FILENAME;
+    }
+
+    for (i, &at) in m.positions.iter().enumerate() {
+        if i > 0 && at == m.positions[i - 1] + 1 {
+            total += BONUS_CONSECUTIVE;
+        }
+        if at < chars.len() && is_boundary(&chars, at) {
+            total += BONUS_BOUNDARY;
+        }
+        if i == 0 {
+            total -= (at.min(40) as i32) * PENALTY_LEADING;
+        }
+    }
+
+    total -= (m.typos as i32) * PENALTY_TYPO;
+    total -= (candidate.path.matches('/').count() as i32) * PENALTY_DEPTH;
+    Some(total + base)
+}

package/native/src/store.rs

@@ -0,0 +1,194 @@
+//! Making the index outlive the process.
+//!
+//! An in-memory index is rebuilt every time pi starts. On this suite that
+//! costs 33ms and nobody notices; on a hundred-thousand-file monorepo it is
+//! seconds, paid again at every session, for a tree that has barely changed.
+//! tgrep's answer is the whole reason it is fast in practice — build once,
+//! persist, and on the next start reload and reconcile instead of rebuilding.
+//!
+//! The format is deliberately dull: a header, the file table, then the posting
+//! lists, all little-endian. No mmap, because a memory-mapped file is a
+//! different set of failure modes on every platform and the load is already
+//! bounded by the disk read.
+//!
+//! Correctness rests on one rule. A stored entry is trusted only while its
+//! size and mtime still match what is on disk; anything that differs, or is
+//! new, or has vanished, is re-read before a query can see it. A stale index
+//! that answers confidently is worse than no index at all.
+
+use std::collections::HashMap;
+use std::fs::File;
+use std::io::{BufWriter, Write};
+use std::path::Path;
+
+/// Bumped whenever the layout changes. A mismatch is not an error — it means
+/// the cache is from another version and the tree is simply re-indexed.
+const MAGIC: &[u8; 8] = b"PIFYSRC2";
+
+pub struct StoredFile {
+    pub rel: String,
+    pub size: u64,
+    pub mtime_ms: i64,
+    pub trigrams: Vec<u32>,
+}
+
+fn write_u32(out: &mut impl Write, value: u32) -> std::io::Result<()> {
+    out.write_all(&value.to_le_bytes())
+}
+
+/// Trigram lists are sorted, so the gaps between neighbours are far smaller
+/// than the values themselves and a varint spends one or two bytes where a
+/// fixed `u32` spends four. This is what decides whether persisting is worth
+/// it at all: a cache that is larger than the source it summarises costs more
+/// to read back than the rebuild it was meant to avoid.
+fn write_varint(out: &mut impl Write, mut value: u32) -> std::io::Result<()> {
+    let mut buf = [0u8; 5];
+    let mut len = 0;
+    loop {
+        let byte = (value & 0x7f) as u8;
+        value >>= 7;
+        if value == 0 {
+            buf[len] = byte;
+            len += 1;
+            break;
+        }
+        buf[len] = byte | 0x80;
+        len += 1;
+    }
+    out.write_all(&buf[..len])
+}
+
+/// Decoding reads from a slice rather than a `Read`. The index is millions of
+/// varints, and one `read_exact` per byte through a `BufReader` spends more
+/// time in the reader than in the decode; the whole file is a handful of
+/// megabytes, so it is simply read once and walked in memory.
+struct Cursor<'a> {
+    bytes: &'a [u8],
+    at: usize,
+}
+
+impl<'a> Cursor<'a> {
+    fn take(&mut self, n: usize) -> Option<&'a [u8]> {
+        let end = self.at.checked_add(n)?;
+        let slice = self.bytes.get(self.at..end)?;
+        self.at = end;
+        Some(slice)
+    }
+
+    fn u32(&mut self) -> Option<u32> {
+        Some(u32::from_le_bytes(self.take(4)?.try_into().ok()?))
+    }
+
+    fn u64(&mut self) -> Option<u64> {
+        Some(u64::from_le_bytes(self.take(8)?.try_into().ok()?))
+    }
+
+    fn i64(&mut self) -> Option<i64> {
+        Some(i64::from_le_bytes(self.take(8)?.try_into().ok()?))
+    }
+
+    fn varint(&mut self) -> Option<u32> {
+        let mut value = 0u32;
+        let mut shift = 0;
+        loop {
+            let byte = *self.bytes.get(self.at)?;
+            self.at += 1;
+            // Five groups of seven bits is the most a u32 can occupy; anything
+            // longer is corruption, not a large number.
+            if shift > 28 {
+                return None;
+            }
+            value |= u32::from(byte & 0x7f) << shift;
+            if byte & 0x80 == 0 {
+                return Some(value);
+            }
+            shift += 7;
+        }
+    }
+}
+
+fn write_u64(out: &mut impl Write, value: u64) -> std::io::Result<()> {
+    out.write_all(&value.to_le_bytes())
+}
+
+/// Write atomically: a half-written index that still has a valid header would
+/// be loaded and believed, so the rename is what makes it safe.
+pub fn save(path: &Path, files: &[StoredFile]) -> std::io::Result<()> {
+    if let Some(parent) = path.parent() {
+        std::fs::create_dir_all(parent)?;
+    }
+    let temp = path.with_extension("tmp");
+    {
+        let mut out = BufWriter::new(File::create(&temp)?);
+        out.write_all(MAGIC)?;
+        write_u32(&mut out, files.len() as u32)?;
+        for file in files {
+            let bytes = file.rel.as_bytes();
+            write_u32(&mut out, bytes.len() as u32)?;
+            out.write_all(bytes)?;
+            write_u64(&mut out, file.size)?;
+            out.write_all(&file.mtime_ms.to_le_bytes())?;
+            write_u32(&mut out, file.trigrams.len() as u32)?;
+            let mut previous = 0u32;
+            for &t in &file.trigrams {
+                write_varint(&mut out, t - previous)?;
+                previous = t;
+            }
+        }
+        out.flush()?;
+    }
+    std::fs::rename(&temp, path)
+}
+
+/// Load, or `None` for anything unreadable, truncated or from another version.
+/// A cache is an optimisation; a broken one costs a rebuild and never a result.
+pub fn load(path: &Path) -> Option<Vec<StoredFile>> {
+    let bytes = std::fs::read(path).ok()?;
+    let mut input = Cursor { bytes: &bytes, at: 0 };
+    if input.take(MAGIC.len())? != MAGIC {
+        return None;
+    }
+
+    let count = input.u32()? as usize;
+    // A corrupt length field must not turn into a huge allocation.
+    if count > 5_000_000 {
+        return None;
+    }
+    let mut files = Vec::with_capacity(count.min(65_536));
+    for _ in 0..count {
+        let name_len = input.u32()? as usize;
+        if name_len > 4096 {
+            return None;
+        }
+        let rel = std::str::from_utf8(input.take(name_len)?).ok()?.to_string();
+        let size = input.u64()?;
+        let mtime_ms = input.i64()?;
+        let trigram_count = input.u32()? as usize;
+        if trigram_count > 4_000_000 {
+            return None;
+        }
+        let mut trigrams = Vec::with_capacity(trigram_count.min(8192));
+        let mut previous = 0u32;
+        for _ in 0..trigram_count {
+            previous = previous.checked_add(input.varint()?)?;
+            trigrams.push(previous);
+        }
+        files.push(StoredFile { rel, size, mtime_ms, trigrams });
+    }
+    Some(files)
+}
+
+/// What a stored index still knows, keyed by path, so a walk can ask whether
+/// each file it finds needs re-reading.
+pub fn index_by_path(files: Vec<StoredFile>) -> HashMap<String, StoredFile> {
+    let mut map = HashMap::with_capacity(files.len());
+    for file in files {
+        map.insert(file.rel.clone(), file);
+    }
+    map
+}
+
+/// A stored entry is only trusted while the file on disk still looks the same.
+pub fn still_valid(stored: &StoredFile, size: u64, mtime_ms: i64) -> bool {
+    stored.size == size && stored.mtime_ms == mtime_ms
+}

package/native/src/trigram.rs

@@ -0,0 +1,292 @@
+//! Trigram index over file contents.
+//!
+//! Every overlapping three-byte window of a file is packed into a `u32` —
+//! `(a << 16) | (b << 8) | c` — which is injective, so a trigram is its own
+//! hash and no two distinct windows collide. The index maps each trigram to
+//! the files containing it, and a search intersects the posting lists of the
+//! trigrams its pattern must contain. Only the files that survive are read.
+//!
+//! The index only ever *narrows*. Every candidate is matched for real
+//! afterwards, so a wrong candidate costs time and never correctness — which
+//! is what makes it safe to narrow aggressively.
+
+use std::collections::{HashMap, HashSet};
+
+pub type Trigram = u32;
+
+/// A trigram key *is* its own hash, so the default SipHash is pure overhead on
+/// a path that runs once per input byte. One multiply-xorshift replaces it;
+/// the xorshift is not optional, because hashbrown takes the bucket index from
+/// the low bits and the low bits of `value * K` depend only on the last byte.
+#[derive(Default, Clone, Copy)]
+pub struct TrigramHasher(u64);
+
+impl std::hash::Hasher for TrigramHasher {
+    #[inline]
+    fn finish(&self) -> u64 {
+        self.0
+    }
+    #[inline]
+    fn write(&mut self, bytes: &[u8]) {
+        for &b in bytes {
+            self.0 = (self.0 ^ u64::from(b)).wrapping_mul(0x0000_0100_0000_01b3);
+        }
+    }
+    #[inline]
+    fn write_u32(&mut self, value: u32) {
+        let mixed = (u64::from(value)).wrapping_mul(0x9E37_79B9_7F4A_7C15);
+        self.0 = mixed ^ (mixed >> 29);
+    }
+}
+
+#[derive(Default, Clone, Copy)]
+pub struct BuildTrigramHasher;
+
+impl std::hash::BuildHasher for BuildTrigramHasher {
+    type Hasher = TrigramHasher;
+    #[inline]
+    fn build_hasher(&self) -> TrigramHasher {
+        TrigramHasher::default()
+    }
+}
+
+type TrigramMap<V> = HashMap<Trigram, V, BuildTrigramHasher>;
+
+/// Every distinct trigram in `bytes`, ASCII-folded so the index and the query
+/// agree on case without storing both.
+pub fn extract(bytes: &[u8], out: &mut HashSet<Trigram, BuildTrigramHasher>) {
+    out.clear();
+    if bytes.len() < 3 {
+        return;
+    }
+    for window in bytes.windows(3) {
+        let a = window[0].to_ascii_lowercase() as u32;
+        let b = window[1].to_ascii_lowercase() as u32;
+        let c = window[2].to_ascii_lowercase() as u32;
+        out.insert((a << 16) | (b << 8) | c);
+    }
+}
+
+pub fn of(text: &str) -> Vec<Trigram> {
+    let mut set = HashSet::with_hasher(BuildTrigramHasher);
+    extract(text.as_bytes(), &mut set);
+    let mut list: Vec<Trigram> = set.into_iter().collect();
+    list.sort_unstable();
+    list
+}
+
+/// What the index can be asked. `All` means the pattern gave nothing
+/// indexable, so no candidate set is safe and everything must be read —
+/// a different answer from "nothing matched", and conflating the two is how
+/// an index silently starts hiding results.
+#[derive(Debug, Clone)]
+pub enum Plan {
+    And(Vec<Trigram>),
+    Or(Vec<Plan>),
+    All,
+}
+
+#[derive(Default)]
+pub struct Index {
+    postings: TrigramMap<Vec<u32>>,
+    indexed: HashSet<u32>,
+}
+
+impl Index {
+    pub fn new() -> Self {
+        Self {
+            postings: HashMap::with_hasher(BuildTrigramHasher),
+            indexed: HashSet::new(),
+        }
+    }
+
+    pub fn len(&self) -> usize {
+        self.indexed.len()
+    }
+
+    pub fn add(&mut self, file: u32, trigrams: &[Trigram]) {
+        if self.indexed.contains(&file) {
+            self.remove(file);
+        }
+        for &t in trigrams {
+            let list = self.postings.entry(t).or_default();
+            // Postings stay sorted so intersection is a merge, not a scan.
+            //
+            // Bulk builds add files in ascending id order, so the new id
+            // belongs at the end and one comparison settles it. Falling
+            // through to a binary search here would cost fifteen scattered
+            // probes on a list held by every file — which measured as most of
+            // the time spent building the index.
+            match list.last() {
+                Some(&last) if last >= file => match list.binary_search(&file) {
+                    Ok(_) => {}
+                    Err(at) => list.insert(at, file),
+                },
+                _ => list.push(file),
+            }
+        }
+        self.indexed.insert(file);
+    }
+
+    pub fn remove(&mut self, file: u32) {
+        if !self.indexed.remove(&file) {
+            return;
+        }
+        self.postings.retain(|_, list| {
+            if let Ok(at) = list.binary_search(&file) {
+                list.remove(at);
+            }
+            !list.is_empty()
+        });
+    }
+
+    /// The index turned back the other way up: every trigram, grouped by the
+    /// file it came from, for writing the index out.
+    ///
+    /// One pass over the postings rather than one pass *per file* — asking each
+    /// file separately would rescan the entire index for every file, which on a
+    /// large tree is quadratic and would cost more than rebuilding from source.
+    pub fn by_file(&self) -> HashMap<u32, Vec<Trigram>> {
+        let mut out: HashMap<u32, Vec<Trigram>> = HashMap::with_capacity(self.indexed.len());
+        for &file in &self.indexed {
+            out.insert(file, Vec::new());
+        }
+        for (&t, files) in &self.postings {
+            for file in files {
+                if let Some(list) = out.get_mut(file) {
+                    list.push(t);
+                }
+            }
+        }
+        for list in out.values_mut() {
+            list.sort_unstable();
+        }
+        out
+    }
+
+    /// Candidate file ids, or `None` meaning every file must be read.
+    pub fn candidates(&self, plan: &Plan) -> Option<Vec<u32>> {
+        match plan {
+            Plan::All => None,
+            Plan::Or(branches) => {
+                let mut union: Vec<u32> = Vec::new();
+                for branch in branches {
+                    let part = self.candidates(branch)?;
+                    union.extend_from_slice(&part);
+                }
+                union.sort_unstable();
+                union.dedup();
+                Some(union)
+            }
+            Plan::And(trigrams) => {
+                if trigrams.is_empty() {
+                    return None;
+                }
+                let mut lists: Vec<&Vec<u32>> = Vec::with_capacity(trigrams.len());
+                for t in trigrams {
+                    match self.postings.get(t) {
+                        // A trigram nobody has means nothing can match — an
+                        // empty answer, emphatically not "read everything".
+                        None => return Some(Vec::new()),
+                        Some(list) => lists.push(list),
+                    }
+                }
+                // Smallest first, so the working set only ever shrinks.
+                lists.sort_by_key(|l| l.len());
+                let mut result = lists[0].clone();
+                for list in &lists[1..] {
+                    if result.is_empty() {
+                        break;
+                    }
+                    result = intersect(&result, list);
+                }
+                Some(result)
+            }
+        }
+    }
+}
+
+/// Both sides are sorted, so this is a linear merge rather than a lookup loop.
+fn intersect(a: &[u32], b: &[u32]) -> Vec<u32> {
+    let mut out = Vec::with_capacity(a.len().min(b.len()));
+    let (mut i, mut j) = (0usize, 0usize);
+    while i < a.len() && j < b.len() {
+        match a[i].cmp(&b[j]) {
+            std::cmp::Ordering::Equal => {
+                out.push(a[i]);
+                i += 1;
+                j += 1;
+            }
+            std::cmp::Ordering::Less => i += 1,
+            std::cmp::Ordering::Greater => j += 1,
+        }
+    }
+    out
+}
+
+/// Literal runs of three or more characters — the only parts of a pattern that
+/// imply required trigrams.
+pub fn literal_runs(pattern: &str) -> Vec<String> {
+    let chars: Vec<char> = pattern.chars().collect();
+    let mut runs = Vec::new();
+    let mut current = String::new();
+    let mut i = 0usize;
+    while i < chars.len() {
+        let ch = chars[i];
+        if ch == '\\' {
+            if let Some(&next) = chars.get(i + 1) {
+                if !next.is_ascii_alphanumeric() {
+                    current.push(next);
+                    i += 2;
+                    continue;
+                }
+            }
+            runs.push(std::mem::take(&mut current));
+            i += 2;
+            continue;
+        }
+        // A quantifier applies to the character before it, so that character
+        // is not required either.
+        if matches!(ch, '?' | '*' | '+' | '{') {
+            current.pop();
+            runs.push(std::mem::take(&mut current));
+            i += 1;
+            continue;
+        }
+        if matches!(ch, '[' | ']' | '(' | ')' | '}' | '|' | '.' | '^' | '$') {
+            runs.push(std::mem::take(&mut current));
+            i += 1;
+            continue;
+        }
+        current.push(ch);
+        i += 1;
+    }
+    runs.push(current);
+    runs.retain(|r| r.chars().count() >= 3);
+    runs
+}
+
+pub fn plan_for_literal(literal: &str) -> Plan {
+    let trigrams = of(literal);
+    if trigrams.is_empty() {
+        Plan::All
+    } else {
+        Plan::And(trigrams)
+    }
+}
+
+pub fn plan_for_regex(pattern: &str) -> Plan {
+    if pattern.contains('|') {
+        let branches: Vec<Plan> = pattern.split('|').map(plan_for_regex).collect();
+        // One branch that can match anywhere makes the whole union unbounded.
+        if branches.iter().any(|b| matches!(b, Plan::All)) {
+            return Plan::All;
+        }
+        return Plan::Or(branches);
+    }
+    let runs = literal_runs(pattern);
+    match runs.iter().max_by_key(|r| r.len()) {
+        None => Plan::All,
+        Some(longest) => plan_for_literal(longest),
+    }
+}