bare-ripgrep 0.0.0-stage → 0.0.1

package/src/lib.rs

@@ -0,0 +1,111 @@
+mod glob;
+mod request;
+mod scope;
+mod search;
+mod walk;
+
+use std::panic::{catch_unwind, AssertUnwindSafe};
+use std::slice;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, Mutex};
+
+use request::{Op, Request};
+use scope::Scope;
+use serde::Serialize;
+
+pub struct Job {
+    request: Vec<u8>,
+    cancel: Arc<AtomicBool>,
+    result: Mutex<Vec<u8>>,
+}
+
+#[derive(Serialize)]
+struct Failure {
+    error: String,
+}
+
+fn execute(job: &Job) -> Result<Vec<u8>, String> {
+    let request: Request =
+        serde_json::from_slice(&job.request).map_err(|err| format!("invalid request: {err}"))?;
+    let scope = Arc::new(Scope::new(&request, Arc::clone(&job.cancel))?);
+    let max_matches = usize::try_from(request.max_matches).unwrap_or(usize::MAX - 1);
+
+    let json = match request.op {
+        Op::Search => {
+            let matcher = search::matcher(&request)?;
+            serde_json::to_vec(&search::search(scope, matcher, max_matches))
+        }
+        Op::Walk => serde_json::to_vec(&walk::walk(scope)),
+    };
+    json.map_err(|err| err.to_string())
+}
+
+fn panic_message(payload: &(dyn std::any::Any + Send)) -> String {
+    let detail = payload
+        .downcast_ref::<&str>()
+        .map(|text| text.to_string())
+        .or_else(|| payload.downcast_ref::<String>().cloned())
+        .unwrap_or_default();
+    format!("internal error: {detail}")
+}
+
+/// # Safety
+/// `json` must point to `len` readable bytes.
+#[no_mangle]
+pub unsafe extern "C" fn rg_job_new(json: *const u8, len: usize) -> *mut Job {
+    let request = slice::from_raw_parts(json, len).to_vec();
+    Box::into_raw(Box::new(Job {
+        request,
+        cancel: Arc::new(AtomicBool::new(false)),
+        result: Mutex::new(Vec::new()),
+    }))
+}
+
+/// Blocks until the job is done; call it off the JS thread. Never unwinds.
+///
+/// # Safety
+/// `job` must come from `rg_job_new` and not be freed while this runs.
+#[no_mangle]
+pub unsafe extern "C" fn rg_job_run(job: *const Job) {
+    let job = &*job;
+    let outcome = catch_unwind(AssertUnwindSafe(|| execute(job)));
+    let bytes = match outcome {
+        Ok(Ok(bytes)) => bytes,
+        Ok(Err(error)) => serde_json::to_vec(&Failure { error }).unwrap_or_default(),
+        Err(payload) => serde_json::to_vec(&Failure {
+            error: panic_message(payload.as_ref()),
+        })
+        .unwrap_or_default(),
+    };
+    *job.result
+        .lock()
+        .unwrap_or_else(|poisoned| poisoned.into_inner()) = bytes;
+}
+
+/// # Safety
+/// `job` must come from `rg_job_new` and not be freed.
+#[no_mangle]
+pub unsafe extern "C" fn rg_job_cancel(job: *const Job) {
+    (*job).cancel.store(true, Ordering::Relaxed);
+}
+
+/// The JSON result, valid until `rg_job_free`. Call after `rg_job_run` returned.
+///
+/// # Safety
+/// `job` must come from `rg_job_new`; `len` must be writable.
+#[no_mangle]
+pub unsafe extern "C" fn rg_job_result(job: *const Job, len: *mut usize) -> *const u8 {
+    let result = (*job)
+        .result
+        .lock()
+        .unwrap_or_else(|poisoned| poisoned.into_inner());
+    *len = result.len();
+    result.as_ptr()
+}
+
+/// # Safety
+/// `job` must come from `rg_job_new` and not be used after.
+#[no_mangle]
+pub unsafe extern "C" fn rg_job_free(job: *mut Job) {
+    drop(Box::from_raw(job));
+}

package/src/request.rs

@@ -0,0 +1,31 @@
+use serde::Deserialize;
+
+#[derive(Deserialize, Clone, Copy, PartialEq, Eq)]
+#[serde(rename_all = "lowercase")]
+pub enum Op {
+    Search,
+    Walk,
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+pub struct Request {
+    pub op: Op,
+    pub root: String,
+    #[serde(default)]
+    pub folder: String,
+    #[serde(default)]
+    pub queries: Vec<String>,
+    #[serde(default)]
+    pub regex: bool,
+    #[serde(default)]
+    pub case_sensitive: bool,
+    #[serde(default)]
+    pub include: Vec<String>,
+    #[serde(default)]
+    pub deny: Vec<String>,
+    #[serde(default)]
+    pub include_ignored: bool,
+    pub max_matches: u64,
+    pub max_entries: u64,
+}

package/src/scope.rs

@@ -0,0 +1,170 @@
+use std::path::{Path, MAIN_SEPARATOR};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::Arc;
+
+use ignore::{DirEntry, WalkBuilder};
+
+use crate::glob::Glob;
+use crate::request::Request;
+
+// Skipped even with includeIgnored.
+const SKIP_DIRS: [&str; 3] = [".git", "node_modules", ".trash"];
+// includeIgnored lifts these.
+const HIDDEN_DIRS: [&str; 5] = [".hg", ".svn", ".venv", ".cache", ".next"];
+
+pub struct Scope {
+    pub root: String,
+    folder: Vec<String>,
+    include_ignored: bool,
+    deny: Vec<String>,
+    include: Vec<Glob>,
+    max_entries: u64,
+    entries: AtomicU64,
+    pub truncated: AtomicBool,
+    pub cancel: Arc<AtomicBool>,
+}
+
+impl Scope {
+    pub fn new(req: &Request, cancel: Arc<AtomicBool>) -> Result<Scope, String> {
+        let mut folder = Vec::new();
+        for part in req.folder.split(['/', '\\']) {
+            match part {
+                "" | "." => {}
+                ".." => return Err(format!("folder escapes root: {}", req.folder)),
+                part => folder.push(part.to_string()),
+            }
+        }
+
+        let mut dir = Path::new(&req.root).to_path_buf();
+        dir.extend(&folder);
+        if !dir.is_dir() {
+            return Err(format!("not a directory: {}", dir.display()));
+        }
+
+        let include = req
+            .include
+            .iter()
+            .map(|pattern| Glob::new(pattern))
+            .collect::<Result<_, _>>()?;
+
+        Ok(Scope {
+            root: req.root.clone(),
+            folder,
+            include_ignored: req.include_ignored,
+            deny: req.deny.iter().map(|path| fold(path)).collect(),
+            include,
+            max_entries: req.max_entries,
+            entries: AtomicU64::new(0),
+            truncated: AtomicBool::new(false),
+            cancel,
+        })
+    }
+
+    pub fn walker(self: &Arc<Scope>, threads: usize) -> WalkBuilder {
+        let scope = Arc::clone(self);
+        let mut builder = WalkBuilder::new(&self.root);
+        builder
+            .standard_filters(false)
+            .git_ignore(!self.include_ignored)
+            .ignore(!self.include_ignored)
+            .require_git(false)
+            .follow_links(false)
+            .threads(threads)
+            .filter_entry(move |entry| scope.admits(entry));
+        builder
+    }
+
+    fn admits(&self, entry: &DirEntry) -> bool {
+        let depth = entry.depth();
+        if depth == 0 {
+            return true;
+        }
+
+        let name = entry.file_name().to_string_lossy();
+        if SKIP_DIRS.contains(&name.as_ref()) {
+            return false;
+        }
+
+        let is_dir = entry.file_type().is_some_and(|kind| kind.is_dir());
+        if is_dir && !self.include_ignored && HIDDEN_DIRS.contains(&name.as_ref()) {
+            return false;
+        }
+
+        // Walking from the root keeps the ignore files between it and the folder in force.
+        if depth <= self.folder.len() && name != self.folder[depth - 1] {
+            return false;
+        }
+
+        !self.is_denied(entry.path())
+    }
+
+    fn is_denied(&self, path: &Path) -> bool {
+        if self.deny.is_empty() {
+            return false;
+        }
+
+        let path = fold(&path.to_string_lossy());
+        self.deny.iter().any(|root| {
+            path == *root
+                || path
+                    .strip_prefix(root.as_str())
+                    .is_some_and(|rest| rest.starts_with('/'))
+        })
+    }
+
+    pub fn cancelled(&self) -> bool {
+        self.cancel.load(Ordering::Relaxed)
+    }
+
+    /// Counts what the walk keeps, below the folder. False once over the cap.
+    pub fn keep(&self, entry: &DirEntry) -> bool {
+        if entry.depth() <= self.folder.len() {
+            return true;
+        }
+
+        let kept = self.entries.fetch_add(1, Ordering::Relaxed) + 1;
+        if kept > self.max_entries {
+            self.truncated.store(true, Ordering::Relaxed);
+            return false;
+        }
+
+        true
+    }
+
+    pub fn relative(&self, entry: &DirEntry) -> String {
+        let path = entry
+            .path()
+            .strip_prefix(&self.root)
+            .unwrap_or(entry.path());
+        let path = path.to_string_lossy();
+        if MAIN_SEPARATOR == '/' {
+            path.into_owned()
+        } else {
+            path.replace(MAIN_SEPARATOR, "/")
+        }
+    }
+
+    pub fn included(&self, entry: &DirEntry, path: &str) -> bool {
+        if self.include.is_empty() {
+            return true;
+        }
+
+        let name = entry.file_name().to_string_lossy();
+        self.include.iter().any(|glob| glob.matches(&name, path))
+    }
+}
+
+pub fn threads() -> usize {
+    std::thread::available_parallelism()
+        .map_or(4, |n| n.get())
+        .min(12)
+}
+
+fn fold(path: &str) -> String {
+    let path = path.to_lowercase();
+    if cfg!(windows) {
+        path.replace('\\', "/")
+    } else {
+        path
+    }
+}

package/src/search.rs

@@ -0,0 +1,224 @@
+use std::collections::BinaryHeap;
+use std::fs::File;
+use std::io::{self, Read};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, Mutex};
+
+use grep_matcher::{LineTerminator, Matcher};
+use grep_regex::{RegexMatcher, RegexMatcherBuilder};
+use grep_searcher::{BinaryDetection, Searcher, SearcherBuilder, Sink, SinkMatch};
+use ignore::WalkState;
+use serde::Serialize;
+
+use crate::request::Request;
+use crate::scope::{threads, Scope};
+
+const MAX_LINE_CHARS: usize = 500;
+// head only: a NUL further in is text, as in the harness
+const BINARY_PROBE_BYTES: usize = 1024;
+// a longer line is searched in its head only
+const MAX_LINE_BYTES: usize = 1024 * 1024;
+const HEAP_LIMIT_BYTES: usize = 16 * 1024 * 1024;
+
+#[derive(Serialize, PartialEq, Eq, PartialOrd, Ord)]
+pub struct Hit {
+    path: String,
+    line: u64,
+    text: String,
+}
+
+#[derive(Serialize)]
+pub struct Searched {
+    hits: Vec<Hit>,
+    truncated: bool,
+    cut: bool,
+}
+
+/// The `max_matches` smallest hits by (path, line), whatever order the files finish in.
+struct Best {
+    max: usize,
+    hits: BinaryHeap<Hit>,
+    over: bool,
+}
+
+impl Best {
+    fn add(&mut self, hits: Vec<Hit>) {
+        for hit in hits {
+            self.hits.push(hit);
+            if self.hits.len() > self.max {
+                self.hits.pop();
+                self.over = true;
+            }
+        }
+    }
+
+    /// A file sorting past a full set can add nothing.
+    fn beyond(&self, path: &str) -> bool {
+        self.over
+            && self
+                .hits
+                .peek()
+                .is_some_and(|last| path > last.path.as_str())
+    }
+}
+
+pub fn matcher(req: &Request) -> Result<RegexMatcher, String> {
+    if req.queries.is_empty() {
+        return Err("queries is empty".into());
+    }
+
+    RegexMatcherBuilder::new()
+        .case_insensitive(!req.case_sensitive)
+        .fixed_strings(!req.regex)
+        .line_terminator(Some(b'\n'))
+        .build_many(&req.queries)
+        .map_err(|err| format!("invalid regex: {err}"))
+}
+
+pub fn search(scope: Arc<Scope>, matcher: RegexMatcher, max_matches: usize) -> Searched {
+    let best = Mutex::new(Best {
+        max: max_matches,
+        hits: BinaryHeap::new(),
+        over: false,
+    });
+    let cut = AtomicBool::new(false);
+
+    scope.walker(threads()).build_parallel().run(|| {
+        let scope = &scope;
+        let best = &best;
+        let cut = &cut;
+        let matcher = matcher.clone();
+        let mut searcher = SearcherBuilder::new()
+            .line_number(true)
+            .line_terminator(LineTerminator::byte(b'\n'))
+            .binary_detection(BinaryDetection::none())
+            .heap_limit(Some(HEAP_LIMIT_BYTES))
+            .build();
+
+        Box::new(move |result| {
+            let Ok(entry) = result else {
+                return WalkState::Continue;
+            };
+            if scope.cancelled() || !scope.keep(&entry) {
+                return WalkState::Quit;
+            }
+            if !entry.file_type().is_some_and(|kind| kind.is_file()) {
+                return WalkState::Continue;
+            }
+
+            let path = scope.relative(&entry);
+            if !scope.included(&entry, &path) || best.lock().unwrap().beyond(&path) {
+                return WalkState::Continue;
+            }
+
+            let mut sink = Collect {
+                matcher: &matcher,
+                path: &path,
+                // one past the cap, so a file alone can show there is more
+                room: max_matches + 1,
+                hits: Vec::new(),
+                cut: false,
+            };
+            scan(entry.path(), &matcher, &mut searcher, &mut sink);
+            if sink.cut {
+                cut.store(true, Ordering::Relaxed);
+            }
+            if !sink.hits.is_empty() {
+                best.lock().unwrap().add(sink.hits);
+            }
+
+            WalkState::Continue
+        })
+    });
+
+    let best = best.into_inner().unwrap();
+    Searched {
+        hits: best.hits.into_sorted_vec(),
+        truncated: best.over || scope.truncated.load(Ordering::Relaxed),
+        cut: cut.load(Ordering::Relaxed),
+    }
+}
+
+fn scan(
+    path: &std::path::Path,
+    matcher: &RegexMatcher,
+    searcher: &mut Searcher,
+    sink: &mut Collect,
+) {
+    let Ok(mut file) = File::open(path) else {
+        return;
+    };
+
+    let mut probe = [0u8; BINARY_PROBE_BYTES];
+    let mut filled = 0;
+    while filled < probe.len() {
+        match file.read(&mut probe[filled..]) {
+            Ok(0) => break,
+            Ok(n) => filled += n,
+            Err(err) if err.kind() == io::ErrorKind::Interrupted => {}
+            Err(_) => return,
+        }
+    }
+    if probe[..filled].contains(&0) {
+        return;
+    }
+
+    // what arrived before a read error still counts, and the file is flagged cut
+    if searcher
+        .search_reader(matcher, (&probe[..filled]).chain(file), &mut *sink)
+        .is_err()
+    {
+        sink.cut = true;
+    }
+}
+
+struct Collect<'a> {
+    matcher: &'a RegexMatcher,
+    path: &'a str,
+    room: usize,
+    hits: Vec<Hit>,
+    cut: bool,
+}
+
+impl Sink for Collect<'_> {
+    type Error = io::Error;
+
+    fn matched(&mut self, _: &Searcher, found: &SinkMatch<'_>) -> Result<bool, io::Error> {
+        let mut line = found.bytes();
+        if line.ends_with(b"\n") {
+            line = &line[..line.len() - 1];
+        }
+
+        if line.len() > MAX_LINE_BYTES {
+            self.cut = true;
+            line = &line[..MAX_LINE_BYTES];
+            if !self.matcher.is_match(line).unwrap_or(false) {
+                return Ok(true);
+            }
+        }
+
+        self.hits.push(Hit {
+            path: self.path.to_string(),
+            line: found.line_number().unwrap_or(0),
+            text: clip(line),
+        });
+        Ok(self.hits.len() < self.room)
+    }
+}
+
+/// The first MAX_LINE_CHARS UTF-16 units, like `text.slice(0, 500)`.
+/// A surrogate pair split by the cut is dropped whole.
+fn clip(line: &[u8]) -> String {
+    // 500 units take at most 1500 bytes, so a cut inside a character lands past them
+    let head = String::from_utf8_lossy(&line[..line.len().min(2048)]);
+    let mut units = 0;
+    let mut text = String::new();
+    for ch in head.chars() {
+        units += ch.len_utf16();
+        if units > MAX_LINE_CHARS {
+            break;
+        }
+        text.push(ch);
+    }
+    text
+}

package/src/walk.rs

@@ -0,0 +1,61 @@
+use std::sync::atomic::Ordering;
+use std::sync::{Arc, Mutex};
+use std::time::UNIX_EPOCH;
+
+use ignore::WalkState;
+use serde::Serialize;
+
+use crate::scope::{threads, Scope};
+
+#[derive(Serialize)]
+pub struct File {
+    path: String,
+    mtime: i64,
+}
+
+#[derive(Serialize)]
+pub struct Walked {
+    files: Vec<File>,
+    truncated: bool,
+}
+
+pub fn walk(scope: Arc<Scope>) -> Walked {
+    let files = Mutex::new(Vec::new());
+
+    scope.walker(threads()).build_parallel().run(|| {
+        let scope = &scope;
+        let files = &files;
+        Box::new(move |result| {
+            let Ok(entry) = result else {
+                return WalkState::Continue;
+            };
+            if scope.cancelled() || !scope.keep(&entry) {
+                return WalkState::Quit;
+            }
+            if !entry.file_type().is_some_and(|kind| kind.is_file()) {
+                return WalkState::Continue;
+            }
+
+            let path = scope.relative(&entry);
+            if !scope.included(&entry, &path) {
+                return WalkState::Continue;
+            }
+
+            let mtime = entry
+                .metadata()
+                .ok()
+                .and_then(|meta| meta.modified().ok())
+                .and_then(|time| time.duration_since(UNIX_EPOCH).ok())
+                .map_or(0, |since| since.as_millis() as i64);
+            files.lock().unwrap().push(File { path, mtime });
+            WalkState::Continue
+        })
+    });
+
+    let mut files = files.into_inner().unwrap();
+    files.sort_unstable_by(|a, b| a.path.cmp(&b.path));
+    Walked {
+        files,
+        truncated: scope.truncated.load(Ordering::Relaxed),
+    }
+}