rubydex 0.2.5 → 0.2.7

data/rust/Cargo.lock

@@ -180,7 +180,7 @@ dependencies = [
  "serde_json",
  "syn",
  "tempfile",
- "toml",
+ "toml 0.8.23",
 ]
 
 [[package]]
@@ -536,9 +536,9 @@ checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2"
 
 [[package]]
 name = "hashbrown"
-version = "0.15.4"
+version = "0.17.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5"
+checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
 
 [[package]]
 name = "heck"
@@ -685,9 +685,9 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "2.10.0"
+version = "2.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661"
+checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
 dependencies = [
  "equivalent",
  "hashbrown",
@@ -823,6 +823,12 @@ version = "1.70.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad"
 
+[[package]]
+name = "paste"
+version = "1.0.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
+
 [[package]]
 name = "pastey"
 version = "0.2.1"
@@ -1048,7 +1054,7 @@ dependencies = [
 
 [[package]]
 name = "rubydex"
-version = "0.1.0"
+version = "0.2.6"
 dependencies = [
  "assert_cmd",
  "bitflags",
@@ -1065,14 +1071,18 @@ dependencies = [
  "ruby-prism",
  "ruby-rbs",
  "rubydex",
+ "serde",
  "tempfile",
+ "tikv-jemalloc-ctl",
+ "tikv-jemallocator",
+ "toml 1.1.2+spec-1.1.0",
  "url",
  "xxhash-rust",
 ]
 
 [[package]]
 name = "rubydex-mcp"
-version = "0.1.0"
+version = "0.2.6"
 dependencies = [
  "assert_cmd",
  "clap",
@@ -1087,7 +1097,7 @@ dependencies = [
 
 [[package]]
 name = "rubydex-sys"
-version = "0.1.0"
+version = "0.2.6"
 dependencies = [
  "cbindgen",
  "libc",
@@ -1155,18 +1165,28 @@ dependencies = [
 
 [[package]]
 name = "serde"
-version = "1.0.219"
+version = "1.0.228"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6"
+checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
+dependencies = [
+ "serde_core",
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.219"
+version = "1.0.228"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
+checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1205,6 +1225,15 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "serde_spanned"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26"
+dependencies = [
+ "serde_core",
+]
+
 [[package]]
 name = "serde_yaml"
 version = "0.9.34+deprecated"
@@ -1315,6 +1344,37 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "tikv-jemalloc-ctl"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a184c43b8ab2f41df2733b55556e3f5f632f4aeaa205b1bb018f574b7f5f142"
+dependencies = [
+ "libc",
+ "paste",
+ "tikv-jemalloc-sys",
+]
+
+[[package]]
+name = "tikv-jemalloc-sys"
+version = "0.7.1+5.3.1-0-g81034ce1f1373e37dc865038e1bc8eeecf559ce8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a2825c78386b4ae0314074867860ba9577875de945f05992c38815cbec327f0"
+dependencies = [
+ "cc",
+ "libc",
+]
+
+[[package]]
+name = "tikv-jemallocator"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "249f09e49ab1609436f34c776e84231bead18d6a955f119f939bdc1d847561bd"
+dependencies = [
+ "libc",
+ "tikv-jemalloc-sys",
+]
+
 [[package]]
 name = "tinystr"
 version = "0.8.1"
@@ -1367,11 +1427,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"
 dependencies = [
  "serde",
- "serde_spanned",
- "toml_datetime",
+ "serde_spanned 0.6.9",
+ "toml_datetime 0.6.11",
  "toml_edit",
 ]
 
+[[package]]
+name = "toml"
+version = "1.1.2+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee"
+dependencies = [
+ "indexmap",
+ "serde_core",
+ "serde_spanned 1.1.1",
+ "toml_datetime 1.1.1+spec-1.1.0",
+ "toml_parser",
+ "toml_writer",
+ "winnow 1.0.3",
+]
+
 [[package]]
 name = "toml_datetime"
 version = "0.6.11"
@@ -1381,6 +1456,15 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "toml_datetime"
+version = "1.1.1+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7"
+dependencies = [
+ "serde_core",
+]
+
 [[package]]
 name = "toml_edit"
 version = "0.22.27"
@@ -1389,10 +1473,19 @@ checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
 dependencies = [
  "indexmap",
  "serde",
- "serde_spanned",
- "toml_datetime",
+ "serde_spanned 0.6.9",
+ "toml_datetime 0.6.11",
  "toml_write",
- "winnow",
+ "winnow 0.7.12",
+]
+
+[[package]]
+name = "toml_parser"
+version = "1.1.2+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526"
+dependencies = [
+ "winnow 1.0.3",
 ]
 
 [[package]]
@@ -1401,6 +1494,12 @@ version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
 
+[[package]]
+name = "toml_writer"
+version = "1.1.1+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db"
+
 [[package]]
 name = "tracing"
 version = "0.1.44"
@@ -1751,6 +1850,12 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "winnow"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1"
+
 [[package]]
 name = "wit-bindgen-rt"
 version = "0.39.0"

data/rust/rubydex/Cargo.toml

@@ -1,9 +1,15 @@
 [package]
 name = "rubydex"
-version = "0.1.0"
+version = "0.2.6"
 edition = "2024"
 rust-version = "1.89.0"
 license = "MIT"
+description = "High-performance Ruby code indexing and static analysis library."
+homepage = "https://github.com/Shopify/rubydex"
+repository = "https://github.com/Shopify/rubydex"
+readme = "README.md"
+keywords = ["ruby", "indexing", "static-analysis"]
+categories = ["development-tools"]
 
 [[bin]]
 name = "rubydex_cli"
@@ -13,6 +19,11 @@ path = "src/main.rs"
 crate-type = ["rlib"]
 
 [features]
+jemalloc = ["dep:tikv-jemallocator"]
+# The Ruby native extension uses a dlopen to load the rubydex shared object. In these scenarios, jemalloc must be linked
+# dynamically or else the process crashes. This has a small performance cost, which we don't need to pay when using the
+# Rust crate by itself, so we gate it with a feature
+jemalloc_dylib = ["jemalloc", "tikv-jemallocator/disable_initial_exec_tls"]
 test_utils = ["dep:tempfile"]
 
 [dependencies]
@@ -26,17 +37,30 @@ bitflags = "2.9"
 bytecount = "0.6.9"
 libc = "0.2"
 line-index = "0.1.2"
+serde = { version = "1.0", features = ["derive"] }
 tempfile = { version = "3.0", optional = true }
 crossbeam-deque = "0.8"
 crossbeam-utils = "0.8"
 crossbeam-channel = "0.5"
+toml = "1.1.2"
+
+[target.'cfg(not(target_os = "windows"))'.dependencies]
+tikv-jemallocator = { version = "0.7.0", optional = true }
 
 [dev-dependencies]
-rubydex = { path = ".", features = ["test_utils"] }
+rubydex = { path = ".", features = ["test_utils", "jemalloc"] }
 tempfile = "3.0"
 assert_cmd = "2.0"
 predicates = "3.1"
 regex = "1.10"
 
+[target.'cfg(not(target_os = "windows"))'.dev-dependencies]
+tikv-jemallocator = { version = "0.7.0", features = ["stats"] }
+tikv-jemalloc-ctl = { version = "0.7.0", features = ["stats"] }
+
+[[bench]]
+name = "graph_memory"
+harness = false
+
 [lints]
 workspace = true

data/rust/rubydex/benches/graph_memory.rs

@@ -0,0 +1,46 @@
+#[cfg(all(feature = "jemalloc", not(target_os = "windows")))]
+mod imp {
+    use std::collections::HashSet;
+
+    use rubydex::{
+        indexing::{self, IndexerBackend},
+        listing,
+        model::graph::Graph,
+        resolution::Resolver,
+    };
+    use tikv_jemalloc_ctl::{epoch, stats};
+
+    /// Advance the jemalloc epoch (stats are cached between epochs) and read the
+    /// number of bytes currently allocated by the application.
+    fn allocated_bytes() -> usize {
+        epoch::advance().expect("failed to advance jemalloc epoch");
+        stats::allocated::read().expect("failed to read stats.allocated (is the `stats` feature on?)")
+    }
+
+    pub fn run() {
+        let paths: Vec<String> = std::env::args().skip(1).collect();
+        let paths = if paths.is_empty() { vec![".".to_string()] } else { paths };
+        let (file_paths, _) = listing::collect_file_paths(paths, &HashSet::new());
+
+        let mut graph = Graph::new();
+        let _ = indexing::index_files(&mut graph, file_paths, IndexerBackend::RubyIndexer);
+        Resolver::new(&mut graph).resolve();
+
+        // Compare the total memory used in the allocator before and after dropping the graph
+        let before_drop = allocated_bytes();
+        drop(graph);
+        let after_drop = allocated_bytes();
+
+        let total_graph_memory = before_drop.saturating_sub(after_drop);
+
+        #[allow(clippy::cast_precision_loss)]
+        let mega_bytes = total_graph_memory as f64 / 1024.0 / 1024.0;
+
+        println!("Total graph memory: {mega_bytes:.2} MB");
+    }
+}
+
+fn main() {
+    #[cfg(all(feature = "jemalloc", not(target_os = "windows")))]
+    imp::run();
+}

data/rust/rubydex/src/config.rs

@@ -0,0 +1,275 @@
+use crate::assert_mem_size;
+use crate::errors::Errors;
+use serde::Deserialize;
+use std::collections::HashSet;
+use std::fs;
+use std::io::ErrorKind;
+use std::path::{Path, PathBuf};
+
+pub const DEFAULT_EXCLUDED_DIRECTORIES: &[&str] = &[
+    ".bundle",
+    ".claude",
+    ".git",
+    ".github",
+    ".ruby-lsp",
+    ".vscode",
+    "log",
+    "node_modules",
+    "tmp",
+];
+
+/// Configuration coming from a config file
+#[derive(Debug, Default, Deserialize)]
+struct ConfigFile {
+    /// Paths to exclude from file discovery during indexing.
+    #[serde(default)]
+    exclude: Vec<PathBuf>,
+}
+
+/// Project configuration
+#[derive(Debug)]
+pub struct Config {
+    /// Path to the workspace being analyzed
+    workspace_path: Box<Path>,
+    /// Paths to exclude from file discovery during indexing.
+    excluded_paths: HashSet<PathBuf>,
+}
+assert_mem_size!(Config, 64);
+
+impl Default for Config {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Config {
+    /// Creates a configuration whose workspace path defaults to the current working directory.
+    #[must_use]
+    pub fn new() -> Self {
+        Self {
+            workspace_path: std::env::current_dir()
+                .unwrap_or_else(|_| PathBuf::from("."))
+                .into_boxed_path(),
+            excluded_paths: DEFAULT_EXCLUDED_DIRECTORIES.iter().map(PathBuf::from).collect(),
+        }
+    }
+
+    /// Returns the root directory of the workspace being analyzed.
+    #[must_use]
+    pub fn workspace_path(&self) -> &Path {
+        &self.workspace_path
+    }
+
+    /// Sets the root directory of the workspace being analyzed.
+    pub fn set_workspace_path(&mut self, workspace_path: PathBuf) {
+        self.workspace_path = workspace_path.into_boxed_path();
+    }
+
+    /// Adds paths to exclude from file discovery during indexing. Excluded directories will be skipped entirely during
+    /// directory traversal.
+    pub fn exclude_paths(&mut self, paths: impl IntoIterator<Item = PathBuf>) {
+        self.excluded_paths.extend(paths);
+    }
+
+    /// Returns the set of paths excluded from file discovery
+    #[must_use]
+    pub fn excluded_paths(&self) -> HashSet<PathBuf> {
+        self.excluded_paths
+            .iter()
+            .map(|path| self.workspace_path.join(path))
+            .collect()
+    }
+
+    /// Merges the default `rubydex.toml` configuration file from the workspace root into this config, if present.
+    ///
+    /// The default config file is optional, so a missing `rubydex.toml` is silently ignored. Any other failure (an
+    /// unreadable or malformed file) is still reported.
+    ///
+    /// # Errors
+    ///
+    /// Will error if the config file exists but cannot be read or has invalid syntax.
+    pub fn load_default(&mut self) -> Result<(), Errors> {
+        let config_path = self.workspace_path.join("rubydex.toml");
+
+        match self.load_file(&config_path) {
+            Err(Errors::ConfigNotFound(_)) => Ok(()),
+            other => other,
+        }
+    }
+
+    /// Merges the configuration at `config_path` into this config
+    ///
+    /// # Errors
+    ///
+    /// Returns [`Errors::ConfigNotFound`] if the file does not exist or [`Errors::ConfigError`] if it cannot otherwise
+    /// be read or has invalid syntax.
+    pub fn load_file(&mut self, config_path: &Path) -> Result<(), Errors> {
+        let content = match fs::read_to_string(config_path) {
+            Ok(content) => content,
+            Err(error) if error.kind() == ErrorKind::NotFound => {
+                return Err(Errors::ConfigNotFound(format!(
+                    "Config file `{}` does not exist",
+                    config_path.display()
+                )));
+            }
+            Err(error) => {
+                return Err(Errors::ConfigError(format!(
+                    "Failed to read config file `{}`: {error}",
+                    config_path.display()
+                )));
+            }
+        };
+
+        let parsed = Self::parse(&content).map_err(|error| {
+            Errors::ConfigError(format!("Invalid config file `{}`: {error}", config_path.display()))
+        })?;
+
+        self.excluded_paths.extend(parsed.exclude);
+        Ok(())
+    }
+
+    /// Parses the content into a [`ConfigFile`]
+    fn parse(content: &str) -> Result<ConfigFile, toml::de::Error> {
+        toml::from_str(content)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::path::Path;
+
+    #[test]
+    fn excluded_paths_are_resolved_against_the_workspace_path() {
+        let mut config = Config::new();
+        config.set_workspace_path(PathBuf::from("/workspace"));
+        config.exclude_paths([PathBuf::from("vendor"), PathBuf::from("/absolute/path")]);
+
+        let excluded = config.excluded_paths();
+
+        // Relative entries (including the defaults) are joined with the workspace path.
+        assert!(excluded.contains(Path::new("/workspace/vendor")));
+        assert!(excluded.contains(Path::new("/workspace/.git")));
+        // Absolute entries pass through unchanged.
+        assert!(excluded.contains(Path::new("/absolute/path")));
+    }
+
+    #[test]
+    fn new_seeds_the_default_excluded_directories() {
+        let config = Config::new();
+
+        for default in DEFAULT_EXCLUDED_DIRECTORIES {
+            assert!(
+                config.excluded_paths.contains(Path::new(default)),
+                "expected `{default}` to be excluded by default"
+            );
+        }
+    }
+
+    #[test]
+    fn load_file_merges_excluded_paths_and_leaves_the_workspace_path_untouched() {
+        let dir = tempfile::tempdir().expect("failed to create temp dir");
+        let config_path = dir.path().join("rubydex.toml");
+        fs::write(&config_path, "exclude = [\"vendor\", \"generated\"]\n").unwrap();
+
+        let mut config = Config::new();
+        config.set_workspace_path(PathBuf::from("/workspace"));
+
+        config
+            .load_file(&config_path)
+            .expect("expected the config file to load");
+
+        let excluded = config.excluded_paths();
+        // Entries from the file are merged in and resolved against the workspace path.
+        assert!(excluded.contains(Path::new("/workspace/vendor")));
+        assert!(excluded.contains(Path::new("/workspace/generated")));
+        // Defaults seeded at construction survive the merge.
+        assert!(excluded.contains(Path::new("/workspace/node_modules")));
+        // A config file cannot override the programmatically-set workspace path.
+        assert_eq!(config.workspace_path(), Path::new("/workspace"));
+    }
+
+    #[test]
+    fn load_file_accumulates_exclusions_across_multiple_loads() {
+        let dir = tempfile::tempdir().expect("failed to create temp dir");
+        fs::write(dir.path().join("a.toml"), "exclude = [\"vendor\"]\n").unwrap();
+        fs::write(dir.path().join("b.toml"), "exclude = [\"generated\"]\n").unwrap();
+
+        let mut config = Config::new();
+        config.set_workspace_path(PathBuf::from("/workspace"));
+        config
+            .load_file(&dir.path().join("a.toml"))
+            .expect("expected the first file to load");
+        config
+            .load_file(&dir.path().join("b.toml"))
+            .expect("expected the second file to load");
+
+        let excluded = config.excluded_paths();
+        assert!(excluded.contains(Path::new("/workspace/vendor")));
+        assert!(excluded.contains(Path::new("/workspace/generated")));
+    }
+
+    #[test]
+    fn load_file_errors_when_the_file_is_missing() {
+        let dir = tempfile::tempdir().expect("failed to create temp dir");
+        let mut config = Config::new();
+
+        let error = config
+            .load_file(&dir.path().join("does_not_exist.toml"))
+            .expect_err("an explicitly requested missing file must be an error");
+
+        assert!(
+            matches!(error, Errors::ConfigNotFound(_)),
+            "unexpected error: {error:?}"
+        );
+    }
+
+    #[test]
+    fn load_default_ignores_a_missing_config_file() {
+        let dir = tempfile::tempdir().expect("failed to create temp dir");
+        let mut config = Config::new();
+        config.set_workspace_path(dir.path().to_path_buf());
+
+        config
+            .load_default()
+            .expect("a missing rubydex.toml must not be an error");
+    }
+
+    #[test]
+    fn load_default_loads_an_existing_config_file() {
+        let dir = tempfile::tempdir().expect("failed to create temp dir");
+        fs::write(dir.path().join("rubydex.toml"), "exclude = [\"vendor\"]\n").unwrap();
+
+        let mut config = Config::new();
+        config.set_workspace_path(dir.path().to_path_buf());
+        config.load_default().expect("expected rubydex.toml to load");
+
+        assert!(config.excluded_paths().contains(&dir.path().join("vendor")));
+    }
+
+    #[test]
+    fn load_default_propagates_malformed_config_errors() {
+        let dir = tempfile::tempdir().expect("failed to create temp dir");
+        fs::write(dir.path().join("rubydex.toml"), "exclude = [\n").unwrap();
+
+        let mut config = Config::new();
+        config.set_workspace_path(dir.path().to_path_buf());
+
+        let error = config
+            .load_default()
+            .expect_err("a malformed default config must still be an error");
+
+        assert!(matches!(error, Errors::ConfigError(_)), "unexpected error: {error:?}");
+    }
+
+    #[test]
+    fn parse_defaults_the_excluded_paths_to_empty_when_the_key_is_absent() {
+        let file = Config::parse("").expect("an empty config is valid");
+        assert!(file.exclude.is_empty());
+    }
+
+    #[test]
+    fn parse_rejects_an_exclude_value_of_the_wrong_type() {
+        Config::parse("exclude = \"vendor\"").expect_err("exclude must be an array of strings, not a string");
+    }
+}