rubydex 0.2.6-aarch64-linux → 0.2.8-aarch64-linux

data/rust/rubydex/benches/graph_memory.rs

@@ -0,0 +1,46 @@
+#[cfg(all(feature = "jemalloc", not(target_os = "windows")))]
+mod imp {
+    use std::collections::HashSet;
+
+    use rubydex::{
+        indexing::{self, IndexerBackend},
+        listing,
+        model::graph::Graph,
+        resolution::Resolver,
+    };
+    use tikv_jemalloc_ctl::{epoch, stats};
+
+    /// Advance the jemalloc epoch (stats are cached between epochs) and read the
+    /// number of bytes currently allocated by the application.
+    fn allocated_bytes() -> usize {
+        epoch::advance().expect("failed to advance jemalloc epoch");
+        stats::allocated::read().expect("failed to read stats.allocated (is the `stats` feature on?)")
+    }
+
+    pub fn run() {
+        let paths: Vec<String> = std::env::args().skip(1).collect();
+        let paths = if paths.is_empty() { vec![".".to_string()] } else { paths };
+        let (file_paths, _) = listing::collect_file_paths(paths, &HashSet::new());
+
+        let mut graph = Graph::new();
+        let _ = indexing::index_files(&mut graph, file_paths, IndexerBackend::RubyIndexer);
+        Resolver::new(&mut graph).resolve();
+
+        // Compare the total memory used in the allocator before and after dropping the graph
+        let before_drop = allocated_bytes();
+        drop(graph);
+        let after_drop = allocated_bytes();
+
+        let total_graph_memory = before_drop.saturating_sub(after_drop);
+
+        #[allow(clippy::cast_precision_loss)]
+        let mega_bytes = total_graph_memory as f64 / 1024.0 / 1024.0;
+
+        println!("Total graph memory: {mega_bytes:.2} MB");
+    }
+}
+
+fn main() {
+    #[cfg(all(feature = "jemalloc", not(target_os = "windows")))]
+    imp::run();
+}

data/rust/rubydex/src/config.rs

@@ -0,0 +1,275 @@
+use crate::assert_mem_size;
+use crate::errors::Errors;
+use serde::Deserialize;
+use std::collections::HashSet;
+use std::fs;
+use std::io::ErrorKind;
+use std::path::{Path, PathBuf};
+
+pub const DEFAULT_EXCLUDED_DIRECTORIES: &[&str] = &[
+    ".bundle",
+    ".claude",
+    ".git",
+    ".github",
+    ".ruby-lsp",
+    ".vscode",
+    "log",
+    "node_modules",
+    "tmp",
+];
+
+/// Configuration coming from a config file
+#[derive(Debug, Default, Deserialize)]
+struct ConfigFile {
+    /// Paths to exclude from file discovery during indexing.
+    #[serde(default)]
+    exclude: Vec<PathBuf>,
+}
+
+/// Project configuration
+#[derive(Debug)]
+pub struct Config {
+    /// Path to the workspace being analyzed
+    workspace_path: Box<Path>,
+    /// Paths to exclude from file discovery during indexing.
+    excluded_paths: HashSet<PathBuf>,
+}
+assert_mem_size!(Config, 64);
+
+impl Default for Config {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Config {
+    /// Creates a configuration whose workspace path defaults to the current working directory.
+    #[must_use]
+    pub fn new() -> Self {
+        Self {
+            workspace_path: std::env::current_dir()
+                .unwrap_or_else(|_| PathBuf::from("."))
+                .into_boxed_path(),
+            excluded_paths: DEFAULT_EXCLUDED_DIRECTORIES.iter().map(PathBuf::from).collect(),
+        }
+    }
+
+    /// Returns the root directory of the workspace being analyzed.
+    #[must_use]
+    pub fn workspace_path(&self) -> &Path {
+        &self.workspace_path
+    }
+
+    /// Sets the root directory of the workspace being analyzed.
+    pub fn set_workspace_path(&mut self, workspace_path: PathBuf) {
+        self.workspace_path = workspace_path.into_boxed_path();
+    }
+
+    /// Adds paths to exclude from file discovery during indexing. Excluded directories will be skipped entirely during
+    /// directory traversal.
+    pub fn exclude_paths(&mut self, paths: impl IntoIterator<Item = PathBuf>) {
+        self.excluded_paths.extend(paths);
+    }
+
+    /// Returns the set of paths excluded from file discovery
+    #[must_use]
+    pub fn excluded_paths(&self) -> HashSet<PathBuf> {
+        self.excluded_paths
+            .iter()
+            .map(|path| self.workspace_path.join(path))
+            .collect()
+    }
+
+    /// Merges the default `rubydex.toml` configuration file from the workspace root into this config, if present.
+    ///
+    /// The default config file is optional, so a missing `rubydex.toml` is silently ignored. Any other failure (an
+    /// unreadable or malformed file) is still reported.
+    ///
+    /// # Errors
+    ///
+    /// Will error if the config file exists but cannot be read or has invalid syntax.
+    pub fn load_default(&mut self) -> Result<(), Errors> {
+        let config_path = self.workspace_path.join("rubydex.toml");
+
+        match self.load_file(&config_path) {
+            Err(Errors::ConfigNotFound(_)) => Ok(()),
+            other => other,
+        }
+    }
+
+    /// Merges the configuration at `config_path` into this config
+    ///
+    /// # Errors
+    ///
+    /// Returns [`Errors::ConfigNotFound`] if the file does not exist or [`Errors::ConfigError`] if it cannot otherwise
+    /// be read or has invalid syntax.
+    pub fn load_file(&mut self, config_path: &Path) -> Result<(), Errors> {
+        let content = match fs::read_to_string(config_path) {
+            Ok(content) => content,
+            Err(error) if error.kind() == ErrorKind::NotFound => {
+                return Err(Errors::ConfigNotFound(format!(
+                    "Config file `{}` does not exist",
+                    config_path.display()
+                )));
+            }
+            Err(error) => {
+                return Err(Errors::ConfigError(format!(
+                    "Failed to read config file `{}`: {error}",
+                    config_path.display()
+                )));
+            }
+        };
+
+        let parsed = Self::parse(&content).map_err(|error| {
+            Errors::ConfigError(format!("Invalid config file `{}`: {error}", config_path.display()))
+        })?;
+
+        self.excluded_paths.extend(parsed.exclude);
+        Ok(())
+    }
+
+    /// Parses the content into a [`ConfigFile`]
+    fn parse(content: &str) -> Result<ConfigFile, toml::de::Error> {
+        toml::from_str(content)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::path::Path;
+
+    #[test]
+    fn excluded_paths_are_resolved_against_the_workspace_path() {
+        let mut config = Config::new();
+        config.set_workspace_path(PathBuf::from("/workspace"));
+        config.exclude_paths([PathBuf::from("vendor"), PathBuf::from("/absolute/path")]);
+
+        let excluded = config.excluded_paths();
+
+        // Relative entries (including the defaults) are joined with the workspace path.
+        assert!(excluded.contains(Path::new("/workspace/vendor")));
+        assert!(excluded.contains(Path::new("/workspace/.git")));
+        // Absolute entries pass through unchanged.
+        assert!(excluded.contains(Path::new("/absolute/path")));
+    }
+
+    #[test]
+    fn new_seeds_the_default_excluded_directories() {
+        let config = Config::new();
+
+        for default in DEFAULT_EXCLUDED_DIRECTORIES {
+            assert!(
+                config.excluded_paths.contains(Path::new(default)),
+                "expected `{default}` to be excluded by default"
+            );
+        }
+    }
+
+    #[test]
+    fn load_file_merges_excluded_paths_and_leaves_the_workspace_path_untouched() {
+        let dir = tempfile::tempdir().expect("failed to create temp dir");
+        let config_path = dir.path().join("rubydex.toml");
+        fs::write(&config_path, "exclude = [\"vendor\", \"generated\"]\n").unwrap();
+
+        let mut config = Config::new();
+        config.set_workspace_path(PathBuf::from("/workspace"));
+
+        config
+            .load_file(&config_path)
+            .expect("expected the config file to load");
+
+        let excluded = config.excluded_paths();
+        // Entries from the file are merged in and resolved against the workspace path.
+        assert!(excluded.contains(Path::new("/workspace/vendor")));
+        assert!(excluded.contains(Path::new("/workspace/generated")));
+        // Defaults seeded at construction survive the merge.
+        assert!(excluded.contains(Path::new("/workspace/node_modules")));
+        // A config file cannot override the programmatically-set workspace path.
+        assert_eq!(config.workspace_path(), Path::new("/workspace"));
+    }
+
+    #[test]
+    fn load_file_accumulates_exclusions_across_multiple_loads() {
+        let dir = tempfile::tempdir().expect("failed to create temp dir");
+        fs::write(dir.path().join("a.toml"), "exclude = [\"vendor\"]\n").unwrap();
+        fs::write(dir.path().join("b.toml"), "exclude = [\"generated\"]\n").unwrap();
+
+        let mut config = Config::new();
+        config.set_workspace_path(PathBuf::from("/workspace"));
+        config
+            .load_file(&dir.path().join("a.toml"))
+            .expect("expected the first file to load");
+        config
+            .load_file(&dir.path().join("b.toml"))
+            .expect("expected the second file to load");
+
+        let excluded = config.excluded_paths();
+        assert!(excluded.contains(Path::new("/workspace/vendor")));
+        assert!(excluded.contains(Path::new("/workspace/generated")));
+    }
+
+    #[test]
+    fn load_file_errors_when_the_file_is_missing() {
+        let dir = tempfile::tempdir().expect("failed to create temp dir");
+        let mut config = Config::new();
+
+        let error = config
+            .load_file(&dir.path().join("does_not_exist.toml"))
+            .expect_err("an explicitly requested missing file must be an error");
+
+        assert!(
+            matches!(error, Errors::ConfigNotFound(_)),
+            "unexpected error: {error:?}"
+        );
+    }
+
+    #[test]
+    fn load_default_ignores_a_missing_config_file() {
+        let dir = tempfile::tempdir().expect("failed to create temp dir");
+        let mut config = Config::new();
+        config.set_workspace_path(dir.path().to_path_buf());
+
+        config
+            .load_default()
+            .expect("a missing rubydex.toml must not be an error");
+    }
+
+    #[test]
+    fn load_default_loads_an_existing_config_file() {
+        let dir = tempfile::tempdir().expect("failed to create temp dir");
+        fs::write(dir.path().join("rubydex.toml"), "exclude = [\"vendor\"]\n").unwrap();
+
+        let mut config = Config::new();
+        config.set_workspace_path(dir.path().to_path_buf());
+        config.load_default().expect("expected rubydex.toml to load");
+
+        assert!(config.excluded_paths().contains(&dir.path().join("vendor")));
+    }
+
+    #[test]
+    fn load_default_propagates_malformed_config_errors() {
+        let dir = tempfile::tempdir().expect("failed to create temp dir");
+        fs::write(dir.path().join("rubydex.toml"), "exclude = [\n").unwrap();
+
+        let mut config = Config::new();
+        config.set_workspace_path(dir.path().to_path_buf());
+
+        let error = config
+            .load_default()
+            .expect_err("a malformed default config must still be an error");
+
+        assert!(matches!(error, Errors::ConfigError(_)), "unexpected error: {error:?}");
+    }
+
+    #[test]
+    fn parse_defaults_the_excluded_paths_to_empty_when_the_key_is_absent() {
+        let file = Config::parse("").expect("an empty config is valid");
+        assert!(file.exclude.is_empty());
+    }
+
+    #[test]
+    fn parse_rejects_an_exclude_value_of_the_wrong_type() {
+        Config::parse("exclude = \"vendor\"").expect_err("exclude must be an array of strings, not a string");
+    }
+}

data/rust/rubydex/src/errors.rs

@@ -25,4 +25,6 @@ macro_rules! errors {
 
 errors!(
     FileError;
+    ConfigError;
+    ConfigNotFound;
 );

data/rust/rubydex/src/lib.rs

@@ -1,4 +1,11 @@
+// Setting the global allocator needs to happen during linking and consumers may not always want to change the global
+// allocator. We gate the usage of jemalloc with a cargo feature, so that consumers can decide if they want to use it
+#[cfg(all(feature = "jemalloc", not(target_os = "windows")))]
+#[global_allocator]
+static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
+
 pub mod compile_assertions;
+pub mod config;
 pub mod diagnostic;
 pub mod dot;
 pub mod errors;

data/rust/rubydex/src/main.rs

@@ -1,9 +1,13 @@
 use clap::{Parser, ValueEnum};
-use std::{collections::HashSet, mem};
+use std::{
+    fs, mem,
+    path::{Path, PathBuf},
+    time::{Duration, Instant},
+};
 
 use rubydex::{
     dot,
-    indexing::{self, IndexerBackend},
+    indexing::{self, IndexerBackend, LanguageId, build_local_graph},
     integrity, listing,
     model::graph::Graph,
     resolution::Resolver,
@@ -12,12 +16,17 @@ use rubydex::{
         timer::{Timer, time_it},
     },
 };
+use url::Url;
 
 #[derive(Parser, Debug)]
 #[command(name = "rubydex_cli", about = "A Static Analysis Toolkit for Ruby", version)]
 #[allow(clippy::struct_excessive_bools)]
 struct Args {
-    #[arg(value_name = "PATHS", default_value = ".")]
+    #[arg(
+        value_name = "PATHS",
+        default_value = ".",
+        help = "Path(s) to index. If the first path is a directory, it is used as the workspace root for rubydex.toml"
+    )]
     paths: Vec<String>,
 
     #[arg(long = "stop-after", help = "Stop after the given stage")]
@@ -52,6 +61,21 @@ struct Args {
         help = "Write orphan definitions report to specified file"
     )]
     report_orphans: Option<String>,
+
+    #[arg(
+        long = "incremental_cycle",
+        value_name = "N",
+        help = "After the initial build, run N incremental resolution cycles and report their timings"
+    )]
+    incremental_cycle: Option<usize>,
+
+    #[arg(
+        long = "incremental_files",
+        value_name = "N",
+        default_value_t = 1,
+        help = "Number of files to re-index per incremental cycle"
+    )]
+    incremental_files: usize,
 }
 
 #[derive(Debug, Clone, ValueEnum)]
@@ -85,6 +109,11 @@ fn exit(print_stats: bool) {
     std::process::exit(0);
 }
 
+fn workspace_path_for(paths: &[String]) -> Option<PathBuf> {
+    let first_path = paths.first()?;
+    fs::canonicalize(first_path).ok().filter(|path| path.is_dir())
+}
+
 fn main() {
     let args = Args::parse();
 
@@ -92,9 +121,21 @@ fn main() {
         Timer::set_global_timer(Timer::new());
     }
 
+    let mut graph = Graph::new();
+
+    if let Some(workspace_path) = workspace_path_for(&args.paths) {
+        graph.set_workspace_path(workspace_path);
+        if let Err(error) = graph.load_config(None) {
+            eprintln!("{error}");
+            std::process::exit(1);
+        }
+    }
+
     // Listing
 
-    let (file_paths, errors) = time_it!(listing, { listing::collect_file_paths(args.paths, &HashSet::new()) });
+    let (file_paths, errors) = time_it!(listing, {
+        listing::collect_file_paths(args.paths, &graph.excluded_paths())
+    });
 
     for error in errors {
         eprintln!("{error}");
@@ -106,8 +147,16 @@ fn main() {
 
     // Indexing
 
-    let mut graph = Graph::new();
     let backend = IndexerBackend::from(&args.indexer);
+
+    // The incremental benchmark re-indexes files after the initial build, so keep a copy of the
+    // paths before `index_files` consumes them.
+    let incremental_paths = if args.incremental_cycle.is_some() {
+        file_paths.clone()
+    } else {
+        Vec::new()
+    };
+
     let errors = time_it!(indexing, { indexing::index_files(&mut graph, file_paths, backend) });
 
     for error in errors {
@@ -125,6 +174,12 @@ fn main() {
         resolver.resolve();
     });
 
+    // Incremental resolution benchmark. Runs before the stop-after check so it can be combined with
+    // `--stop-after=resolution`.
+    if let Some(cycles) = args.incremental_cycle {
+        run_incremental_resolution(&mut graph, &incremental_paths, cycles, args.incremental_files, backend);
+    }
+
     if let Some(StopAfter::Resolution) = args.stop_after {
         return exit(args.stats);
     }
@@ -186,3 +241,91 @@ fn main() {
     // Forget the graph so we don't have to wait for deallocation and let the system reclaim the memory at exit
     mem::forget(graph);
 }
+
+/// Simulates incremental editing to measure incremental resolution cost. For each cycle it
+/// re-indexes a rotating window of `files_per_cycle` files (parsing plus the same invalidation the
+/// LSP performs on save) and then re-runs resolution over the resulting pending work, reporting
+/// per-cycle and aggregate `resolve()` timings.
+///
+/// With `--stats`, the `compute_descendants` breakdown printed in the timing summary reflects the
+/// last incremental cycle, since it is recorded on every `resolve()`.
+fn run_incremental_resolution(
+    graph: &mut Graph,
+    paths: &[PathBuf],
+    cycles: usize,
+    files_per_cycle: usize,
+    backend: IndexerBackend,
+) {
+    if paths.is_empty() || cycles == 0 || files_per_cycle == 0 {
+        eprintln!("Skipping incremental resolution: nothing to re-index");
+        return;
+    }
+
+    let files_per_cycle = files_per_cycle.min(paths.len());
+
+    let mut reindex_total = Duration::ZERO;
+    let mut resolve_total = Duration::ZERO;
+    let mut resolve_min = Duration::MAX;
+    let mut resolve_max = Duration::ZERO;
+
+    println!();
+    println!("Incremental resolution ({cycles} cycle(s), {files_per_cycle} file(s)/cycle)");
+    println!("  Scenario: no-op reindex; files are indexed again without changing their contents.");
+    println!("  This should be the fastest incremental resolution path.");
+    println!("  Add scenario-based benchmarks before optimizing incremental resolution.");
+
+    for cycle in 0..cycles {
+        // Re-index the window of files for this cycle. Rotating the window across cycles samples
+        // different parts of the codebase rather than measuring the same delta repeatedly.
+        let reindex_start = Instant::now();
+        for i in 0..files_per_cycle {
+            let path = &paths[(cycle * files_per_cycle + i) % paths.len()];
+            reindex_file(graph, path, backend);
+        }
+        reindex_total += reindex_start.elapsed();
+
+        let resolve_start = Instant::now();
+        Resolver::new(graph).resolve();
+        let elapsed = resolve_start.elapsed();
+
+        resolve_total += elapsed;
+        resolve_min = resolve_min.min(elapsed);
+        resolve_max = resolve_max.max(elapsed);
+
+        println!(
+            "  cycle {:>3}: resolve {:9.3}ms",
+            cycle + 1,
+            elapsed.as_secs_f64() * 1000.0
+        );
+    }
+
+    let avg = resolve_total / u32::try_from(cycles).expect("cycle count fits in u32");
+
+    println!(
+        "  resolve total {:.3}ms  avg {:.3}ms  min {:.3}ms  max {:.3}ms",
+        resolve_total.as_secs_f64() * 1000.0,
+        avg.as_secs_f64() * 1000.0,
+        resolve_min.as_secs_f64() * 1000.0,
+        resolve_max.as_secs_f64() * 1000.0,
+    );
+    println!(
+        "  reindex+invalidate total {:.3}ms (parse + document merge, excluded from resolve above)",
+        reindex_total.as_secs_f64() * 1000.0,
+    );
+}
+
+/// Re-indexes a single file into the graph, running the same invalidation the LSP performs on save.
+fn reindex_file(graph: &mut Graph, path: &Path, backend: IndexerBackend) {
+    let Ok(source) = fs::read_to_string(path) else {
+        eprintln!("Failed to read file `{}`", path.display());
+        return;
+    };
+    let Ok(url) = Url::from_file_path(path) else {
+        eprintln!("Couldn't build URI from path `{}`", path.display());
+        return;
+    };
+
+    let language = path.extension().map_or(LanguageId::Ruby, LanguageId::from);
+    let local_graph = build_local_graph(url.to_string(), &source, &language, backend);
+    graph.consume_document_changes(local_graph);
+}