rubydex 0.1.0.beta12-aarch64-linux

data/rust/rubydex/src/lib.rs

@@ -0,0 +1,17 @@
+pub mod compile_assertions;
+pub mod diagnostic;
+pub mod errors;
+pub mod indexing;
+pub mod integrity;
+pub mod job_queue;
+pub mod listing;
+pub mod model;
+pub mod offset;
+pub mod position;
+pub mod query;
+pub mod resolution;
+pub mod stats;
+pub mod visualization;
+
+#[cfg(any(test, feature = "test_utils"))]
+pub mod test_utils;

data/rust/rubydex/src/listing.rs

@@ -0,0 +1,371 @@
+use crate::{
+    errors::Errors,
+    job_queue::{Job, JobQueue},
+};
+use crossbeam_channel::{Sender, unbounded};
+use std::{
+    collections::HashSet,
+    fs,
+    hash::BuildHasher,
+    path::{Path, PathBuf},
+    sync::Arc,
+};
+
+pub struct FileDiscoveryJob {
+    path: PathBuf,
+    queue: Arc<JobQueue>,
+    paths_tx: Sender<PathBuf>,
+    errors_tx: Sender<Errors>,
+    excluded_paths: Arc<HashSet<PathBuf>>,
+}
+
+impl FileDiscoveryJob {
+    #[must_use]
+    pub fn new(
+        path: PathBuf,
+        queue: Arc<JobQueue>,
+        paths_tx: Sender<PathBuf>,
+        errors_tx: Sender<Errors>,
+        excluded_paths: Arc<HashSet<PathBuf>>,
+    ) -> Self {
+        Self {
+            path,
+            queue,
+            paths_tx,
+            errors_tx,
+            excluded_paths,
+        }
+    }
+}
+
+impl FileDiscoveryJob {
+    fn handle_file(&self, path: &Path) {
+        if path.extension().is_some_and(|ext| ext == "rb" || ext == "rbs") {
+            self.paths_tx
+                .send(path.to_path_buf())
+                .expect("file receiver dropped before run completion");
+        }
+    }
+
+    fn handle_symlink(&self, path: &PathBuf) {
+        let Ok(canonicalized) = fs::canonicalize(path) else {
+            self.send_error(Errors::FileError(format!(
+                "Failed to canonicalize symlink: `{}`",
+                path.display(),
+            )));
+
+            return;
+        };
+
+        if self.excluded_paths.contains(&canonicalized) {
+            return;
+        }
+
+        self.queue.push(Box::new(FileDiscoveryJob::new(
+            canonicalized,
+            Arc::clone(&self.queue),
+            self.paths_tx.clone(),
+            self.errors_tx.clone(),
+            Arc::clone(&self.excluded_paths),
+        )));
+    }
+
+    fn send_error(&self, error: Errors) {
+        self.errors_tx
+            .send(error)
+            .expect("error receiver dropped before run completion");
+    }
+}
+
+impl Job for FileDiscoveryJob {
+    fn run(&self) {
+        if self.path.is_dir() {
+            let Ok(read_dir) = self.path.read_dir() else {
+                self.send_error(Errors::FileError(format!(
+                    "Failed to read directory `{}`",
+                    self.path.display(),
+                )));
+
+                return;
+            };
+
+            for result in read_dir {
+                let Ok(entry) = result else {
+                    self.send_error(Errors::FileError(format!(
+                        "Failed to read directory `{}`: {result:?}",
+                        self.path.display(),
+                    )));
+
+                    continue;
+                };
+
+                let kind = entry.file_type().unwrap();
+
+                if kind.is_dir() {
+                    if self.excluded_paths.contains(&entry.path()) {
+                        continue;
+                    }
+
+                    self.queue.push(Box::new(FileDiscoveryJob::new(
+                        entry.path(),
+                        Arc::clone(&self.queue),
+                        self.paths_tx.clone(),
+                        self.errors_tx.clone(),
+                        Arc::clone(&self.excluded_paths),
+                    )));
+                } else if kind.is_file() {
+                    self.handle_file(&entry.path());
+                } else if kind.is_symlink() {
+                    self.handle_symlink(&entry.path());
+                } else {
+                    self.send_error(Errors::FileError(format!(
+                        "Path `{}` is not a file or directory",
+                        entry.path().display()
+                    )));
+                }
+            }
+        } else if self.path.is_file() {
+            self.handle_file(&self.path);
+        } else if self.path.is_symlink() {
+            self.handle_symlink(&self.path);
+        } else {
+            self.send_error(Errors::FileError(format!(
+                "Path `{}` is not a file or directory",
+                self.path.display()
+            )));
+        }
+    }
+}
+
+/// Recursively collects all Ruby files for the given workspace and dependencies, returning a vector of document instances
+///
+/// # Errors
+///
+/// Returns a `MultipleErrors` if any of the paths do not exist
+///
+/// # Panics
+///
+/// Panics if the errors receiver is dropped before the run completion
+#[must_use]
+pub fn collect_file_paths<S: BuildHasher>(
+    paths: Vec<String>,
+    excluded: &HashSet<PathBuf, S>,
+) -> (Vec<PathBuf>, Vec<Errors>) {
+    let queue = Arc::new(JobQueue::new());
+    let (files_tx, files_rx) = unbounded();
+    let (errors_tx, errors_rx) = unbounded();
+
+    // Canonicalize the excluded paths since they may be symlinks
+    let excluded: Arc<HashSet<PathBuf>> = Arc::new(excluded.iter().filter_map(|p| fs::canonicalize(p).ok()).collect());
+
+    for path in paths {
+        let Ok(canonicalized) = fs::canonicalize(&path) else {
+            errors_tx
+                .send(Errors::FileError(format!("Path `{path}` does not exist")))
+                .expect("errors receiver dropped before run completion");
+
+            continue;
+        };
+
+        if excluded.contains(&canonicalized) {
+            continue;
+        }
+
+        queue.push(Box::new(FileDiscoveryJob::new(
+            canonicalized,
+            Arc::clone(&queue),
+            files_tx.clone(),
+            errors_tx.clone(),
+            Arc::clone(&excluded),
+        )));
+    }
+
+    JobQueue::run(&queue);
+
+    drop(files_tx);
+    drop(errors_tx);
+
+    (files_rx.iter().collect(), errors_rx.iter().collect())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::test_utils::Context;
+
+    fn collect_document_paths(context: &Context, paths: &[&str]) -> (Vec<String>, Vec<Errors>) {
+        collect_document_paths_with_exclusions(context, paths, &HashSet::new())
+    }
+
+    fn collect_document_paths_with_exclusions(
+        context: &Context,
+        paths: &[&str],
+        excluded: &HashSet<PathBuf>,
+    ) -> (Vec<String>, Vec<Errors>) {
+        let (files, errors) = collect_file_paths(
+            paths
+                .iter()
+                .map(|p| context.absolute_path_to(p).to_string_lossy().into_owned())
+                .collect(),
+            excluded,
+        );
+
+        let mut files: Vec<String> = files
+            .iter()
+            .map(|path| context.relative_path_to(path).to_string_lossy().into_owned())
+            .collect();
+
+        files.sort();
+
+        (files, errors)
+    }
+
+    #[test]
+    fn collect_all_documents() {
+        let context = Context::new();
+        let baz = PathBuf::from("bar").join("baz.rb");
+        let qux = PathBuf::from("bar").join("qux.rb");
+        let bar = PathBuf::from("foo").join("bar.rb");
+        context.touch(&baz);
+        context.touch(&qux);
+        context.touch(&bar);
+
+        let (files, errors) = collect_document_paths(&context, &["foo", "bar"]);
+
+        assert!(errors.is_empty());
+
+        assert_eq!(
+            files,
+            [
+                baz.to_str().unwrap().to_string(),
+                qux.to_str().unwrap().to_string(),
+                bar.to_str().unwrap().to_string()
+            ]
+        );
+    }
+
+    #[test]
+    fn collect_some_documents_based_on_paths() {
+        let context = Context::new();
+        let baz = PathBuf::from("bar").join("baz.rb");
+        let qux = PathBuf::from("bar").join("qux.rb");
+        let bar = PathBuf::from("foo").join("bar.rb");
+
+        context.touch(&baz);
+        context.touch(&qux);
+        context.touch(&bar);
+
+        let (files, errors) = collect_document_paths(&context, &["bar"]);
+
+        assert!(errors.is_empty());
+
+        assert_eq!(
+            files,
+            [baz.to_str().unwrap().to_string(), qux.to_str().unwrap().to_string()]
+        );
+    }
+
+    #[test]
+    fn collect_rbs_files() {
+        let context = Context::new();
+        let ruby_file = PathBuf::from("lib").join("foo.rb");
+        let rbs_file = PathBuf::from("sig").join("foo.rbs");
+        let txt_file = PathBuf::from("lib").join("notes.txt");
+        context.touch(&ruby_file);
+        context.touch(&rbs_file);
+        context.touch(&txt_file);
+
+        let (files, errors) = collect_document_paths(&context, &["lib", "sig"]);
+
+        assert!(errors.is_empty());
+
+        assert_eq!(
+            [
+                ruby_file.to_str().unwrap().to_string(),
+                rbs_file.to_str().unwrap().to_string(),
+            ],
+            files.as_slice()
+        );
+    }
+
+    #[test]
+    fn collect_non_existing_paths() {
+        let context = Context::new();
+
+        let (files, errors) = collect_file_paths(
+            vec![
+                context
+                    .absolute_path_to("non_existing_path")
+                    .to_string_lossy()
+                    .into_owned(),
+            ],
+            &HashSet::new(),
+        );
+
+        assert!(files.is_empty());
+
+        assert_eq!(
+            errors,
+            [Errors::FileError(format!(
+                "Path `{}` does not exist",
+                context.absolute_path_to("non_existing_path").display()
+            ))]
+        );
+    }
+
+    #[test]
+    fn collect_files_excludes_directories() {
+        let context = Context::new();
+        let included = PathBuf::from("included").join("foo.rb");
+        let excluded_file = PathBuf::from("excluded").join("bar.rb");
+        context.touch(&included);
+        context.touch(&excluded_file);
+
+        let mut excluded = HashSet::new();
+        excluded.insert(context.absolute_path_to("excluded"));
+
+        let (files, errors) = collect_document_paths_with_exclusions(&context, &["included", "excluded"], &excluded);
+
+        assert!(errors.is_empty());
+        assert_eq!(files, [included.to_str().unwrap().to_string()]);
+    }
+
+    #[test]
+    fn collect_files_excludes_nested_directories() {
+        let context = Context::new();
+        let kept = PathBuf::from("root").join("kept.rb");
+        let nested = PathBuf::from("root").join("skip").join("nested.rb");
+        context.touch(&kept);
+        context.touch(&nested);
+
+        let mut excluded = HashSet::new();
+        excluded.insert(context.absolute_path_to("root/skip"));
+
+        let (files, errors) = collect_document_paths_with_exclusions(&context, &["root"], &excluded);
+
+        assert!(errors.is_empty());
+        assert_eq!(files, [kept.to_str().unwrap().to_string()]);
+    }
+
+    #[cfg(unix)]
+    #[test]
+    fn collect_files_excludes_symlinked_directories() {
+        let context = Context::new();
+        let included = PathBuf::from("included").join("foo.rb");
+        let excluded_file = PathBuf::from("real_dir").join("bar.rb");
+        context.touch(&included);
+        context.touch(&excluded_file);
+
+        // Create a symlink: link -> real_dir
+        std::os::unix::fs::symlink(context.absolute_path_to("real_dir"), context.absolute_path_to("link")).unwrap();
+
+        // Excluding the real directory while requesting to index the symlink should properly exclude the link
+        let mut excluded = HashSet::new();
+        excluded.insert(context.absolute_path_to("real_dir"));
+
+        let (files, errors) = collect_document_paths_with_exclusions(&context, &["included", "link"], &excluded);
+
+        assert!(errors.is_empty());
+        assert_eq!(files, [included.to_str().unwrap().to_string()]);
+    }
+}

data/rust/rubydex/src/main.rs

@@ -0,0 +1,160 @@
+use clap::{Parser, ValueEnum};
+use std::{collections::HashSet, mem};
+
+use rubydex::{
+    indexing, integrity, listing,
+    model::graph::Graph,
+    resolution::Resolver,
+    stats::{
+        memory::MemoryStats,
+        timer::{Timer, time_it},
+    },
+    visualization::dot,
+};
+
+#[derive(Parser, Debug)]
+#[command(name = "rubydex_cli", about = "A Static Analysis Toolkit for Ruby", version)]
+#[allow(clippy::struct_excessive_bools)]
+struct Args {
+    #[arg(value_name = "PATHS", default_value = ".")]
+    paths: Vec<String>,
+
+    #[arg(long = "stop-after", help = "Stop after the given stage")]
+    stop_after: Option<StopAfter>,
+
+    #[arg(long = "visualize")]
+    visualize: bool,
+
+    #[arg(long = "stats", help = "Show detailed performance statistics")]
+    stats: bool,
+
+    #[arg(long = "check-integrity", help = "Check the integrity of the graph after resolution")]
+    check_integrity: bool,
+
+    #[arg(
+        long = "report-orphans",
+        value_name = "PATH",
+        num_args = 0..=1,
+        require_equals = true,
+        default_missing_value = "/tmp/rubydex-orphan-report.txt",
+        help = "Write orphan definitions report to specified file"
+    )]
+    report_orphans: Option<String>,
+}
+
+#[derive(Debug, Clone, ValueEnum)]
+enum StopAfter {
+    Listing,
+    Indexing,
+    Resolution,
+}
+
+fn exit(print_stats: bool) {
+    if print_stats {
+        Timer::print_breakdown();
+        MemoryStats::print_memory_usage();
+    }
+
+    std::process::exit(0);
+}
+
+fn main() {
+    let args = Args::parse();
+
+    if args.stats {
+        Timer::set_global_timer(Timer::new());
+    }
+
+    // Listing
+
+    let (file_paths, errors) = time_it!(listing, { listing::collect_file_paths(args.paths, &HashSet::new()) });
+
+    for error in errors {
+        eprintln!("{error}");
+    }
+
+    if let Some(StopAfter::Listing) = args.stop_after {
+        return exit(args.stats);
+    }
+
+    // Indexing
+
+    let mut graph = Graph::new();
+    let errors = time_it!(indexing, { indexing::index_files(&mut graph, file_paths) });
+
+    for error in errors {
+        eprintln!("{error}");
+    }
+
+    if let Some(StopAfter::Indexing) = args.stop_after {
+        return exit(args.stats);
+    }
+
+    // Resolution
+
+    time_it!(resolution, {
+        let mut resolver = Resolver::new(&mut graph);
+        resolver.resolve();
+    });
+
+    if let Some(StopAfter::Resolution) = args.stop_after {
+        return exit(args.stats);
+    }
+
+    // Integrity check
+    if args.check_integrity {
+        let errors = time_it!(integrity_check, { integrity::check_integrity(&graph) });
+
+        if errors.is_empty() {
+            println!("Integrity check passed: no issues found");
+        } else {
+            eprintln!("Integrity check found {} issue(s):", errors.len());
+
+            for error in &errors {
+                eprintln!("  - {error}");
+            }
+
+            std::process::exit(1);
+        }
+    }
+
+    // Querying
+
+    if args.stats {
+        time_it!(querying, {
+            graph.print_query_statistics();
+        });
+    }
+
+    if args.stats {
+        Timer::print_breakdown();
+        MemoryStats::print_memory_usage();
+    }
+
+    // Orphan report
+    if let Some(ref path) = args.report_orphans {
+        match std::fs::File::create(path) {
+            Ok(mut file) => {
+                if let Err(e) = graph.write_orphan_report(&mut file) {
+                    eprintln!("Failed to write orphan report: {e}");
+                } else {
+                    println!("Orphan report written to {path}");
+                }
+            }
+            Err(e) => eprintln!("Failed to create orphan report file: {e}"),
+        }
+    }
+
+    // Generate visualization or print statistics
+    if args.visualize {
+        println!("{}", dot::generate(&graph));
+    } else {
+        println!("Indexed {} files", graph.documents().len());
+        println!("Found {} names", graph.declarations().len());
+        println!("Found {} definitions", graph.definitions().len());
+        println!("Found {} URIs", graph.documents().len());
+    }
+
+    // Forget the graph so we don't have to wait for deallocation and let the system reclaim the memory at exit
+    mem::forget(graph);
+}

data/rust/rubydex/src/model/built_in.rs

@@ -0,0 +1,83 @@
+use std::sync::LazyLock;
+
+use url::Url;
+
+use crate::{
+    indexing::{self, LanguageId},
+    model::{
+        declaration::{ClassDeclaration, Declaration, Namespace},
+        graph::Graph,
+        ids::DeclarationId,
+    },
+};
+
+pub static KERNEL_ID: LazyLock<DeclarationId> = LazyLock::new(|| DeclarationId::from("Kernel"));
+pub static BASIC_OBJECT_ID: LazyLock<DeclarationId> = LazyLock::new(|| DeclarationId::from("BasicObject"));
+pub static OBJECT_ID: LazyLock<DeclarationId> = LazyLock::new(|| DeclarationId::from("Object"));
+pub static MODULE_ID: LazyLock<DeclarationId> = LazyLock::new(|| DeclarationId::from("Module"));
+pub static CLASS_ID: LazyLock<DeclarationId> = LazyLock::new(|| DeclarationId::from("Class"));
+
+/// Adds core classes and modules data to the graph so that resolution can provide correct results even when not
+/// indexing the complete RBS core definitions
+///
+/// # Panics
+///
+/// Will panic if the built-in URI is invalid
+pub fn add_built_in_data(graph: &mut Graph) {
+    // We need definitions to ensure that ancestor linearization happens naturally through the algorithm. Trying to set
+    // ancestors directly on declarations doesn't work because the algorithm erases the ancestors and there are no
+    // definitions to inform it of the superclasses and mixins.
+    let uri = Url::parse("rubydex:built-in").unwrap();
+    let source = r"
+      class BasicObject
+      end
+
+      module Kernel
+      end
+
+      class Object < BasicObject
+        include Kernel
+      end
+
+      class Module < Object
+      end
+
+      class Class < Module
+      end
+    ";
+    indexing::index_source(graph, uri.as_ref(), source, &LanguageId::Rbs);
+
+    // Creating declarations eagerly is still necessary because we need to associate correct ownership data no matter in
+    // what order we discover classes and modules
+    let declarations = graph.declarations_mut();
+
+    // Built-in declarations that always exist in the Ruby object model
+    declarations.insert(
+        *BASIC_OBJECT_ID,
+        Declaration::Namespace(Namespace::Class(Box::new(ClassDeclaration::new(
+            "BasicObject".to_string(),
+            *OBJECT_ID,
+        )))),
+    );
+    declarations.insert(
+        *OBJECT_ID,
+        Declaration::Namespace(Namespace::Class(Box::new(ClassDeclaration::new(
+            "Object".to_string(),
+            *OBJECT_ID,
+        )))),
+    );
+    declarations.insert(
+        *MODULE_ID,
+        Declaration::Namespace(Namespace::Class(Box::new(ClassDeclaration::new(
+            "Module".to_string(),
+            *OBJECT_ID,
+        )))),
+    );
+    declarations.insert(
+        *CLASS_ID,
+        Declaration::Namespace(Namespace::Class(Box::new(ClassDeclaration::new(
+            "Class".to_string(),
+            *OBJECT_ID,
+        )))),
+    );
+}

data/rust/rubydex/src/model/comment.rs

@@ -0,0 +1,24 @@
+use crate::offset::Offset;
+
+#[derive(Debug, Clone)]
+pub struct Comment {
+    offset: Offset,
+    string: String,
+}
+
+impl Comment {
+    #[must_use]
+    pub fn new(offset: Offset, string: String) -> Self {
+        Self { offset, string }
+    }
+
+    #[must_use]
+    pub fn offset(&self) -> &Offset {
+        &self.offset
+    }
+
+    #[must_use]
+    pub fn string(&self) -> &String {
+        &self.string
+    }
+}