rubydex 0.1.0.beta1-x86_64-linux → 0.1.0.beta2-x86_64-linux

data/rust/rubydex/src/indexing.rs

@@ -0,0 +1,128 @@
+use crate::{
+    errors::Errors,
+    indexing::{local_graph::LocalGraph, ruby_indexer::RubyIndexer},
+    job_queue::{Job, JobQueue},
+    model::graph::Graph,
+};
+use crossbeam_channel::{Sender, unbounded};
+use std::{fs, path::PathBuf, sync::Arc};
+use url::Url;
+
+pub mod local_graph;
+pub mod ruby_indexer;
+
+/// Job that indexes a single Ruby file
+pub struct IndexingRubyFileJob {
+    path: PathBuf,
+    local_graph_tx: Sender<LocalGraph>,
+    errors_tx: Sender<Errors>,
+}
+
+impl IndexingRubyFileJob {
+    #[must_use]
+    pub fn new(path: PathBuf, local_graph_tx: Sender<LocalGraph>, errors_tx: Sender<Errors>) -> Self {
+        Self {
+            path,
+            local_graph_tx,
+            errors_tx,
+        }
+    }
+
+    fn send_error(&self, error: Errors) {
+        self.errors_tx
+            .send(error)
+            .expect("errors receiver dropped before run completion");
+    }
+}
+
+impl Job for IndexingRubyFileJob {
+    fn run(&self) {
+        let Ok(source) = fs::read_to_string(&self.path) else {
+            self.send_error(Errors::FileError(format!(
+                "Failed to read file `{}`",
+                self.path.display()
+            )));
+
+            return;
+        };
+
+        let Ok(url) = Url::from_file_path(&self.path) else {
+            self.send_error(Errors::FileError(format!(
+                "Couldn't build URI from path `{}`",
+                self.path.display()
+            )));
+
+            return;
+        };
+
+        let mut ruby_indexer = RubyIndexer::new(url.to_string(), &source);
+        ruby_indexer.index();
+
+        self.local_graph_tx
+            .send(ruby_indexer.local_graph())
+            .expect("graph receiver dropped before merge");
+    }
+}
+
+/// Indexes the given paths, reading the content from disk and populating the given `Graph` instance.
+///
+/// # Panics
+///
+/// Will panic if the graph cannot be wrapped in an Arc<Mutex<>>
+pub fn index_files(graph: &mut Graph, paths: Vec<PathBuf>) -> Vec<Errors> {
+    let queue = Arc::new(JobQueue::new());
+    let (local_graphs_tx, local_graphs_rx) = unbounded();
+    let (errors_tx, errors_rx) = unbounded();
+
+    for path in paths {
+        queue.push(Box::new(IndexingRubyFileJob::new(
+            path,
+            local_graphs_tx.clone(),
+            errors_tx.clone(),
+        )));
+    }
+
+    drop(local_graphs_tx);
+    drop(errors_tx);
+
+    JobQueue::run(&queue);
+
+    while let Ok(local_graph) = local_graphs_rx.recv() {
+        graph.update(local_graph);
+    }
+
+    errors_rx.iter().collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use std::path::PathBuf;
+
+    use super::*;
+    use crate::test_utils::Context;
+    use std::path::Path;
+
+    #[test]
+    fn index_relative_paths() {
+        let relative_path = Path::new("foo").join("bar.rb");
+        let context = Context::new();
+        context.touch(&relative_path);
+
+        let working_directory = std::env::current_dir().unwrap();
+        let absolute_path = context.absolute_path_to("foo/bar.rb");
+
+        let mut dots = PathBuf::from("..");
+
+        for _ in 0..working_directory.components().count() - 1 {
+            dots = dots.join("..");
+        }
+
+        let relative_to_pwd = &dots.join(absolute_path);
+
+        let mut graph = Graph::new();
+        let errors = index_files(&mut graph, vec![relative_to_pwd.clone()]);
+
+        assert!(errors.is_empty());
+        assert_eq!(graph.documents().len(), 1);
+    }
+}

data/rust/rubydex/src/job_queue.rs

@@ -0,0 +1,186 @@
+use std::panic::{self, AssertUnwindSafe};
+use std::{
+    sync::Arc,
+    sync::atomic::{AtomicUsize, Ordering},
+    thread,
+};
+
+use crossbeam_deque::{Injector, Steal, Stealer, Worker};
+use crossbeam_utils::Backoff;
+
+/// Work-stealing queue that balances jobs across worker threads.
+///
+/// Jobs are pushed onto the global `injector`, then workers move them into their
+/// own local queues. Work stealing lets idle workers drain the global injector
+/// first and then steal from peers, keeping the CPU busy without coarse locks.
+/// `in_flight` tracks outstanding jobs so threads can tell when all work
+/// (including work spawned by other jobs) has finished.
+#[derive(Default)]
+pub struct JobQueue {
+    /// Global queue feeding newly discovered jobs to workers.
+    injector: Injector<Box<dyn Job + Send + 'static>>,
+    /// Count of jobs that have been queued but not yet completed.
+    in_flight: AtomicUsize,
+}
+
+impl JobQueue {
+    #[must_use]
+    pub fn new() -> Self {
+        Self {
+            injector: Injector::new(),
+            in_flight: AtomicUsize::new(0),
+        }
+    }
+
+    /// Enqueue a job for processing.
+    pub fn push(&self, job: Box<dyn Job + Send + 'static>) {
+        // Increment the count of in-flight jobs to indicate a new job has been enqueued.
+        self.in_flight.fetch_add(1, Ordering::Relaxed);
+        // Push the job onto the global injector queue for later execution by workers.
+        self.injector.push(job);
+    }
+
+    /// Run jobs until the queue is empty.
+    ///
+    /// Accepts the shared queue so jobs can enqueue more work while the runner is processing.
+    pub fn run(queue: &Arc<JobQueue>) {
+        // Determine the number of worker threads to launch.
+        // Use the number of available logical CPUs, falling back to 4 if detection fails.
+        let worker_count = thread::available_parallelism()
+            .map(std::num::NonZeroUsize::get)
+            .unwrap_or(4);
+
+        Self::run_with_workers(queue, worker_count);
+    }
+
+    /// Spin up `worker_count` threads, each with its own local queue, and block until all threads finish.
+    ///
+    /// Workers steal from each other and from the global injector to keep the work balanced.
+    fn run_with_workers(queue: &Arc<JobQueue>, worker_count: usize) {
+        let mut handles = Vec::with_capacity(worker_count);
+        let mut workers = Vec::with_capacity(worker_count);
+        let mut stealers = Vec::with_capacity(worker_count);
+
+        for _ in 0..worker_count {
+            let worker = Worker::new_fifo();
+            stealers.push(worker.stealer()); // For stealing jobs from this queue
+            workers.push(worker);
+        }
+
+        // Wrap all stealers in an Arc so they can be shared across all threads.
+        let stealers = Arc::new(stealers);
+
+        // Start a worker thread for each local queue.
+        for worker in workers {
+            let queue = Arc::clone(queue);
+            let stealers = Arc::clone(&stealers);
+            handles.push(thread::spawn(move || queue.worker_loop(&worker, &stealers)));
+        }
+
+        // Wait for all worker threads to finish before returning.
+        for handle in handles {
+            handle.join().expect("Worker thread panicked");
+        }
+    }
+
+    /// Drain work for a single worker.
+    ///
+    /// The worker prefers its local queue, then the global injector, then other
+    /// workers. If no job is immediately available, it backs off briefly and
+    /// exits once `in_flight` reaches zero (meaning no pending work anywhere).
+    fn worker_loop(
+        &self,
+        local: &Worker<Box<dyn Job + Send + 'static>>,
+        stealers: &Arc<Vec<Stealer<Box<dyn Job + Send + 'static>>>>,
+    ) {
+        // Create a backoff utility for yielding when no job is immediately available
+        let backoff = Backoff::new();
+
+        // Loop until all work is done (in_flight reaches zero)
+        loop {
+            // Try to steal the next job to execute. Prioritize own local queue, then global, then peers.
+            let Some(job) = Self::steal_job(local, stealers, &self.injector) else {
+                if self.in_flight.load(Ordering::Acquire) == 0 {
+                    // All work is done, exit the worker loop
+                    break;
+                }
+                // No work found and still pending jobs: brief pause before retrying
+                backoff.snooze();
+                continue;
+            };
+
+            // Reset backoff as we've obtained a job to process
+            backoff.reset();
+
+            // Run the job and capture any panics so we can propagate them safely
+            let result = panic::catch_unwind(AssertUnwindSafe(|| job.run()));
+
+            // Job completed; decrement the in-flight job counter
+            self.in_flight.fetch_sub(1, Ordering::Relaxed);
+
+            // If the job panicked, resume the panic on this thread
+            if let Err(payload) = result {
+                panic::resume_unwind(payload);
+            }
+        }
+    }
+
+    /// Find the next job for a worker.
+    ///
+    /// Priority: pop from the worker's local queue, steal a batch from the
+    /// global injector, then try stealing from peer workers. Returning `None`
+    /// signals the caller to back off or eventually exit if no jobs remain.
+    fn steal_job(
+        local: &Worker<Box<dyn Job + Send + 'static>>,
+        stealers: &[Stealer<Box<dyn Job + Send + 'static>>],
+        injector: &Injector<Box<dyn Job + Send + 'static>>,
+    ) -> Option<Box<dyn Job + Send + 'static>> {
+        // First, try to pop a job from the worker's own local queue (fastest, lowest contention).
+        if let Some(job) = local.pop() {
+            return Some(job);
+        }
+
+        // If the local queue is empty, try to steal a batch of jobs from the global injector queue.
+        match injector.steal_batch_and_pop(local) {
+            Steal::Success(job) => return Some(job), // Successfully stole a job from the global injector.
+            Steal::Retry => return None, // Transient failure; signal to caller to back off and try again later.
+            Steal::Empty => {}           // No jobs available in the global injector; continue to peers.
+        }
+
+        // As a last resort, attempt to steal jobs from each of the peer workers' queues.
+        for stealer in stealers {
+            match stealer.steal_batch_and_pop(local) {
+                Steal::Success(job) => return Some(job), // Successfully stole a job from a peer.
+                Steal::Retry => return None,             // Retry advised; caller should back off and loop.
+                Steal::Empty => {}                       // No jobs in this peer; try next peer.
+            }
+        }
+
+        // No jobs available from any source.
+        None
+    }
+}
+
+/// Unit of work scheduled on a `JobQueue`.
+///
+/// # Example
+///
+/// ```
+/// use std::sync::Arc;
+/// use rubydex::job_queue::{Job, JobQueue};
+///
+/// struct PrintJob;
+///
+/// impl Job for PrintJob {
+///     fn run(&self) {
+///         println!("hello from a worker");
+///     }
+/// }
+///
+/// let queue = Arc::new(JobQueue::new());
+/// queue.push(Box::new(PrintJob));
+/// JobQueue::run(&queue);
+/// ```
+pub trait Job: Send {
+    fn run(&self);
+}

data/rust/rubydex/src/lib.rs

@@ -0,0 +1,15 @@
+pub mod diagnostic;
+pub mod errors;
+pub mod indexing;
+pub mod job_queue;
+pub mod listing;
+pub mod model;
+pub mod offset;
+pub mod position;
+pub mod query;
+pub mod resolution;
+pub mod stats;
+pub mod visualization;
+
+#[cfg(any(test, feature = "test_utils"))]
+pub mod test_utils;

data/rust/rubydex/src/listing.rs

@@ -0,0 +1,249 @@
+use crate::{
+    errors::Errors,
+    job_queue::{Job, JobQueue},
+};
+use crossbeam_channel::{Sender, unbounded};
+use std::{
+    fs,
+    path::{Path, PathBuf},
+    sync::Arc,
+};
+
+pub struct FileDiscoveryJob {
+    path: PathBuf,
+    queue: Arc<JobQueue>,
+    paths_tx: Sender<PathBuf>,
+    errors_tx: Sender<Errors>,
+}
+
+impl FileDiscoveryJob {
+    #[must_use]
+    pub fn new(path: PathBuf, queue: Arc<JobQueue>, paths_tx: Sender<PathBuf>, errors_tx: Sender<Errors>) -> Self {
+        Self {
+            path,
+            queue,
+            paths_tx,
+            errors_tx,
+        }
+    }
+}
+
+impl FileDiscoveryJob {
+    fn handle_file(&self, path: &Path) {
+        if path.extension().is_some_and(|ext| ext == "rb") {
+            self.paths_tx
+                .send(path.to_path_buf())
+                .expect("file receiver dropped before run completion");
+        }
+    }
+
+    fn handle_symlink(&self, path: &PathBuf) {
+        let Ok(canonicalized) = fs::canonicalize(path) else {
+            self.send_error(Errors::FileError(format!(
+                "Failed to canonicalize symlink: `{}`",
+                path.display(),
+            )));
+
+            return;
+        };
+
+        self.queue.push(Box::new(FileDiscoveryJob::new(
+            canonicalized,
+            Arc::clone(&self.queue),
+            self.paths_tx.clone(),
+            self.errors_tx.clone(),
+        )));
+    }
+
+    fn send_error(&self, error: Errors) {
+        self.errors_tx
+            .send(error)
+            .expect("error receiver dropped before run completion");
+    }
+}
+
+impl Job for FileDiscoveryJob {
+    fn run(&self) {
+        if self.path.is_dir() {
+            let Ok(read_dir) = self.path.read_dir() else {
+                self.send_error(Errors::FileError(format!(
+                    "Failed to read directory `{}`",
+                    self.path.display(),
+                )));
+
+                return;
+            };
+
+            for result in read_dir {
+                let Ok(entry) = result else {
+                    self.send_error(Errors::FileError(format!(
+                        "Failed to read directory `{}`: {result:?}",
+                        self.path.display(),
+                    )));
+
+                    continue;
+                };
+
+                let kind = entry.file_type().unwrap();
+
+                if kind.is_dir() {
+                    self.queue.push(Box::new(FileDiscoveryJob::new(
+                        entry.path(),
+                        Arc::clone(&self.queue),
+                        self.paths_tx.clone(),
+                        self.errors_tx.clone(),
+                    )));
+                } else if kind.is_file() {
+                    self.handle_file(&entry.path());
+                } else if kind.is_symlink() {
+                    self.handle_symlink(&entry.path());
+                } else {
+                    self.send_error(Errors::FileError(format!(
+                        "Path `{}` is not a file or directory",
+                        entry.path().display()
+                    )));
+                }
+            }
+        } else if self.path.is_file() {
+            self.handle_file(&self.path);
+        } else if self.path.is_symlink() {
+            self.handle_symlink(&self.path);
+        } else {
+            self.send_error(Errors::FileError(format!(
+                "Path `{}` is not a file or directory",
+                self.path.display()
+            )));
+        }
+    }
+}
+
+/// Recursively collects all Ruby files for the given workspace and dependencies, returning a vector of document instances
+///
+/// # Errors
+///
+/// Returns a `MultipleErrors` if any of the paths do not exist
+///
+/// # Panics
+///
+/// Panics if the errors receiver is dropped before the run completion
+#[must_use]
+pub fn collect_file_paths(paths: Vec<String>) -> (Vec<PathBuf>, Vec<Errors>) {
+    let queue = Arc::new(JobQueue::new());
+    let (files_tx, files_rx) = unbounded();
+    let (errors_tx, errors_rx) = unbounded();
+
+    for path in paths {
+        let Ok(canonicalized) = fs::canonicalize(&path) else {
+            errors_tx
+                .send(Errors::FileError(format!("Path `{path}` does not exist")))
+                .expect("errors receiver dropped before run completion");
+
+            continue;
+        };
+
+        queue.push(Box::new(FileDiscoveryJob::new(
+            canonicalized,
+            Arc::clone(&queue),
+            files_tx.clone(),
+            errors_tx.clone(),
+        )));
+    }
+
+    JobQueue::run(&queue);
+
+    drop(files_tx);
+    drop(errors_tx);
+
+    (files_rx.iter().collect(), errors_rx.iter().collect())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::test_utils::Context;
+
+    fn collect_document_paths(context: &Context, paths: &[&str]) -> (Vec<String>, Vec<Errors>) {
+        let (files, errors) = collect_file_paths(
+            paths
+                .iter()
+                .map(|p| context.absolute_path_to(p).to_string_lossy().into_owned())
+                .collect(),
+        );
+
+        let mut files: Vec<String> = files
+            .iter()
+            .map(|path| context.relative_path_to(path).to_string_lossy().into_owned())
+            .collect();
+
+        files.sort();
+
+        (files, errors)
+    }
+
+    #[test]
+    fn collect_all_documents() {
+        let context = Context::new();
+        let baz = PathBuf::from("bar").join("baz.rb");
+        let qux = PathBuf::from("bar").join("qux.rb");
+        let bar = PathBuf::from("foo").join("bar.rb");
+        context.touch(&baz);
+        context.touch(&qux);
+        context.touch(&bar);
+
+        let (files, errors) = collect_document_paths(&context, &["foo", "bar"]);
+
+        assert!(errors.is_empty());
+
+        assert_eq!(
+            files,
+            vec![
+                baz.to_str().unwrap().to_string(),
+                qux.to_str().unwrap().to_string(),
+                bar.to_str().unwrap().to_string()
+            ]
+        );
+    }
+
+    #[test]
+    fn collect_some_documents_based_on_paths() {
+        let context = Context::new();
+        let baz = PathBuf::from("bar").join("baz.rb");
+        let qux = PathBuf::from("bar").join("qux.rb");
+        let bar = PathBuf::from("foo").join("bar.rb");
+
+        context.touch(&baz);
+        context.touch(&qux);
+        context.touch(&bar);
+
+        let (files, errors) = collect_document_paths(&context, &["bar"]);
+
+        assert!(errors.is_empty());
+
+        assert_eq!(
+            files,
+            vec![baz.to_str().unwrap().to_string(), qux.to_str().unwrap().to_string()]
+        );
+    }
+
+    #[test]
+    fn collect_non_existing_paths() {
+        let context = Context::new();
+
+        let (files, errors) = collect_file_paths(vec![
+            context
+                .absolute_path_to("non_existing_path")
+                .to_string_lossy()
+                .into_owned(),
+        ]);
+
+        assert!(files.is_empty());
+
+        assert_eq!(
+            errors,
+            vec![Errors::FileError(format!(
+                "Path `{}` does not exist",
+                context.absolute_path_to("non_existing_path").display()
+            ))]
+        );
+    }
+}

data/rust/rubydex/src/main.rs

@@ -0,0 +1,116 @@
+use clap::{Parser, ValueEnum};
+use std::mem;
+
+use rubydex::{
+    indexing, listing,
+    model::graph::Graph,
+    resolution::Resolver,
+    stats::{
+        memory::MemoryStats,
+        timer::{Timer, time_it},
+    },
+    visualization::dot,
+};
+
+#[derive(Parser, Debug)]
+#[command(name = "rubydex_cli", about = "A Static Analysis Toolkit for Ruby", version)]
+#[allow(clippy::struct_excessive_bools)]
+struct Args {
+    #[arg(value_name = "PATHS", default_value = ".")]
+    paths: Vec<String>,
+
+    #[arg(long = "stop-after", help = "Stop after the given stage")]
+    stop_after: Option<StopAfter>,
+
+    #[arg(long = "visualize")]
+    visualize: bool,
+
+    #[arg(long = "stats", help = "Show detailed performance statistics")]
+    stats: bool,
+}
+
+#[derive(Debug, Clone, ValueEnum)]
+enum StopAfter {
+    Listing,
+    Indexing,
+    Resolution,
+}
+
+fn exit(print_stats: bool) {
+    if print_stats {
+        Timer::print_breakdown();
+        MemoryStats::print_memory_usage();
+    }
+
+    std::process::exit(0);
+}
+
+fn main() {
+    let args = Args::parse();
+
+    if args.stats {
+        Timer::set_global_timer(Timer::new());
+    }
+
+    // Listing
+
+    let (file_paths, errors) = time_it!(listing, { listing::collect_file_paths(args.paths) });
+
+    for error in errors {
+        eprintln!("{error}");
+    }
+
+    if let Some(StopAfter::Listing) = args.stop_after {
+        return exit(args.stats);
+    }
+
+    // Indexing
+
+    let mut graph = Graph::new();
+    let errors = time_it!(indexing, { indexing::index_files(&mut graph, file_paths) });
+
+    for error in errors {
+        eprintln!("{error}");
+    }
+
+    if let Some(StopAfter::Indexing) = args.stop_after {
+        return exit(args.stats);
+    }
+
+    // Resolution
+
+    time_it!(resolution, {
+        let mut resolver = Resolver::new(&mut graph);
+        resolver.resolve_all();
+    });
+
+    if let Some(StopAfter::Resolution) = args.stop_after {
+        return exit(args.stats);
+    }
+
+    // Querying
+
+    if args.stats {
+        time_it!(querying, {
+            graph.print_query_statistics();
+        });
+    }
+
+    if args.stats {
+        Timer::print_breakdown();
+        MemoryStats::print_memory_usage();
+    }
+
+    // Generate visualization or print statistics
+    if args.visualize {
+        println!("{}", dot::generate(&graph));
+    } else {
+        println!("Indexed {} files", graph.documents().len());
+        println!("Found {} names", graph.declarations().len());
+        println!("Found {} definitions", graph.definitions().len());
+        println!("Found {} URIs", graph.documents().len());
+    }
+
+    // Forget the graph so we don't have to wait for deallocation and let the system reclaim the memory at exit
+    mem::forget(graph);
+}