rubydex 0.1.0.beta12-aarch64-linux

data/rust/rubydex/src/indexing.rs

@@ -0,0 +1,210 @@
+use crate::{
+    errors::Errors,
+    indexing::{local_graph::LocalGraph, rbs_indexer::RBSIndexer, ruby_indexer::RubyIndexer},
+    job_queue::{Job, JobQueue},
+    model::graph::Graph,
+};
+use crossbeam_channel::{Sender, unbounded};
+use std::{ffi::OsStr, fs, path::PathBuf, sync::Arc};
+use url::Url;
+
+pub mod local_graph;
+pub mod rbs_indexer;
+pub mod ruby_indexer;
+
+/// The language of a source document, used to dispatch to the appropriate indexer
+pub enum LanguageId {
+    Ruby,
+    Rbs,
+}
+
+impl From<&OsStr> for LanguageId {
+    fn from(ext: &OsStr) -> Self {
+        if ext == "rbs" { Self::Rbs } else { Self::Ruby }
+    }
+}
+
+impl LanguageId {
+    /// Determines the language from an LSP language ID string.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if the language ID is not recognized.
+    pub fn from_language_id(language_id: &str) -> Result<Self, Errors> {
+        match language_id {
+            "ruby" => Ok(Self::Ruby),
+            "rbs" => Ok(Self::Rbs),
+            _ => Err(Errors::FileError(format!("Unsupported language_id `{language_id}`"))),
+        }
+    }
+}
+
+/// Job that indexes a single file
+pub struct IndexingJob {
+    path: PathBuf,
+    local_graph_tx: Sender<LocalGraph>,
+    errors_tx: Sender<Errors>,
+}
+
+impl IndexingJob {
+    #[must_use]
+    pub fn new(path: PathBuf, local_graph_tx: Sender<LocalGraph>, errors_tx: Sender<Errors>) -> Self {
+        Self {
+            path,
+            local_graph_tx,
+            errors_tx,
+        }
+    }
+
+    fn send_error(&self, error: Errors) {
+        self.errors_tx
+            .send(error)
+            .expect("errors receiver dropped before run completion");
+    }
+}
+
+impl Job for IndexingJob {
+    fn run(&self) {
+        let Ok(source) = fs::read_to_string(&self.path) else {
+            self.send_error(Errors::FileError(format!(
+                "Failed to read file `{}`",
+                self.path.display()
+            )));
+
+            return;
+        };
+
+        let Ok(url) = Url::from_file_path(&self.path) else {
+            self.send_error(Errors::FileError(format!(
+                "Couldn't build URI from path `{}`",
+                self.path.display()
+            )));
+
+            return;
+        };
+
+        let language = self.path.extension().map_or(LanguageId::Ruby, LanguageId::from);
+        let local_graph = build_local_graph(url.to_string(), &source, &language);
+
+        self.local_graph_tx
+            .send(local_graph)
+            .expect("graph receiver dropped before merge");
+    }
+}
+
+/// Indexes a single source string in memory, dispatching to the appropriate indexer based on `language_id`.
+pub fn index_source(graph: &mut Graph, uri: &str, source: &str, language_id: &LanguageId) {
+    let local_graph = build_local_graph(uri.to_string(), source, language_id);
+    graph.consume_document_changes(local_graph);
+}
+
+/// Indexes the given paths, reading the content from disk and populating the given `Graph` instance.
+///
+/// # Panics
+///
+/// Will panic if the graph cannot be wrapped in an Arc<Mutex<>>
+pub fn index_files(graph: &mut Graph, paths: Vec<PathBuf>) -> Vec<Errors> {
+    let queue = Arc::new(JobQueue::new());
+    let (local_graphs_tx, local_graphs_rx) = unbounded();
+    let (errors_tx, errors_rx) = unbounded();
+
+    for path in paths {
+        queue.push(Box::new(IndexingJob::new(
+            path,
+            local_graphs_tx.clone(),
+            errors_tx.clone(),
+        )));
+    }
+
+    drop(local_graphs_tx);
+    drop(errors_tx);
+
+    let handles = JobQueue::run_without_waiting(&queue);
+
+    // Merge graphs as they arrive, overlapping with indexing work on other threads.
+    while let Ok(local_graph) = local_graphs_rx.recv() {
+        graph.consume_document_changes(local_graph);
+    }
+
+    for handle in handles {
+        handle.join().expect("Worker thread panicked");
+    }
+
+    errors_rx.iter().collect()
+}
+
+/// Indexes a source string using the appropriate indexer for the given language.
+fn build_local_graph(uri: String, source: &str, language: &LanguageId) -> LocalGraph {
+    match language {
+        LanguageId::Ruby => {
+            let mut indexer = RubyIndexer::new(uri, source);
+            indexer.index();
+            indexer.local_graph()
+        }
+        LanguageId::Rbs => {
+            let mut indexer = RBSIndexer::new(uri, source);
+            indexer.index();
+            indexer.local_graph()
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::path::PathBuf;
+
+    use super::*;
+    use crate::test_utils::Context;
+    use std::path::Path;
+
+    #[test]
+    fn index_relative_paths() {
+        let relative_path = Path::new("foo").join("bar.rb");
+        let context = Context::new();
+        context.touch(&relative_path);
+
+        let working_directory = std::env::current_dir().unwrap();
+        let absolute_path = context.absolute_path_to("foo/bar.rb");
+
+        let mut dots = PathBuf::from("..");
+
+        for _ in 0..working_directory.components().count() - 1 {
+            dots = dots.join("..");
+        }
+
+        let relative_to_pwd = &dots.join(absolute_path);
+
+        let mut graph = Graph::new();
+        let errors = index_files(&mut graph, vec![relative_to_pwd.clone()]);
+
+        assert!(errors.is_empty());
+        assert_eq!(graph.documents().len(), 2);
+    }
+
+    #[test]
+    fn from_language_id_unknown() {
+        let result = LanguageId::from_language_id("python");
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn updating_document_from_in_memory_source() {
+        let context = Context::new();
+        let path = context.absolute_path_to("foo/bar.rb");
+        context.write(&path, "class Foo; end");
+
+        let uri = Url::from_file_path(&path).unwrap().to_string();
+
+        let mut graph = Graph::new();
+        let errors = index_files(&mut graph, vec![path]);
+
+        assert!(errors.is_empty(), "Expected no errors, got: {errors:#?}");
+        assert_eq!(6, graph.definitions().len());
+        assert_eq!(2, graph.documents().len());
+
+        index_source(&mut graph, &uri, "", &LanguageId::Ruby);
+
+        assert_eq!(5, graph.definitions().len());
+        assert_eq!(2, graph.documents().len());
+    }
+}

data/rust/rubydex/src/integrity.rs

@@ -0,0 +1,279 @@
+use std::fmt;
+
+use crate::model::{
+    built_in::{BASIC_OBJECT_ID, OBJECT_ID},
+    declaration::{Declaration, Namespace},
+    graph::Graph,
+    ids::DeclarationId,
+};
+
+#[derive(Debug, PartialEq, Eq)]
+pub enum IntegrityErrorKind {
+    /// A declaration's owner is not a namespace (module, class, or singleton class)
+    OwnerIsNotNamespace,
+    /// A declaration's owner does not exist in the graph
+    OwnerDoesNotExist,
+    /// A singleton class chain never resolves to a non-singleton namespace
+    SingletonClassChainDoesNotTerminate,
+    /// A non-root declaration unexpectedly owns itself
+    UnexpectedSelfOwnership,
+}
+
+/// An integrity error found during graph validation
+#[derive(Debug, PartialEq, Eq)]
+pub struct IntegrityError {
+    kind: IntegrityErrorKind,
+    declaration_name: String,
+    uris: Vec<String>,
+}
+
+impl fmt::Display for IntegrityError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let message = match self.kind {
+            IntegrityErrorKind::OwnerIsNotNamespace => {
+                format!("Declaration `{}` is owned by a non-namespace", self.declaration_name)
+            }
+            IntegrityErrorKind::OwnerDoesNotExist => {
+                format!(
+                    "Declaration `{}` has an owner that does not exist in the graph",
+                    self.declaration_name
+                )
+            }
+            IntegrityErrorKind::SingletonClassChainDoesNotTerminate => {
+                format!(
+                    "Singleton class `{}` does not eventually attach to a non-singleton namespace",
+                    self.declaration_name
+                )
+            }
+            IntegrityErrorKind::UnexpectedSelfOwnership => {
+                format!("Declaration `{}` unexpectedly owns itself", self.declaration_name)
+            }
+        };
+
+        write!(f, "{message}. Defined in: {}", self.uris.join(", "))
+    }
+}
+
+impl std::error::Error for IntegrityError {}
+
+/// Checks the integrity of the graph data
+#[must_use]
+pub fn check_integrity(graph: &Graph) -> Vec<IntegrityError> {
+    let mut errors = Vec::new();
+    let self_owners = [*OBJECT_ID, *BASIC_OBJECT_ID];
+
+    for (id, declaration) in graph.declarations() {
+        let owner_id = declaration.owner_id();
+
+        // Check for constants that own themselves. Only `Object` and `BasicObject` own themselves and no other constant
+        if *id == *owner_id {
+            if self_owners.contains(id) {
+                continue;
+            }
+            errors.push(IntegrityError {
+                kind: IntegrityErrorKind::UnexpectedSelfOwnership,
+                declaration_name: declaration.name().to_string(),
+                uris: collect_uris(graph, declaration),
+            });
+            continue;
+        }
+
+        // Check that the owner exists
+        let Some(owner) = graph.declarations().get(owner_id) else {
+            errors.push(IntegrityError {
+                kind: IntegrityErrorKind::OwnerDoesNotExist,
+                declaration_name: declaration.name().to_string(),
+                uris: collect_uris(graph, declaration),
+            });
+            continue;
+        };
+
+        // Check that the owner is a namespace
+        if owner.as_namespace().is_none() {
+            errors.push(IntegrityError {
+                kind: IntegrityErrorKind::OwnerIsNotNamespace,
+                declaration_name: declaration.name().to_string(),
+                uris: collect_uris(graph, declaration),
+            });
+            continue;
+        }
+
+        // Check singleton class chain termination
+        if let Declaration::Namespace(Namespace::SingletonClass(_)) = declaration
+            && !singleton_chain_terminates(graph, *owner_id)
+        {
+            errors.push(IntegrityError {
+                kind: IntegrityErrorKind::SingletonClassChainDoesNotTerminate,
+                declaration_name: declaration.name().to_string(),
+                uris: collect_uris(graph, declaration),
+            });
+        }
+    }
+
+    errors
+}
+
+/// Collects the URIs where a declaration is defined, sorted and deduplicated
+fn collect_uris(graph: &Graph, declaration: &Declaration) -> Vec<String> {
+    declaration
+        .definitions()
+        .iter()
+        .map(|def_id| {
+            let definition = graph.definitions().get(def_id).unwrap();
+            let document = graph.documents().get(definition.uri_id()).unwrap();
+            document.uri().to_string()
+        })
+        .collect()
+}
+
+/// Walks the singleton class chain to verify that it eventually finds a module or class as its attached object
+fn singleton_chain_terminates(graph: &Graph, start_owner_id: DeclarationId) -> bool {
+    const MAX_SINGLETON_DEPTH: usize = 128;
+    let mut current_id = start_owner_id;
+
+    for _ in 0..MAX_SINGLETON_DEPTH {
+        let Some(current) = graph.declarations().get(&current_id) else {
+            return false;
+        };
+
+        match current {
+            Declaration::Namespace(Namespace::SingletonClass(_)) => {
+                current_id = *current.owner_id();
+            }
+            Declaration::Namespace(_) => return true,
+            _ => return false,
+        }
+    }
+
+    false
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::model::declaration::{ClassDeclaration, MethodDeclaration, SingletonClassDeclaration};
+
+    #[test]
+    fn test_unexpected_self_ownership() {
+        let mut graph = Graph::new();
+
+        // Object and BasicObject are exempt from self-ownership
+        graph.declarations_mut().insert(
+            *OBJECT_ID,
+            Declaration::Namespace(Namespace::Class(Box::new(ClassDeclaration::new(
+                "Object".to_string(),
+                *OBJECT_ID,
+            )))),
+        );
+        graph.declarations_mut().insert(
+            *BASIC_OBJECT_ID,
+            Declaration::Namespace(Namespace::Class(Box::new(ClassDeclaration::new(
+                "BasicObject".to_string(),
+                *BASIC_OBJECT_ID,
+            )))),
+        );
+
+        // Foo owns itself — should be an error
+        let foo_id = DeclarationId::from("Foo");
+        graph.declarations_mut().insert(
+            foo_id,
+            Declaration::Namespace(Namespace::Class(Box::new(ClassDeclaration::new(
+                "Foo".to_string(),
+                foo_id,
+            )))),
+        );
+
+        let errors = check_integrity(&graph);
+        assert_eq!(errors.len(), 1);
+        assert_eq!(errors[0].kind, IntegrityErrorKind::UnexpectedSelfOwnership);
+        assert_eq!(errors[0].declaration_name, "Foo");
+    }
+
+    #[test]
+    fn test_owner_does_not_exist() {
+        let mut graph = Graph::new();
+        let foo_id = DeclarationId::from("Foo");
+        let bogus_owner = DeclarationId::from("NonExistent");
+
+        graph.declarations_mut().insert(
+            foo_id,
+            Declaration::Namespace(Namespace::Class(Box::new(ClassDeclaration::new(
+                "Foo".to_string(),
+                bogus_owner,
+            )))),
+        );
+
+        let errors = check_integrity(&graph);
+        assert_eq!(errors.len(), 1);
+        assert_eq!(errors[0].kind, IntegrityErrorKind::OwnerDoesNotExist);
+        assert_eq!(errors[0].declaration_name, "Foo");
+    }
+
+    #[test]
+    fn test_owner_is_not_namespace() {
+        let mut graph = Graph::new();
+
+        // Object (self-owned, exempt)
+        graph.declarations_mut().insert(
+            *OBJECT_ID,
+            Declaration::Namespace(Namespace::Class(Box::new(ClassDeclaration::new(
+                "Object".to_string(),
+                *OBJECT_ID,
+            )))),
+        );
+
+        // A method owned by Object (valid)
+        let method_id = DeclarationId::from("Object#foo");
+        graph.declarations_mut().insert(
+            method_id,
+            Declaration::Method(Box::new(MethodDeclaration::new("Object#foo".to_string(), *OBJECT_ID))),
+        );
+
+        // A class owned by the method (invalid — owner is not a namespace)
+        let bar_id = DeclarationId::from("Bar");
+        graph.declarations_mut().insert(
+            bar_id,
+            Declaration::Namespace(Namespace::Class(Box::new(ClassDeclaration::new(
+                "Bar".to_string(),
+                method_id,
+            )))),
+        );
+
+        let errors = check_integrity(&graph);
+        assert_eq!(errors.len(), 1);
+        assert_eq!(errors[0].kind, IntegrityErrorKind::OwnerIsNotNamespace);
+        assert_eq!(errors[0].declaration_name, "Bar");
+    }
+
+    #[test]
+    fn test_singleton_class_chain_does_not_terminate() {
+        let mut graph = Graph::new();
+
+        // Two singleton classes that own each other, forming a cycle
+        let s1_id = DeclarationId::from("<Class:Foo>");
+        let s2_id = DeclarationId::from("<Class:Bar>");
+
+        graph.declarations_mut().insert(
+            s1_id,
+            Declaration::Namespace(Namespace::SingletonClass(Box::new(SingletonClassDeclaration::new(
+                "<Class:Foo>".to_string(),
+                s2_id,
+            )))),
+        );
+        graph.declarations_mut().insert(
+            s2_id,
+            Declaration::Namespace(Namespace::SingletonClass(Box::new(SingletonClassDeclaration::new(
+                "<Class:Bar>".to_string(),
+                s1_id,
+            )))),
+        );
+
+        let errors = check_integrity(&graph);
+        assert_eq!(errors.len(), 2);
+        assert!(
+            errors
+                .iter()
+                .all(|e| e.kind == IntegrityErrorKind::SingletonClassChainDoesNotTerminate)
+        );
+    }
+}

data/rust/rubydex/src/job_queue.rs

@@ -0,0 +1,205 @@
+use std::panic::{self, AssertUnwindSafe};
+use std::{
+    sync::Arc,
+    sync::atomic::{AtomicUsize, Ordering},
+    thread,
+};
+
+use crossbeam_deque::{Injector, Steal, Stealer, Worker};
+use crossbeam_utils::Backoff;
+
+/// Work-stealing queue that balances jobs across worker threads.
+///
+/// Jobs are pushed onto the global `injector`, then workers move them into their
+/// own local queues. Work stealing lets idle workers drain the global injector
+/// first and then steal from peers, keeping the CPU busy without coarse locks.
+/// `in_flight` tracks outstanding jobs so threads can tell when all work
+/// (including work spawned by other jobs) has finished.
+#[derive(Default)]
+pub struct JobQueue {
+    /// Global queue feeding newly discovered jobs to workers.
+    injector: Injector<Box<dyn Job + Send + 'static>>,
+    /// Count of jobs that have been queued but not yet completed.
+    in_flight: AtomicUsize,
+}
+
+impl JobQueue {
+    #[must_use]
+    pub fn new() -> Self {
+        Self {
+            injector: Injector::new(),
+            in_flight: AtomicUsize::new(0),
+        }
+    }
+
+    /// Enqueue a job for processing.
+    pub fn push(&self, job: Box<dyn Job + Send + 'static>) {
+        // Increment the count of in-flight jobs to indicate a new job has been enqueued.
+        self.in_flight.fetch_add(1, Ordering::Relaxed);
+        // Push the job onto the global injector queue for later execution by workers.
+        self.injector.push(job);
+    }
+
+    /// Run jobs until the queue is empty.
+    ///
+    /// Accepts the shared queue so jobs can enqueue more work while the runner is processing.
+    pub fn run(queue: &Arc<JobQueue>) {
+        // Determine the number of worker threads to launch.
+        // Use the number of available logical CPUs, falling back to 4 if detection fails.
+        let worker_count = thread::available_parallelism()
+            .map(std::num::NonZeroUsize::get)
+            .unwrap_or(4);
+
+        Self::run_with_workers(queue, worker_count);
+    }
+
+    /// Spin up worker threads and return their handles without waiting for completion.
+    ///
+    /// The caller is responsible for joining the returned handles. This allows
+    /// overlapping other work (e.g. merging results) with job processing.
+    pub fn run_without_waiting(queue: &Arc<JobQueue>) -> Vec<thread::JoinHandle<()>> {
+        let worker_count = thread::available_parallelism()
+            .map(std::num::NonZeroUsize::get)
+            .unwrap_or(4);
+
+        Self::spawn_workers(queue, worker_count)
+    }
+
+    /// Spin up `worker_count` threads, each with its own local queue, and block until all threads finish.
+    ///
+    /// Workers steal from each other and from the global injector to keep the work balanced.
+    fn run_with_workers(queue: &Arc<JobQueue>, worker_count: usize) {
+        let handles = Self::spawn_workers(queue, worker_count);
+
+        // Wait for all worker threads to finish before returning.
+        for handle in handles {
+            handle.join().expect("Worker thread panicked");
+        }
+    }
+
+    /// Spin up `worker_count` threads and return their join handles.
+    fn spawn_workers(queue: &Arc<JobQueue>, worker_count: usize) -> Vec<thread::JoinHandle<()>> {
+        let mut handles = Vec::with_capacity(worker_count);
+        let mut workers = Vec::with_capacity(worker_count);
+        let mut stealers = Vec::with_capacity(worker_count);
+
+        for _ in 0..worker_count {
+            let worker = Worker::new_fifo();
+            stealers.push(worker.stealer()); // For stealing jobs from this queue
+            workers.push(worker);
+        }
+
+        // Wrap all stealers in an Arc so they can be shared across all threads.
+        let stealers = Arc::new(stealers);
+
+        // Start a worker thread for each local queue.
+        for worker in workers {
+            let queue = Arc::clone(queue);
+            let stealers = Arc::clone(&stealers);
+            handles.push(thread::spawn(move || queue.worker_loop(&worker, &stealers)));
+        }
+
+        handles
+    }
+
+    /// Drain work for a single worker.
+    ///
+    /// The worker prefers its local queue, then the global injector, then other
+    /// workers. If no job is immediately available, it backs off briefly and
+    /// exits once `in_flight` reaches zero (meaning no pending work anywhere).
+    fn worker_loop(
+        &self,
+        local: &Worker<Box<dyn Job + Send + 'static>>,
+        stealers: &Arc<Vec<Stealer<Box<dyn Job + Send + 'static>>>>,
+    ) {
+        // Create a backoff utility for yielding when no job is immediately available
+        let backoff = Backoff::new();
+
+        // Loop until all work is done (in_flight reaches zero)
+        loop {
+            // Try to steal the next job to execute. Prioritize own local queue, then global, then peers.
+            let Some(job) = Self::steal_job(local, stealers, &self.injector) else {
+                if self.in_flight.load(Ordering::Acquire) == 0 {
+                    // All work is done, exit the worker loop
+                    break;
+                }
+                // No work found and still pending jobs: brief pause before retrying
+                backoff.snooze();
+                continue;
+            };
+
+            // Reset backoff as we've obtained a job to process
+            backoff.reset();
+
+            // Run the job and capture any panics so we can propagate them safely
+            let result = panic::catch_unwind(AssertUnwindSafe(|| job.run()));
+
+            // Job completed; decrement the in-flight job counter
+            self.in_flight.fetch_sub(1, Ordering::Relaxed);
+
+            // If the job panicked, resume the panic on this thread
+            if let Err(payload) = result {
+                panic::resume_unwind(payload);
+            }
+        }
+    }
+
+    /// Find the next job for a worker.
+    ///
+    /// Priority: pop from the worker's local queue, steal a batch from the
+    /// global injector, then try stealing from peer workers. Returning `None`
+    /// signals the caller to back off or eventually exit if no jobs remain.
+    fn steal_job(
+        local: &Worker<Box<dyn Job + Send + 'static>>,
+        stealers: &[Stealer<Box<dyn Job + Send + 'static>>],
+        injector: &Injector<Box<dyn Job + Send + 'static>>,
+    ) -> Option<Box<dyn Job + Send + 'static>> {
+        // First, try to pop a job from the worker's own local queue (fastest, lowest contention).
+        if let Some(job) = local.pop() {
+            return Some(job);
+        }
+
+        // If the local queue is empty, try to steal a batch of jobs from the global injector queue.
+        match injector.steal_batch_and_pop(local) {
+            Steal::Success(job) => return Some(job), // Successfully stole a job from the global injector.
+            Steal::Retry => return None, // Transient failure; signal to caller to back off and try again later.
+            Steal::Empty => {}           // No jobs available in the global injector; continue to peers.
+        }
+
+        // As a last resort, attempt to steal jobs from each of the peer workers' queues.
+        for stealer in stealers {
+            match stealer.steal_batch_and_pop(local) {
+                Steal::Success(job) => return Some(job), // Successfully stole a job from a peer.
+                Steal::Retry => return None,             // Retry advised; caller should back off and loop.
+                Steal::Empty => {}                       // No jobs in this peer; try next peer.
+            }
+        }
+
+        // No jobs available from any source.
+        None
+    }
+}
+
+/// Unit of work scheduled on a `JobQueue`.
+///
+/// # Example
+///
+/// ```
+/// use std::sync::Arc;
+/// use rubydex::job_queue::{Job, JobQueue};
+///
+/// struct PrintJob;
+///
+/// impl Job for PrintJob {
+///     fn run(&self) {
+///         println!("hello from a worker");
+///     }
+/// }
+///
+/// let queue = Arc::new(JobQueue::new());
+/// queue.push(Box::new(PrintJob));
+/// JobQueue::run(&queue);
+/// ```
+pub trait Job: Send {
+    fn run(&self);
+}