rubydex 0.1.0.beta3-x86_64-linux → 0.1.0.beta6-x86_64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2de40c7335743a85b7d61d9a513d40b29fcecf62e19cfff8ef2637cd4297b6a8
-  data.tar.gz: 99bed4d413ab7e1bd97f76965906037d1da29f12af8dba7f19f48d5665c73bc5
+  metadata.gz: ba2df1c014d93db599af99939667aa410786bae376f66caba253fa0cdc6cddb5
+  data.tar.gz: 2a6b68cf1658cb5f87225cff5a336565541325cd3356a34e96abd77f527789ae
 SHA512:
-  metadata.gz: 29ef5b84c94af6eb1608d42acbc7a0c5c115fef6317645b369fca27c1c1714c64a21dba7a1b10f490939fbf547583fbed20db30d26e9ad4f42b9c8079c8b7e6c
-  data.tar.gz: e4d9dce9acf24ef5c704dab14d89a82143c39e259e8923fc71830abe600b54240094a8e287aa647b62e419fac42db6e1c3f4b59d20b29498d1b5872865083dd5
+  metadata.gz: 5e407f9ade85f4febeadc490b3f306229f3197d323b7d404b0a9f7032b13b93eb78e4794037490ee2fa909a440302f16708437350c8ec946c4e64c5430ab3ec6
+  data.tar.gz: 595952a7249abe53c78b35196f8a03f5defe24fb997f9f67cb6fca8f22769d5e718d4795c3487d273f3e179eb5d1368a93294fd4fb6a0fde2b95e6ccc82eeb75

data/README.md

@@ -11,13 +11,70 @@ of using the Ruby API:
 ```ruby
 # Create a new graph representing the current workspace
 graph = Rubydex::Graph.new
+# Configuring graph LSP encoding
+graph.encoding = "utf16"
 # Index the entire workspace with all dependencies
 graph.index_workspace
+# Or index specific file paths
+graph.index_all(["path/to/file.rb"])
 # Transform the initially collected information into its semantic understanding by running resolution
 graph.resolve
+# Get all diagnostics acquired during the analysis
+graph.diagnostics
 
-# Access the information as needed
-graph["Foo"]
+# Iterating over graph nodes
+graph.declarations
+graph.documents
+graph.constant_references
+graph.method_references
+
+# Analyzing require paths
+graph.resolve_require_path("rails/engine", load_paths) # => document pointed by `rails/engine`
+graph.require_paths(load_paths) # => array of all indexed require paths
+
+# Querying
+graph["Foo"] # Get declaration by fully qualified name
+graph.search("Foo#b") # Name search
+graph.resolve_constant("Bar", ["Foo", "Baz::Qux"]) # Resolve constant reference based on nesting
+
+# Declarations
+declaration = graph["Foo"]
+
+# All declarations include
+declaration.name
+declaration.unqualified_name
+declaration.definitions
+declaration.owner
+
+# Namespace declarations include
+declaration.member("bar()")
+declaration.member("@ivar")
+declaration.singleton_class
+declaration.ancestors
+declaration.descendants
+
+# Documents
+document = graph.documents.first
+document.uri
+document.definitions # => list of definitions discovered in this document
+
+# Definitions
+definition = declaration.definitions.first
+definition.location
+definition.comments
+definition.name
+definition.deprecated?
+definition.name_location
+
+# Locations
+location = definition.location
+location.path
+
+# Diagnostics
+diagnostic = graph.diagnostics.first
+diagnostic.rule
+diagnostic.message
+diagnostic.location
 ```
 
 ## Contributing

data/ext/rubydex/graph.c

@@ -9,7 +9,6 @@
 #include "utils.h"
 
 static VALUE cGraph;
-static VALUE eIndexingError;
 
 // Free function for the custom Graph allocator. We always have to call into Rust to free data allocated by it
 static void graph_free(void *ptr) {
@@ -27,8 +26,8 @@ static VALUE rdxr_graph_alloc(VALUE klass) {
     return TypedData_Wrap_Struct(klass, &graph_type, graph);
 }
 
-// Graph#index_all: (Array[String] file_paths) -> nil
-// Raises IndexingError if anything failed during indexing
+// Graph#index_all: (Array[String] file_paths) -> Array[String]
+// Returns an array of IO error messages encountered during indexing
 static VALUE rdxr_graph_index_all(VALUE self, VALUE file_paths) {
     rdxi_check_array_of_strings(file_paths);
 
@@ -36,10 +35,12 @@ static VALUE rdxr_graph_index_all(VALUE self, VALUE file_paths) {
     size_t length = RARRAY_LEN(file_paths);
     char **converted_file_paths = rdxi_str_array_to_char(file_paths, length);
 
-    // Get the underying graph pointer and then invoke the Rust index all implementation
+    // Get the underlying graph pointer and then invoke the Rust index all implementation
     void *graph;
     TypedData_Get_Struct(self, void *, &graph_type, graph);
-    const char *error_messages = rdx_index_all(graph, (const char **)converted_file_paths, length);
+
+    size_t error_count = 0;
+    const char *const *errors = rdx_index_all(graph, (const char **)converted_file_paths, length, &error_count);
 
     // Free the converted file paths and allow the GC to collect them
     for (size_t i = 0; i < length; i++) {
@@ -47,12 +48,36 @@ static VALUE rdxr_graph_index_all(VALUE self, VALUE file_paths) {
     }
     free(converted_file_paths);
 
-    // If indexing errors were returned, turn them into a Ruby string, call Rust to free the CString it allocated and
-    // return the Ruby string
-    if (error_messages != NULL) {
-        VALUE error_string = rb_utf8_str_new_cstr(error_messages);
-        free_c_string(error_messages);
-        rb_raise(eIndexingError, "%s", StringValueCStr(error_string));
+    if (errors == NULL) {
+        return rb_ary_new();
+    }
+
+    VALUE array = rb_ary_new_capa((long)error_count);
+    for (size_t i = 0; i < error_count; i++) {
+        rb_ary_push(array, rb_utf8_str_new_cstr(errors[i]));
+    }
+
+    free_c_string_array(errors, error_count);
+    return array;
+}
+
+// Indexes a single source string in memory, dispatching to the appropriate indexer based on language_id
+//
+// Graph#index_source: (String uri, String source, String language_id) -> void
+static VALUE rdxr_graph_index_source(VALUE self, VALUE uri, VALUE source, VALUE language_id) {
+    Check_Type(uri, T_STRING);
+    Check_Type(source, T_STRING);
+    Check_Type(language_id, T_STRING);
+
+    void *graph;
+    TypedData_Get_Struct(self, void *, &graph_type, graph);
+
+    const char *uri_str = StringValueCStr(uri);
+    const char *source_str = StringValueCStr(source);
+    const char *language_id_str = StringValueCStr(language_id);
+
+    if (!rdx_index_source(graph, uri_str, source_str, language_id_str)) {
+        rb_raise(rb_eArgError, "unsupported language_id `%s`", language_id_str);
     }
 
     return Qnil;
@@ -272,6 +297,25 @@ static VALUE rdxr_graph_method_references(VALUE self) {
     return self;
 }
 
+// Graph#delete_document: (String uri) -> Document?
+// Deletes a document and all of its definitions from the graph.
+// Returns the removed Document or nil if it doesn't exist.
+static VALUE rdxr_graph_delete_document(VALUE self, VALUE uri) {
+    Check_Type(uri, T_STRING);
+
+    void *graph;
+    TypedData_Get_Struct(self, void *, &graph_type, graph);
+    const uint64_t *uri_id = rdx_graph_delete_document(graph, StringValueCStr(uri));
+
+    if (uri_id == NULL) {
+        return Qnil;
+    }
+
+    VALUE argv[] = {self, ULL2NUM(*uri_id)};
+    free_u64(uri_id);
+    return rb_class_new_instance(2, argv, cDocument);
+}
+
 // Graph#resolve: () -> self
 // Runs the resolver to compute declarations and ownership
 static VALUE rdxr_graph_resolve(VALUE self) {
@@ -281,7 +325,7 @@ static VALUE rdxr_graph_resolve(VALUE self) {
     return self;
 }
 
-// Graph#set_encoding: (String) -> void
+// Graph#encoding=: (String) -> void
 // Sets the encoding used for transforming byte offsets into LSP code unit line/column positions
 static VALUE rdxr_graph_set_encoding(VALUE self, VALUE encoding) {
     Check_Type(encoding, T_STRING);
@@ -358,6 +402,38 @@ static VALUE rdxr_graph_resolve_require_path(VALUE self, VALUE require_path, VAL
     return rb_class_new_instance(2, argv, cDocument);
 }
 
+// Graph#require_paths: (Array[String] load_path) -> Array[String]
+// Returns all require paths for completion.
+static VALUE rdxr_graph_require_paths(VALUE self, VALUE load_path) {
+    rdxi_check_array_of_strings(load_path);
+
+    void *graph;
+    TypedData_Get_Struct(self, void *, &graph_type, graph);
+
+    size_t paths_len = RARRAY_LEN(load_path);
+    char **converted_paths = rdxi_str_array_to_char(load_path, paths_len);
+
+    size_t out_count = 0;
+    const char *const *results = rdx_require_paths(graph, (const char **)converted_paths, paths_len, &out_count);
+
+    for (size_t i = 0; i < paths_len; i++) {
+        free(converted_paths[i]);
+    }
+    free(converted_paths);
+
+    if (results == NULL) {
+        return rb_ary_new();
+    }
+
+    VALUE array = rb_ary_new_capa((long)out_count);
+    for (size_t i = 0; i < out_count; i++) {
+        rb_ary_push(array, rb_utf8_str_new_cstr(results[i]));
+    }
+
+    free_c_string_array(results, out_count);
+    return array;
+}
+
 // Graph#diagnostics -> Array[Rubydex::Diagnostic]
 static VALUE rdxr_graph_diagnostics(VALUE self) {
     void *graph;
@@ -392,12 +468,11 @@ static VALUE rdxr_graph_diagnostics(VALUE self) {
 }
 
 void rdxi_initialize_graph(VALUE mRubydex) {
-    VALUE eRubydexError = rb_const_get(mRubydex, rb_intern("Error"));
-    eIndexingError = rb_define_class_under(mRubydex, "IndexingError", eRubydexError);
-
     cGraph = rb_define_class_under(mRubydex, "Graph", rb_cObject);
     rb_define_alloc_func(cGraph, rdxr_graph_alloc);
     rb_define_method(cGraph, "index_all", rdxr_graph_index_all, 1);
+    rb_define_method(cGraph, "index_source", rdxr_graph_index_source, 3);
+    rb_define_method(cGraph, "delete_document", rdxr_graph_delete_document, 1);
     rb_define_method(cGraph, "resolve", rdxr_graph_resolve, 0);
     rb_define_method(cGraph, "resolve_constant", rdxr_graph_resolve_constant, 2);
     rb_define_method(cGraph, "declarations", rdxr_graph_declarations, 0);
@@ -407,6 +482,7 @@ void rdxi_initialize_graph(VALUE mRubydex) {
     rb_define_method(cGraph, "diagnostics", rdxr_graph_diagnostics, 0);
     rb_define_method(cGraph, "[]", rdxr_graph_aref, 1);
     rb_define_method(cGraph, "search", rdxr_graph_search, 1);
-    rb_define_method(cGraph, "set_encoding", rdxr_graph_set_encoding, 1);
+    rb_define_method(cGraph, "encoding=", rdxr_graph_set_encoding, 1);
     rb_define_method(cGraph, "resolve_require_path", rdxr_graph_resolve_require_path, 2);
+    rb_define_method(cGraph, "require_paths", rdxr_graph_require_paths, 1);
 }

data/lib/rubydex/graph.rb

@@ -5,40 +5,68 @@ module Rubydex
   #
   # Note: this class is partially defined in C to integrate with the Rust backend
   class Graph
+    IGNORED_DIRECTORIES = [
+      ".bundle",
+      ".git",
+      ".github",
+      "node_modules",
+      "tmp",
+    ].freeze
+
     #: (?workspace_path: String) -> void
     def initialize(workspace_path: Dir.pwd)
       @workspace_path = workspace_path
     end
 
     # Index all files and dependencies of the workspace that exists in `@workspace_path`
-    #: -> String?
+    #: -> Array[String]
     def index_workspace
-      paths = Dir.glob("#{@workspace_path}/**/*.rb")
-      paths.concat(
-        workspace_dependency_paths.flat_map do |path|
-          Dir.glob("#{path}/**/*.rb")
-        end,
-      )
-      index_all(paths)
+      index_all(workspace_paths)
+    end
+
+    # Returns all workspace paths that should be indexed, excluding directories that we don't need to descend into such
+    # as `.git`, `node_modules`. Also includes any top level Ruby files
+    #
+    #: -> Array[String]
+    def workspace_paths
+      paths = []
+
+      Dir.each_child(@workspace_path) do |entry|
+        full_path = File.join(@workspace_path, entry)
+
+        if File.directory?(full_path)
+          paths << full_path unless IGNORED_DIRECTORIES.include?(entry)
+        elsif File.extname(entry) == ".rb"
+          paths << full_path
+        end
+      end
+
+      add_workspace_dependency_paths(paths)
+      paths.uniq!
+      paths
     end
 
     private
 
     # Gathers the paths we have to index for all workspace dependencies
-    def workspace_dependency_paths
+    #: (Array[String]) -> void
+    def add_workspace_dependency_paths(paths)
       specs = Bundler.locked_gems&.specs
-      return [] unless specs
+      return unless specs
 
-      paths = specs.filter_map do |lazy_spec|
+      specs.each do |lazy_spec|
         spec = Gem::Specification.find_by_name(lazy_spec.name)
-        spec.require_paths.map { |path| File.join(spec.full_gem_path, path) }
+        spec.require_paths.each do |path|
+          # For native extensions, RubyGems inserts an absolute require path pointing to
+          # `gems/some-gem-1.0.0/extensions`. Those paths don't actually include any Ruby files inside, so we can skip
+          # descending them
+          next if File.absolute_path?(path)
+
+          paths << File.join(spec.full_gem_path, path)
+        end
       rescue Gem::MissingSpecError
         nil
       end
-
-      paths.flatten!
-      paths.uniq!
-      paths
     end
   end
 end

data/lib/rubydex/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Rubydex
-  VERSION = "0.1.0.beta3"
+  VERSION = "0.1.0.beta6"
 end

data/rust/rubydex/src/indexing/rbs_indexer.rs

@@ -0,0 +1,239 @@
+//! Visit the RBS AST and create type definitions.
+
+use ruby_rbs::node::{self, ModuleNode, Node, TypeNameNode, Visit};
+
+use crate::diagnostic::Rule;
+use crate::indexing::local_graph::LocalGraph;
+use crate::model::comment::Comment;
+use crate::model::definitions::{Definition, DefinitionFlags, ModuleDefinition};
+use crate::model::document::Document;
+use crate::model::ids::{DefinitionId, NameId, UriId};
+use crate::model::name::{Name, ParentScope};
+use crate::offset::Offset;
+
+pub struct RBSIndexer<'a> {
+    uri_id: UriId,
+    local_graph: LocalGraph,
+    source: &'a str,
+    nesting_stack: Vec<DefinitionId>,
+}
+
+impl<'a> RBSIndexer<'a> {
+    #[must_use]
+    pub fn new(uri: String, source: &'a str) -> Self {
+        let uri_id = UriId::from(&uri);
+        let local_graph = LocalGraph::new(uri_id, Document::new(uri, source));
+
+        Self {
+            uri_id,
+            local_graph,
+            source,
+            nesting_stack: Vec::new(),
+        }
+    }
+
+    #[must_use]
+    pub fn local_graph(self) -> LocalGraph {
+        self.local_graph
+    }
+
+    pub fn index(&mut self) {
+        let Ok(signature) = node::parse(self.source.as_bytes()) else {
+            self.local_graph.add_diagnostic(
+                Rule::ParseError,
+                Offset::new(0, 0),
+                "Failed to parse RBS document".to_string(),
+            );
+            return;
+        };
+
+        self.visit(&signature.as_node());
+    }
+
+    fn bytes_to_string(name: &[u8]) -> String {
+        String::from_utf8_lossy(name).into_owned()
+    }
+
+    /// Converts an RBS `TypeNameNode` into a rubydex `NameId`.
+    ///
+    /// Walks the namespace path (e.g. `Foo::Bar` in `Foo::Bar::Baz`) to build
+    /// a `ParentScope` chain, then creates the final `Name` for the leaf segment.
+    fn index_type_name(&mut self, type_name: &TypeNameNode, nesting_name_id: Option<NameId>) -> NameId {
+        let namespace = type_name.namespace();
+
+        let mut parent_scope = if namespace.absolute() {
+            ParentScope::TopLevel
+        } else {
+            ParentScope::None
+        };
+
+        for path_node in namespace.path().iter() {
+            let Node::Symbol(symbol) = path_node else {
+                continue;
+            };
+            parent_scope = ParentScope::Some(self.intern_name(symbol.name(), parent_scope, nesting_name_id));
+        }
+
+        self.intern_name(type_name.name().name(), parent_scope, nesting_name_id)
+    }
+
+    fn intern_name(&mut self, name_bytes: &[u8], parent_scope: ParentScope, nesting_name_id: Option<NameId>) -> NameId {
+        let string_id = self.local_graph.intern_string(Self::bytes_to_string(name_bytes));
+        self.local_graph
+            .add_name(Name::new(string_id, parent_scope, nesting_name_id))
+    }
+
+    fn parent_lexical_scope_id(&self) -> Option<DefinitionId> {
+        self.nesting_stack.last().copied()
+    }
+
+    fn add_member_to_current_lexical_scope(&mut self, owner_id: DefinitionId, member_id: DefinitionId) {
+        let owner = self
+            .local_graph
+            .get_definition_mut(owner_id)
+            .expect("owner definition should exist");
+
+        match owner {
+            Definition::Module(module) => module.add_member(member_id),
+            Definition::Class(class) => class.add_member(member_id),
+            _ => unreachable!("RBS nesting stack only contains modules/classes"),
+        }
+    }
+}
+
+impl Visit for RBSIndexer<'_> {
+    fn visit_module_node(&mut self, module_node: &ModuleNode) {
+        let lexical_nesting_id = self.parent_lexical_scope_id();
+        let nesting_name_id = lexical_nesting_id.map(|id| {
+            let owner = self
+                .local_graph
+                .definitions()
+                .get(&id)
+                .expect("owner definition should exist");
+            *owner.name_id().expect("nesting definition should have a name")
+        });
+
+        let type_name = module_node.name();
+        let name_id = self.index_type_name(&type_name, nesting_name_id);
+        let offset = Offset::from_rbs_location(&module_node.location());
+        let name_offset = Offset::from_rbs_location(&type_name.name().location());
+
+        let comments: Vec<_> = module_node
+            .comment()
+            .into_iter()
+            .map(|comment| {
+                let text = Self::bytes_to_string(comment.string().as_bytes());
+                Comment::new(Offset::from_rbs_location(&comment.location()), text)
+            })
+            .collect();
+
+        let definition = Definition::Module(Box::new(ModuleDefinition::new(
+            name_id,
+            self.uri_id,
+            offset,
+            name_offset,
+            comments,
+            DefinitionFlags::empty(),
+            lexical_nesting_id,
+        )));
+
+        let definition_id = self.local_graph.add_definition(definition);
+        if let Some(id) = lexical_nesting_id {
+            self.add_member_to_current_lexical_scope(id, definition_id);
+        }
+
+        self.nesting_stack.push(definition_id);
+
+        for member in module_node.members().iter() {
+            self.visit(&member);
+        }
+
+        self.nesting_stack.pop();
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::test_utils::LocalGraphTest;
+    use crate::{
+        assert_def_name_eq, assert_def_name_offset_eq, assert_definition_at, assert_local_diagnostics_eq,
+        assert_no_local_diagnostics,
+    };
+
+    fn index_source(source: &str) -> LocalGraphTest {
+        LocalGraphTest::new_rbs("file:///foo.rbs", source)
+    }
+
+    #[test]
+    fn index_source_with_errors() {
+        let context = index_source("module");
+
+        assert_local_diagnostics_eq!(&context, ["parse-error: Failed to parse RBS document (1:1-1:1)"]);
+
+        assert!(context.graph().definitions().is_empty());
+    }
+
+    #[test]
+    fn index_module_node() {
+        let context = index_source({
+            "
+            module Foo
+              module Bar
+              end
+            end
+            "
+        });
+
+        assert_no_local_diagnostics!(&context);
+        assert_eq!(context.graph().definitions().len(), 2);
+
+        assert_definition_at!(&context, "1:1-4:4", Module, |def| {
+            assert_def_name_eq!(&context, def, "Foo");
+            assert_def_name_offset_eq!(&context, def, "1:8-1:11");
+            assert_eq!(1, def.members().len());
+            assert!(def.lexical_nesting_id().is_none());
+        });
+
+        assert_definition_at!(&context, "2:3-3:6", Module, |def| {
+            assert_def_name_eq!(&context, def, "Bar");
+            assert_def_name_offset_eq!(&context, def, "2:10-2:13");
+
+            assert_definition_at!(&context, "1:1-4:4", Module, |parent_nesting| {
+                assert_eq!(parent_nesting.id(), def.lexical_nesting_id().unwrap());
+                assert_eq!(parent_nesting.members()[0], def.id());
+            });
+        });
+    }
+
+    #[test]
+    fn index_module_node_with_qualified_name() {
+        let context = index_source({
+            "
+            module Foo
+              module Bar::Baz
+              end
+            end
+            "
+        });
+
+        assert_no_local_diagnostics!(&context);
+        assert_eq!(context.graph().definitions().len(), 2);
+
+        assert_definition_at!(&context, "1:1-4:4", Module, |def| {
+            assert_def_name_eq!(&context, def, "Foo");
+            assert_def_name_offset_eq!(&context, def, "1:8-1:11");
+            assert_eq!(1, def.members().len());
+            assert!(def.lexical_nesting_id().is_none());
+        });
+
+        assert_definition_at!(&context, "2:3-3:6", Module, |def| {
+            assert_def_name_eq!(&context, def, "Bar::Baz");
+            assert_def_name_offset_eq!(&context, def, "2:15-2:18");
+
+            assert_definition_at!(&context, "1:1-4:4", Module, |parent_nesting| {
+                assert_eq!(parent_nesting.id(), def.lexical_nesting_id().unwrap());
+                assert_eq!(parent_nesting.members()[0], def.id());
+            });
+        });
+    }
+}