hind 0.1.14 → 0.1.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: edc5339a6fe71b30f48781500d996901bef7732b5d6ed6a54ce561d057edce19
-  data.tar.gz: 8dd64ea610eaf53314e918d91846d42839b522550e2b29a2a626228f7c459fbd
+  metadata.gz: 67b64fd17d7735528e37b9098db56f71a5332e01e58e9dfd958052da1b68e220
+  data.tar.gz: 7d5ab4b8b54fd439be53eba3f40df928ea35852d321945f972b09bfad5a4de24
 SHA512:
-  metadata.gz: d85651252a56a48c32c3285205cea32d0aa99e8098b396168cb1bf3357b61a9deff6ef90ceecee3b891761205bcb1d420235d4859b780629555080c040d9b25e
-  data.tar.gz: e69fa976556441508e1f46f7bb2636989a3cb810922636b1e460e3fa68130c7dc60538e55609349365506bf3fe21807a379c4afd5f54f4272582f02071164eef
+  metadata.gz: 462b167b1073fcdbe74dc9d1731e07b068a5b49c9f13e1204f7a8c58925a972dfef541f28388ed163efa59b8057df2f535f46ced8fdeb0e76308a505e58231fb
+  data.tar.gz: '035070842f07d55edc194ec60b8210858e01a68b8261f444c4e1c6af5d8418e5436ebc8a8aca071dbf608ba0baabdce3ae6b7d08adbf12e05317eeb0bb4387d7'

data/README.md

@@ -18,6 +18,10 @@ gem 'hind'
 
 ## Usage
 
+### GitLab CI Integration
+
+See [GitLab CI Integration Guide](docs/gitlab_ci.md) for detailed instructions.
+
 ### Generating LSIF Data
 
 To generate LSIF data for your Ruby project:
@@ -39,6 +43,16 @@ hind lsif -e "test/**/*" -e "spec/**/*"
 hind lsif -v
 ```
 
+### Generating SCIP Data
+
+To generate SCIP data for your Ruby project:
+
+```bash
+hind scip
+```
+
+This will produce an `index.scip` file which is a JSON serialization of the SCIP index.
+
 Options:
 - `-d, --directory DIR` - Root directory to process (default: current directory)
 - `-o, --output FILE` - Output file path (default: dump.lsif)
@@ -100,6 +114,25 @@ bundle exec rspec
 bundle exec bin/hind
 ```
 
+## Roadmap / TODO
+
+The following features are planned for future releases to improve indexing fidelity and performance:
+
+- **Ruby Semantic Fidelity**
+  - [ ] Ancestor Chain Resolution (`include`, `extend`, `prepend`)
+  - [ ] Local Variables & Method Parameters
+  - [ ] Dynamic Method Support (`attr_accessor`, `delegate`, etc.)
+  - [ ] Block & Lambda Scopes
+- **External Dependencies**
+  - [ ] External Gem Resolution (linking to symbols in dependencies)
+  - [ ] Core & Standard Library Linking
+- **Advanced SCIP/LSIF Features**
+  - [ ] SCIP Relationships (implementation/inheritance edges)
+  - [ ] LSIF Monikers for cross-repo resolution
+- **Performance**
+  - [ ] Parallel Processing for large codebases
+  - [ ] Incremental Indexing
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/yourusername/hind.

data/lib/hind/cli.rb

@@ -5,33 +5,98 @@ require 'json'
 require 'pathname'
 require 'fileutils'
 
+require_relative 'lsif/global_state'
+require_relative 'lsif/generator'
+require_relative 'scip/generator'
+
 module Hind
   class CLI < Thor
     class_option :verbose, type: :boolean, aliases: '-v', desc: 'Enable verbose output'
 
-    desc 'lsif', 'Generate LSIF index for Ruby classes, modules, and constants'
-    method_option :directory, type: :string, aliases: '-d', default: '.', desc: 'Root directory to process'
+    desc 'lsif [DIRECTORY]', 'Generate LSIF index for Ruby classes, modules, and constants'
+    method_option :directory, type: :string, aliases: '-d', desc: 'Root directory to process (deprecated, use positional argument)'
     method_option :output, type: :string, aliases: '-o', default: 'dump.lsif', desc: 'Output file path'
     method_option :glob, type: :string, aliases: '-g', default: '**/*.rb', desc: 'File pattern to match'
     method_option :force, type: :boolean, aliases: '-f', desc: 'Overwrite output file if it exists'
     method_option :exclude, type: :array, aliases: '-e', desc: 'Patterns to exclude'
-    def lsif
-      validate_directory(options[:directory])
+    def lsif(dir = options[:directory] || '.')
+      validate_directory(dir)
       validate_output_file(options[:output], options[:force])
 
-      files = find_files(options[:directory], options[:glob], options[:exclude])
-      abort "No files found matching pattern '#{options[:glob]}'" if files.empty?
+      files = find_files(dir, options[:glob], options[:exclude])
+      abort "No files found matching pattern '#{options[:glob]}' in #{dir}" if files.empty?
 
-      say "Found #{files.length} files to process", :green if options[:verbose]
+      say "Found #{files.length} files to process in #{dir}", :green if options[:verbose]
 
       begin
-        generate_lsif(files, options)
+        generate_lsif(files, dir, options)
+
+        # Add debug info from the global state
+        if options[:verbose]
+          debug_info = LSIF::GlobalState.instance.debug_info
+          say "\nGlobal State Summary:", :cyan
+          say "  Classes: #{debug_info[:classes_count]} (#{debug_info[:open_classes_count]} open classes)", :cyan
+          say "  Modules: #{debug_info[:modules_count]} (#{debug_info[:open_modules_count]} open modules)", :cyan
+          say "  Constants: #{debug_info[:constants_count]}", :cyan
+          say "  References: #{debug_info[:references_count]}", :cyan
+          say "  Result Sets: #{debug_info[:result_sets_count]}", :cyan
+
+          # Report on the most frequently reopened classes/modules
+          if debug_info[:open_classes_count] > 0
+            most_opened_classes = LSIF::GlobalState.instance.classes
+              .map { |name, data| [name, data[:definitions].size] }
+              .select { |_, count| count > 1 }
+              .sort_by { |_, count| -count }
+              .take(5)
+
+            say "\nMost frequently reopened classes:", :cyan
+            most_opened_classes.each do |name, count|
+              say "  #{name}: #{count} definitions", :cyan
+            end
+          end
+
+          if debug_info[:open_modules_count] > 0
+            most_opened_modules = LSIF::GlobalState.instance.modules
+              .map { |name, data| [name, data[:definitions].size] }
+              .select { |_, count| count > 1 }
+              .sort_by { |_, count| -count }
+              .take(5)
+
+            say "\nMost frequently reopened modules:", :cyan
+            most_opened_modules.each do |name, count|
+              say "  #{name}: #{count} definitions", :cyan
+            end
+          end
+        end
+
         say "\nLSIF data has been written to: #{options[:output]}", :green if options[:verbose]
       rescue => e
         handle_error(e, options[:verbose])
       end
     end
 
+    desc 'scip [DIRECTORY]', 'Generate SCIP index'
+    method_option :directory, type: :string, aliases: '-d', desc: 'Root directory to process (deprecated, use positional argument)'
+    method_option :output, type: :string, aliases: '-o', default: 'index.scip', desc: 'Output file path'
+    method_option :glob, type: :string, aliases: '-g', default: '**/*.rb', desc: 'File pattern to match'
+    method_option :force, type: :boolean, aliases: '-f', desc: 'Overwrite output file if it exists'
+    method_option :exclude, type: :array, aliases: '-e', desc: 'Patterns to exclude'
+    def scip(dir = options[:directory] || '.')
+      validate_directory(dir)
+      validate_output_file(options[:output], options[:force])
+      
+      files = find_files(dir, options[:glob], options[:exclude])
+      abort "No files found matching pattern '#{options[:glob]}' in #{dir}" if files.empty?
+
+      say "Found #{files.length} files to process in #{dir}", :green if options[:verbose]
+
+      generator = Hind::SCIP::Generator.new(File.expand_path(dir))
+      index = generator.execute(files)
+      
+      File.write(options[:output], index.to_proto, mode: 'wb')
+      say "SCIP index written to #{options[:output]}", :green
+    end
+
     desc 'version', 'Show version'
     def version
       say "Hind version #{Hind::VERSION}"
@@ -39,76 +104,26 @@ module Hind
 
     private
 
-    def generate_lsif(files, options)
+    def generate_lsif(files, directory, options)
       # Initialize generator with absolute project root
       generator = Hind::LSIF::Generator.new(
         {
           vertex_id: 1,
-          initial: true,
-          projectRoot: File.expand_path(options[:directory])
+          projectRoot: File.expand_path(directory)
         }
       )
 
       File.open(options[:output], 'w') do |output_file|
-        say 'First pass: Collecting declarations...', :cyan if options[:verbose]
+        say 'Processing files...', :cyan if options[:verbose]
 
-        # Write initial LSIF data (metadata and project vertices)
-        initial_data = generator.get_initial_data
-        if initial_data&.any?
-          say 'Writing initial LSIF data...', :cyan if options[:verbose]
-          output_file.puts(initial_data.map(&:to_json).join("\n"))
-        end
+        lsif_data = generator.execute(files, options)
 
-        # First pass: Process all files to collect declarations
-        declaration_data = {}
-        files.each do |file|
-          absolute_path = File.expand_path(file)
-          relative_path = Pathname.new(absolute_path)
-            .relative_path_from(Pathname.new(generator.metadata[:projectRoot]))
-            .to_s
-
-          begin
-            content = File.read(absolute_path)
-            file_declaration_data = generator.collect_file_declarations(content, relative_path)
-            declaration_data.merge!(file_declaration_data)
-          rescue => e
-            warn "Warning: Failed to read file '#{file}': #{e.message}"
-            next
-          end
+        # Get counts from global state
+        if options[:verbose]
+          say "Found #{LSIF::GlobalState.instance.classes.size} classes, #{LSIF::GlobalState.instance.modules.size} modules, and #{LSIF::GlobalState.instance.constants.size} constants", :cyan
         end
 
-        say "Found #{declaration_data[:declarations]&.size} declarations (classes, modules, constants)", :cyan if options[:verbose]
-
-        # Write declaration LSIF data next
-        if declaration_data[:lsif_data].any?
-          output_file.puts(declaration_data[:lsif_data].map(&:to_json).join("\n"))
-        end
-
-        say 'Processing files for references...', :cyan if options[:verbose]
-
-        # Second pass: Process each file for references
-        files.each do |file|
-          absolute_path = File.expand_path(file)
-          relative_path = Pathname.new(absolute_path)
-            .relative_path_from(Pathname.new(generator.metadata[:projectRoot]))
-            .to_s
-
-          if options[:verbose]
-            say "Processing file: #{relative_path}", :cyan
-          end
-
-          begin
-            content = File.read(absolute_path)
-            reference_lsif_data = generator.process_file(
-              content: content,
-              uri: relative_path
-            )
-            output_file.puts(reference_lsif_data.map(&:to_json).join("\n"))
-          rescue => e
-            warn "Warning: Failed to read file '#{file}': #{e.message}"
-            next
-          end
-        end
+        output_file.puts(lsif_data.map(&:to_json).join("\n"))
       end
     end
 
@@ -127,7 +142,12 @@ module Hind
 
     def find_files(directory, glob, exclude_patterns)
       pattern = File.join(directory, glob)
-      files = Dir.glob(pattern)
+      absolute_directory = File.expand_path(directory)
+      
+      files = Dir.glob(pattern).map do |file|
+        # Return relative path to the directory for indexing
+        Pathname.new(File.expand_path(file)).relative_path_from(Pathname.new(absolute_directory)).to_s
+      end
 
       exclude_patterns&.each do |exclude|
         files.reject! { |f| File.fnmatch?(exclude, f) }

data/lib/hind/lsif/generator.rb

@@ -7,13 +7,14 @@ require 'pathname'
 
 require_relative 'visitors/declaration_visitor'
 require_relative 'visitors/reference_visitor'
+require_relative 'global_state'
 
 module Hind
   module LSIF
     class Generator
       LSIF_VERSION = '0.4.3'
 
-      attr_reader :metadata, :global_state, :document_id, :current_uri
+      attr_reader :metadata, :document_id, :current_uri
 
       def initialize(metadata = {})
         @vertex_id = metadata[:vertex_id] || 1
@@ -22,73 +23,166 @@ module Hind
           projectRoot: File.expand_path(metadata[:projectRoot] || Dir.pwd)
         }.merge(metadata)
 
-        @global_state = GlobalState.new
+        # Reset the global state when initializing a new generator
+        GlobalState.instance.reset
         @document_ids = {}
         @current_document_id = nil
         @lsif_data = []
         @current_uri = nil
         @last_vertex_id = @vertex_id
-
-        initialize_project if metadata[:initial]
+        @last_reference_index = 0
+        @initial_data = []
+        @initial_data_emitted = false
       end
 
-      def collect_file_declarations(content, path)
-        @current_uri = path
-        @document_id = nil
-        @current_document_id = nil
+      def execute(files, options)
+        # We need to ensure metadata/project vertices are the very first ones.
+        # If we collect declarations first, we might emit vertices.
+        # So we should initialize project first.
+        
+        unless @initial_data_emitted
+           initialize_project
+           @initial_data_emitted = true
+           # Ensure these are at the start of @lsif_data
+           # specific logic: if we just added them, they are at the end (idx 0 if empty)
+           # but if we called collect_declarations separately?
+           # Actually, execute is the main entry point. @lsif_data is empty approx.
+        end
 
-        begin
-          ast = Parser.new(content).parse
-          setup_document
-          visitor = DeclarationVisitor.new(self, path)
-          visitor.visit(ast)
-          finalize_document_state
-        rescue => e
-          warn "Warning: Failed to collect declarations from '#{path}': #{e.message}"
+        # First pass: Declarations
+        files.each do |file|
+          @document_id = nil # Reset for each file
+          absolute_path = File.join(@metadata[:projectRoot], file)
+          next unless File.exist?(absolute_path)
+          
+          source = File.read(absolute_path)
+          collect_file_declarations(source, file)
         end
 
-        # Store the last used vertex ID and reference index
-        @last_vertex_id = @vertex_id
-        @last_reference_index = @lsif_data.length
+        # Second pass: References
+        files.each do |file|
+          @document_id = nil # Reset for each file
+          absolute_path = File.join(@metadata[:projectRoot], file)
+          next unless File.exist?(absolute_path)
+          
+          source = File.read(absolute_path)
+          process_file(content: source, uri: file)
+        end
+        
+        finalize_document_state
+        
+        @lsif_data
+      end
 
-        {
-          declarations: @global_state.declarations,
-          lsif_data: @lsif_data
-        }
+      def collect_file_declarations(content, uri)
+        @current_uri = uri
+        result = Prism.parse(content)
+        
+        declaration_visitor = DeclarationVisitor.new(self, uri)
+        result.value.accept(declaration_visitor)
+        
+        { lsif_data: @lsif_data - @initial_data }
+      ensure
+        @current_uri = nil
       end
 
-      def process_file(params)
-        @current_uri = params[:uri]
-        content = params[:content]
+      def process_file(content:, uri:)
+        @current_uri = uri
+        setup_document if @document_id.nil? || @document_ids[uri].nil?
+        @document_id = @document_ids[uri]
+        @current_document_id = @document_id
+        
+        result = Prism.parse(content)
+        
+        reference_visitor = ReferenceVisitor.new(self, uri)
+        result.value.accept(reference_visitor)
+        
+        finalize_document_state
+        @lsif_data
+      ensure
+        @current_uri = nil
+      end
+      def get_initial_data
+        @initial_data
+      end
 
-        # Restore vertex ID from last declaration pass
-        @vertex_id = @last_vertex_id
+      def register_class_declaration(declaration)
+        return unless @current_uri && declaration[:node]
 
-        @document_id = nil
-        @current_document_id = nil
+        qualified_name = declaration[:name]
 
-        setup_document
-        ast = Parser.new(content).parse
+        setup_document if @document_id.nil?
+        current_doc_id = @document_id
 
-        visitor = ReferenceVisitor.new(self, @current_uri)
-        visitor.visit(ast)
+        range_id = create_range(declaration[:node].constant_path.location)
+        return unless range_id
 
-        finalize_document_state
+        result_set_id = emit_vertex('resultSet')
+        emit_edge('next', range_id, result_set_id)
 
-        # Update last vertex ID
-        @last_vertex_id = @vertex_id
+        def_result_id = emit_vertex('definitionResult')
+        emit_edge('textDocument/definition', result_set_id, def_result_id)
+
+        emit_edge('item', def_result_id, [range_id], 'definitions', current_doc_id)
+
+        hover_content = generate_class_hover_content(declaration)
+        hover_id = emit_vertex('hoverResult', {
+          contents: [{
+            language: 'ruby',
+            value: hover_content
+          }]
+        })
+        emit_edge('textDocument/hover', result_set_id, hover_id)
+
+        declaration[:range_id] = range_id
+        declaration[:result_set_id] = result_set_id
+        declaration[:document_id] = current_doc_id
+        declaration[:file] = @current_uri
 
-        # Return only the new LSIF data since last call
-        result = @lsif_data[@last_reference_index..]
-        @last_reference_index = @lsif_data.length
-        result
+        GlobalState.instance.add_class(qualified_name, declaration)
+
+        result_set_id
       end
 
-      def get_initial_data
-        @initial_data
+      def register_module_declaration(declaration)
+        return unless @current_uri && declaration[:node]
+
+        qualified_name = declaration[:name]
+
+        setup_document if @document_id.nil?
+        current_doc_id = @document_id
+
+        range_id = create_range(declaration[:node].constant_path.location)
+        return unless range_id
+
+        result_set_id = emit_vertex('resultSet')
+        emit_edge('next', range_id, result_set_id)
+
+        def_result_id = emit_vertex('definitionResult')
+        emit_edge('textDocument/definition', result_set_id, def_result_id)
+
+        emit_edge('item', def_result_id, [range_id], 'definitions', current_doc_id)
+
+        hover_content = generate_module_hover_content(declaration)
+        hover_id = emit_vertex('hoverResult', {
+          contents: [{
+            language: 'ruby',
+            value: hover_content
+          }]
+        })
+        emit_edge('textDocument/hover', result_set_id, hover_id)
+
+        declaration[:range_id] = range_id
+        declaration[:result_set_id] = result_set_id
+        declaration[:document_id] = current_doc_id
+        declaration[:file] = @current_uri
+
+        GlobalState.instance.add_module(qualified_name, declaration)
+
+        result_set_id
       end
 
-      def register_declaration(declaration)
+      def register_constant_declaration(declaration)
         return unless @current_uri && declaration[:node]
 
         qualified_name = declaration[:name]
@@ -96,15 +190,7 @@ module Hind
         setup_document if @document_id.nil?
         current_doc_id = @document_id
 
-        range_id = if declaration[:type] == :constant
-          create_range(declaration[:node].name_loc)
-        elsif declaration[:type] == :module
-          create_range(declaration[:node].constant_path.location)
-        elsif declaration[:type] == :class
-          create_range(declaration[:node].constant_path.location)
-        else
-          create_range(declaration[:node].location)
-        end
+        range_id = create_range(declaration[:node].name_loc)
         return unless range_id
 
         result_set_id = emit_vertex('resultSet')
@@ -115,7 +201,7 @@ module Hind
 
         emit_edge('item', def_result_id, [range_id], 'definitions', current_doc_id)
 
-        hover_content = generate_hover_content(declaration)
+        hover_content = generate_constant_hover_content(declaration)
         hover_id = emit_vertex('hoverResult', {
           contents: [{
             language: 'ruby',
@@ -124,21 +210,19 @@ module Hind
         })
         emit_edge('textDocument/hover', result_set_id, hover_id)
 
-        @global_state.add_declaration(qualified_name, {
-          type: declaration[:type],
-          scope: declaration[:scope],
-          file: @current_uri,
-          range_id: range_id,
-          result_set_id: result_set_id,
-          document_id: current_doc_id
-        }.merge(declaration))
+        declaration[:range_id] = range_id
+        declaration[:result_set_id] = result_set_id
+        declaration[:document_id] = current_doc_id
+        declaration[:file] = @current_uri
+
+        GlobalState.instance.add_constant(qualified_name, declaration)
 
         result_set_id
       end
 
       def register_reference(reference)
         return unless @current_uri && reference[:node]
-        return unless @global_state.has_declaration?(reference[:name])
+        return unless GlobalState.instance.has_declaration?(reference[:name])
 
         setup_document if @document_id.nil?
         current_doc_id = @document_id
@@ -146,10 +230,11 @@ module Hind
         range_id = create_range(reference[:node].location)
         return unless range_id
 
-        declaration = @global_state.declarations[reference[:name]]
-        return unless declaration[:result_set_id]
+        # Get the primary declaration for this reference
+        declaration = GlobalState.instance.get_declaration(reference[:name])
+        return unless declaration && declaration[:result_set_id]
 
-        @global_state.add_reference(reference[:name], @current_uri, range_id, current_doc_id)
+        GlobalState.instance.add_reference(reference[:name], @current_uri, range_id, current_doc_id)
         emit_edge('next', range_id, declaration[:result_set_id])
 
         reference_result = emit_vertex('referenceResult')
@@ -160,7 +245,7 @@ module Hind
       private
 
       def initialize_project
-        emit_vertex('metaData', {
+        metadata_vertex = emit_vertex('metaData', {
           version: LSIF_VERSION,
           projectRoot: path_to_uri(@metadata[:projectRoot]),
           positionEncoding: 'utf-16',
@@ -170,11 +255,18 @@ module Hind
           }
         })
 
-        @global_state.project_id = emit_vertex('project', {kind: 'ruby'})
+        project_id = emit_vertex('project', {kind: 'ruby'})
+        GlobalState.instance.project_id = project_id
+
+        # Store initial data separately
+        @initial_data = @lsif_data.dup
       end
 
       def setup_document
-        return if @document_id
+        if @document_ids[@current_uri]
+          @document_id = @document_ids[@current_uri]
+          return
+        end
         return unless @current_uri
 
         file_path = File.join(@metadata[:projectRoot], @current_uri)
@@ -187,13 +279,13 @@ module Hind
         @document_ids[@current_uri] = @document_id
         @current_document_id = @document_id
 
-        emit_edge('contains', @global_state.project_id, [@document_id]) if @global_state.project_id
+        emit_edge('contains', GlobalState.instance.project_id, [@document_id]) if GlobalState.instance.project_id
       end
 
       def finalize_document_state
         return unless @current_uri && @document_id
 
-        ranges = @global_state.get_ranges_for_file(@current_uri)
+        ranges = GlobalState.instance.get_ranges_for_file(@current_uri)
         if ranges&.any?
           emit_edge('contains', @document_id, ranges, nil, @document_id)
         end
@@ -213,7 +305,7 @@ module Hind
           }
         })
 
-        @global_state.add_range(@current_uri, range_id)
+        GlobalState.instance.add_range(@current_uri, range_id)
         range_id
       end
 
@@ -263,20 +355,19 @@ module Hind
         @vertex_id - 1
       end
 
-      def generate_hover_content(declaration)
-        case declaration[:type]
-        when :class
-          hover = ["class #{declaration[:name]}"]
-          hover << " < #{declaration[:superclass]}" if declaration[:superclass]
-          hover.join
-        when :module
-          "module #{declaration[:name]}"
-        when :constant
-          value_info = declaration[:node].value.respond_to?(:content) ? " = #{declaration[:node].value.content}" : ''
-          "#{declaration[:name]}#{value_info}"
-        else
-          declaration[:name].to_s
-        end
+      def generate_class_hover_content(declaration)
+        hover = ["class #{declaration[:name]}"]
+        hover << " < #{declaration[:superclass]}" if declaration[:superclass]
+        hover.join
+      end
+
+      def generate_module_hover_content(declaration)
+        "module #{declaration[:name]}"
+      end
+
+      def generate_constant_hover_content(declaration)
+        value_info = declaration[:node].value.respond_to?(:content) ? " = #{declaration[:node].value.content}" : ''
+        "#{declaration[:name]}#{value_info}"
       end
 
       def format_hover_data(data)