reference_extractor 0.1.0

data/lib/reference_extractor/internal/node_helpers.rb

@@ -0,0 +1,304 @@
+# frozen_string_literal: true
+
+require "parser"
+require "parser/ast/node"
+
+module ReferenceExtractor
+  module Internal
+    # Convenience methods for working with Parser::AST::Node nodes.
+    module NodeHelpers
+      class TypeError < ArgumentError; end
+
+      class << self
+        def class_or_module_name(class_or_module_node)
+          case type_of(class_or_module_node)
+          when CLASS, MODULE
+            # (class (const nil :Foo) (const nil :Bar) (nil))
+            #   "class Foo < Bar; end"
+            # (module (const nil :Foo) (nil))
+            #   "module Foo; end"
+            identifier = class_or_module_node.children[0]
+            constant_name(identifier)
+          else
+            raise TypeError
+          end
+        end
+
+        def constant_name(constant_node)
+          case type_of(constant_node)
+          when CONSTANT_ROOT_NAMESPACE
+            ""
+          when CONSTANT, CONSTANT_ASSIGNMENT, SELF
+            # (const nil :Foo)
+            #   "Foo"
+            # (const (cbase) :Foo)
+            #   "::Foo"
+            # (const (lvar :a) :Foo)
+            #   "a::Foo"
+            # (casgn nil :Foo (int 1))
+            #   "Foo = 1"
+            # (casgn (cbase) :Foo (int 1))
+            #   "::Foo = 1"
+            # (casgn (lvar :a) :Foo (int 1))
+            #   "a::Foo = 1"
+            # (casgn (self) :Foo (int 1))
+            #   "self::Foo = 1"
+            namespace, name = constant_node.children
+
+            if namespace
+              [constant_name(namespace), name].join("::")
+            else
+              name.to_s
+            end
+          else
+            raise TypeError
+          end
+        end
+
+        def each_child(node, &block)
+          if block
+            node.children.each do |child|
+              yield(child) if child.is_a?(Parser::AST::Node)
+            end
+          else
+            enum_for(:each_child, node)
+          end
+        end
+
+        def enclosing_namespace_path(starting_node, ancestors:)
+          ancestors.select { |n| [CLASS, MODULE].include?(type_of(n)) }
+            .each_with_object([]) do |node, namespace|
+            # when evaluating `class Child < Parent`, the const node for `Parent` is a child of the class
+            # node, so it'll be an ancestor, but `Parent` is not evaluated in the namespace of `Child`, so
+            # we need to skip it here
+            next if type_of(node) == CLASS && parent_class(node) == starting_node
+
+            namespace.prepend(class_or_module_name(node))
+          end
+        end
+
+        def literal_value(string_or_symbol_node)
+          case type_of(string_or_symbol_node)
+          when STRING, SYMBOL
+            # (str "foo")
+            #   "'foo'"
+            # (sym :foo)
+            #   ":foo"
+            string_or_symbol_node.children[0]
+          else
+            raise TypeError
+          end
+        end
+
+        def location(node)
+          location = node.location
+          Node::Location.new(location.line, location.column)
+        end
+
+        def constant?(node)
+          type_of(node) == CONSTANT
+        end
+
+        def constant_assignment?(node)
+          type_of(node) == CONSTANT_ASSIGNMENT
+        end
+
+        def class?(node)
+          type_of(node) == CLASS
+        end
+
+        def method_call?(node)
+          type_of(node) == METHOD_CALL
+        end
+
+        def hash?(node)
+          type_of(node) == HASH
+        end
+
+        def string?(node)
+          type_of(node) == STRING
+        end
+
+        def symbol?(node)
+          type_of(node) == SYMBOL
+        end
+
+        def method_arguments(method_call_node)
+          raise TypeError unless method_call?(method_call_node)
+
+          # (send (lvar :foo) :bar (int 1))
+          #   "foo.bar(1)"
+          method_call_node.children.slice(2..-1)
+        end
+
+        def method_name(method_call_node)
+          raise TypeError unless method_call?(method_call_node)
+
+          # (send (lvar :foo) :bar (int 1))
+          #   "foo.bar(1)"
+          method_call_node.children[1]
+        end
+
+        def module_name_from_definition(node)
+          case type_of(node)
+          when CLASS, MODULE
+            # "class My::Class; end"
+            # "module My::Module; end"
+            class_or_module_name(node)
+          when CONSTANT_ASSIGNMENT
+            # "My::Class = ..."
+            # "My::Module = ..."
+            rvalue = node.children.last
+
+            case type_of(rvalue)
+            when METHOD_CALL
+              # "Class.new"
+              # "Module.new"
+              constant_name(node) if module_creation?(rvalue)
+            when BLOCK
+              # "Class.new do end"
+              # "Module.new do end"
+              constant_name(node) if module_creation?(method_call_node(rvalue))
+            end
+          end
+        end
+
+        def name_location(node)
+          location = node.location
+
+          if location.respond_to?(:name)
+            name = location.name
+            Node::Location.new(name.line, name.column)
+          end
+        end
+
+        def parent_class(class_node)
+          raise TypeError unless type_of(class_node) == CLASS
+
+          # (class (const nil :Foo) (const nil :Bar) (nil))
+          #   "class Foo < Bar; end"
+          class_node.children[1]
+        end
+
+        def parent_module_name(ancestors:)
+          definitions = ancestors
+            .select { |n| [CLASS, MODULE, CONSTANT_ASSIGNMENT, BLOCK].include?(type_of(n)) }
+
+          names = definitions.map do |definition|
+            name_part_from_definition(definition)
+          end.compact
+
+          names.empty? ? "Object" : names.reverse.join("::")
+        end
+
+        def value_from_hash(hash_node, key)
+          raise TypeError unless hash?(hash_node)
+
+          pair = hash_pairs(hash_node).detect { |pair_node| literal_value(hash_pair_key(pair_node)) == key }
+          hash_pair_value(pair) if pair
+        end
+
+        private
+
+        BLOCK = :block
+        CLASS = :class
+        CONSTANT = :const
+        CONSTANT_ASSIGNMENT = :casgn
+        CONSTANT_ROOT_NAMESPACE = :cbase
+        HASH = :hash
+        HASH_PAIR = :pair
+        METHOD_CALL = :send
+        MODULE = :module
+        SELF = :self
+        STRING = :str
+        SYMBOL = :sym
+
+        private_constant(
+          :BLOCK, :CLASS, :CONSTANT, :CONSTANT_ASSIGNMENT, :CONSTANT_ROOT_NAMESPACE, :HASH, :HASH_PAIR, :METHOD_CALL,
+          :MODULE, :SELF, :STRING, :SYMBOL
+        )
+
+        def type_of(node)
+          node.type
+        end
+
+        def hash_pair_key(hash_pair_node)
+          raise TypeError unless type_of(hash_pair_node) == HASH_PAIR
+
+          # (pair (int 1) (int 2))
+          #   "1 => 2"
+          # (pair (sym :answer) (int 42))
+          #   "answer: 42"
+          hash_pair_node.children[0]
+        end
+
+        def hash_pair_value(hash_pair_node)
+          raise TypeError unless type_of(hash_pair_node) == HASH_PAIR
+
+          # (pair (int 1) (int 2))
+          #   "1 => 2"
+          # (pair (sym :answer) (int 42))
+          #   "answer: 42"
+          hash_pair_node.children[1]
+        end
+
+        def hash_pairs(hash_node)
+          raise TypeError unless hash?(hash_node)
+
+          # (hash (pair (int 1) (int 2)) (pair (int 3) (int 4)))
+          #   "{1 => 2, 3 => 4}"
+          hash_node.children.select { |n| type_of(n) == HASH_PAIR }
+        end
+
+        def method_call_node(block_node)
+          raise TypeError unless type_of(block_node) == BLOCK
+
+          # (block (send (lvar :foo) :bar) (args) (int 42))
+          #   "foo.bar do 42 end"
+          block_node.children[0]
+        end
+
+        def module_creation?(node)
+          # "Class.new"
+          # "Module.new"
+          method_call?(node) &&
+            dynamic_class_creation?(receiver(node)) &&
+            method_name(node) == :new
+        end
+
+        def dynamic_class_creation?(node)
+          !!node &&
+            constant?(node) &&
+            ["Class", "Module"].include?(constant_name(node))
+        end
+
+        def name_from_block_definition(node)
+          if method_name(method_call_node(node)) == :class_eval
+            receiver = receiver(node)
+            constant_name(receiver) if receiver && constant?(receiver)
+          end
+        end
+
+        def name_part_from_definition(node)
+          case type_of(node)
+          when CLASS, MODULE, CONSTANT_ASSIGNMENT
+            module_name_from_definition(node)
+          when BLOCK
+            name_from_block_definition(node)
+          end
+        end
+
+        def receiver(method_call_or_block_node)
+          case type_of(method_call_or_block_node)
+          when METHOD_CALL
+            method_call_or_block_node.children[0]
+          when BLOCK
+            receiver(method_call_node(method_call_or_block_node))
+          else
+            raise TypeError
+          end
+        end
+      end
+    end
+  end
+end

data/lib/reference_extractor/internal/parsed_constant_definitions.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "ast/node"
+
+module ReferenceExtractor
+  module Internal
+    # A collection of constant definitions parsed from an Abstract Syntax Tree (AST).
+    class ParsedConstantDefinitions
+      class << self
+        # What fully qualified constants can this constant refer to in this context?
+        def reference_qualifications(constant_name, namespace_path:)
+          return [constant_name] if constant_name.start_with?("::")
+
+          resolved_constant_name = "::#{constant_name}"
+
+          possible_namespaces = namespace_path.each_with_object([""]) do |current, acc|
+            acc << "#{acc.last}::#{current}" if current
+          end
+
+          possible_namespaces.map { |namespace| namespace + resolved_constant_name }
+        end
+      end
+
+      def initialize(root_node:)
+        @local_definitions = {}
+
+        collect_local_definitions_from_root(root_node) if root_node
+      end
+
+      def local_reference?(constant_name, location: nil, namespace_path: [])
+        qualifications = self.class.reference_qualifications(constant_name, namespace_path: namespace_path)
+
+        qualifications.any? do |name|
+          @local_definitions[name] &&
+            @local_definitions[name] != location
+        end
+      end
+
+      private
+
+      def collect_local_definitions_from_root(node, current_namespace_path = [])
+        if NodeHelpers.constant_assignment?(node)
+          add_definition(NodeHelpers.constant_name(node), current_namespace_path, NodeHelpers.name_location(node))
+        elsif NodeHelpers.module_name_from_definition(node)
+          # handle compact constant nesting (e.g. "module Sales::Order")
+          tempnode = node
+          while (tempnode = NodeHelpers.each_child(tempnode).find { |node| NodeHelpers.constant?(node) })
+            add_definition(NodeHelpers.constant_name(tempnode), current_namespace_path,
+              NodeHelpers.name_location(tempnode))
+          end
+
+          current_namespace_path += NodeHelpers.class_or_module_name(node).split("::")
+        end
+
+        NodeHelpers.each_child(node) { |child| collect_local_definitions_from_root(child, current_namespace_path) }
+      end
+
+      def add_definition(constant_name, current_namespace_path, location)
+        resolved_constant = [""].concat(current_namespace_path).push(constant_name).join("::")
+
+        @local_definitions[resolved_constant] = location
+      end
+    end
+  end
+end

data/lib/reference_extractor/internal/parsers/erb.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "stringio"
+require "herb"
+
+module ReferenceExtractor
+  module Internal
+    module Parsers
+      class Erb
+        def initialize(parser: Herb.method(:extract_ruby), ruby_parser: Ruby.new)
+          @parser = parser
+          @ruby_parser = ruby_parser
+        end
+
+        def call(io:, file_path: "<unknown>")
+          erb_source = io.read
+          ruby_code = @parser.call(erb_source)
+
+          @ruby_parser.call(
+            io: StringIO.new(ruby_code),
+            file_path: file_path
+          )
+        end
+      end
+    end
+  end
+end

data/lib/reference_extractor/internal/parsers/factory.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "singleton"
+
+module ReferenceExtractor
+  module Internal
+    module Parsers
+      class Factory
+        include Singleton
+
+        RUBY_REGEX = %r{
+          # Although not important for regex, these are ordered from most likely to match to least likely.
+          \.(rb|rake|builder|gemspec|ru)\Z
+          |
+          (Gemfile|Rakefile)\Z
+        }x
+        private_constant :RUBY_REGEX
+
+        ERB_REGEX = /\.erb\Z/
+        private_constant :ERB_REGEX
+
+        def initialize
+          @ruby_parser = nil
+          @erb_parser = nil
+          @erb_parser_class = nil
+        end
+
+        def for_path(path)
+          case path
+          when RUBY_REGEX
+            @ruby_parser ||= Ruby.new
+          when ERB_REGEX
+            @erb_parser ||= erb_parser_class.new
+          end
+        end
+
+        def erb_parser_class
+          @erb_parser_class ||= Erb
+        end
+
+        def erb_parser_class=(klass)
+          @erb_parser_class = klass
+          @erb_parser = nil
+        end
+      end
+    end
+  end
+end

data/lib/reference_extractor/internal/parsers/parse_result.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+# require "parser/source/map"
+
+module ReferenceExtractor
+  module Internal
+    module Parsers
+      class ParseResult
+        attr_reader :location
+        attr_reader :file
+        attr_reader :message
+
+        def initialize(file:, message:, location: nil)
+          @location = location
+          @file = file
+          @message = message
+        end
+
+        def to_s(style = OutputStyles::Plain.new)
+          location = self.location
+          if location
+            <<~EOS
+              #{style.filename}#{file}#{style.reset}:#{location.line}:#{location.column}
+              #{@message}
+            EOS
+          else
+            <<~EOS
+              #{style.filename}#{file}#{style.reset}
+              #{@message}
+            EOS
+          end
+        end
+      end
+    end
+  end
+end

data/lib/reference_extractor/internal/parsers/ruby.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "parser"
+require "prism"
+
+module ReferenceExtractor
+  module Internal
+    module Parsers
+      class Ruby
+        class RaiseExceptionsParser < Prism::Translation::Parser
+          def initialize(builder)
+            super
+            super.diagnostics.all_errors_are_fatal = true
+          end
+        end
+
+        class TolerateInvalidUtf8Builder < Prism::Translation::Parser::Builder
+          def string_value(token)
+            value(token)
+          end
+        end
+
+        def initialize(parser_class: RaiseExceptionsParser)
+          @builder = TolerateInvalidUtf8Builder.new
+          @parser_class = parser_class
+        end
+
+        def call(io:, file_path: "<unknown>")
+          buffer = Parser::Source::Buffer.new(file_path)
+          buffer.source = io.read
+          parser = @parser_class.new(@builder)
+          parser.parse(buffer)
+        rescue EncodingError => e
+          result = ParseResult.new(file: file_path, message: e.message)
+          raise ParseError, result
+        rescue Parser::SyntaxError => e
+          result = ParseResult.new(file: file_path, message: "Syntax error: #{e}")
+          raise ParseError, result
+        end
+      end
+    end
+  end
+end

data/lib/reference_extractor/internal/parsers.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module ReferenceExtractor
+  module Internal
+    module Parsers
+      extend ActiveSupport::Autoload
+
+      autoload :Erb
+      autoload :Factory
+      autoload :Ruby
+      autoload :ParseResult
+    end
+  end
+end

data/lib/reference_extractor/internal/unresolved_reference.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module ReferenceExtractor
+  module Internal
+    # An unresolved reference from a file in one package to a constant that may be defined in a different package.
+    # Unresolved means that we know how it's referred to in the file,
+    # and we have enough context on that reference to figure out the fully qualified reference such that we
+    # can produce a Reference in a separate pass. However, we have not yet resolved it to its fully qualified version.
+    UnresolvedReference = Struct.new(
+      :constant_name,
+      :namespace_path,
+      :relative_path,
+      :source_location,
+      keyword_init: true
+    )
+  end
+end

data/lib/reference_extractor/internal.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module ReferenceExtractor
+  module Internal
+    extend ActiveSupport::Autoload
+
+    autoload :AstReferenceExtractor
+    autoload :ConstNodeInspector
+    autoload :ConstantDiscovery
+    autoload :Extractor
+    autoload :Node
+    autoload :NodeHelpers
+    autoload :ParsedConstantDefinitions
+    autoload :Parsers
+    autoload :UnresolvedReference
+  end
+end

data/lib/reference_extractor/parse_error.rb

@@ -0,0 +1,10 @@
+module ReferenceExtractor
+  class ParseError < StandardError
+    attr_reader(:result)
+
+    def initialize(result)
+      super(result.message)
+      @result = result
+    end
+  end
+end

data/lib/reference_extractor/reference.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module ReferenceExtractor
+  # A constant reference from one file to another.
+  Reference = Struct.new(
+    :relative_path,
+    :constant,
+    :source_location,
+    keyword_init: true
+  )
+end

data/lib/reference_extractor/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module ReferenceExtractor
+  VERSION = "0.1.0"
+end

data/lib/reference_extractor.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "active_support"
+# Provides String#pluralize, ends_with?, and others
+require "active_support/core_ext/string"
+
+# ReferenceExtractor extracts constant references from Ruby code, giving you your implicit application structure as a neat graph.
+#
+# @example Extract references from a string snippet
+#   extractor = ReferenceExtractor::Extractor.new(
+#     autoloaders: Rails.autoloaders,
+#     root_path: Rails.root
+#   )
+#   references = extractor.references_from_string("Order.find(1)")
+#   # => [#<ReferenceExtractor::Reference constant=#<ReferenceExtractor::ConstantContext name="::Order" ...>>]
+#
+# @example Extract references from a file
+#   references = extractor.references_from_file("app/models/user.rb")
+#   # => [#<ReferenceExtractor::Reference ...>, ...]
+module ReferenceExtractor
+  extend ActiveSupport::Autoload
+
+  # public API
+  autoload :ConstantContext
+  autoload :Extractor
+  autoload :ParseError
+  autoload :Reference
+
+  # private API
+  autoload :Internal
+
+  private_constant :Internal
+end
+
+require "reference_extractor/version"