referral 0.0.1

data/lib/referral/prints_results.rb

@@ -0,0 +1,67 @@
+require "time"
+require "referral/file_store"
+require "referral/git_store"
+
+module Referral
+  COLUMN_FUNCTIONS = {
+    id: ->(token) {
+      token.id
+    },
+    location: ->(token) {
+      "#{token.file}:#{token.line}:#{token.column}:"
+    },
+    file: ->(token) {
+      token.file
+    },
+    line: ->(token) {
+      token.line
+    },
+    column: ->(token) {
+      token.column
+    },
+    type: ->(token) {
+      token.type_name
+    },
+    scope: ->(token) {
+      token.scope
+    },
+    name: ->(token) {
+      token.literal_name
+    },
+    full_name: ->(token) {
+      token.full_name
+    },
+    source: ->(token) {
+      FileStore.read_line(token.file, token.line)
+    },
+    git_sha: ->(token) {
+      GitStore.sha(token.file, token.line)
+    },
+    git_author: ->(token) {
+      GitStore.author(token.file, token.line)
+    },
+    git_commit_at: ->(token) {
+      GitStore.time(token.file, token.line)&.utc&.iso8601
+    },
+
+  }
+
+  class PrintsResults
+    def call(result, options)
+      if options[:print_headers]
+        puts options[:columns].join(options[:delimiter])
+      end
+
+      result.tokens.each do |token|
+        cells = options[:columns].map { |column_name|
+          if (column = COLUMN_FUNCTIONS[column_name.to_sym])
+            column.call(token)
+          else
+            raise Referral::Error.new("Column '#{column_name}' not found in Referral::COLUMN_FUNCTIONS")
+          end
+        }
+        puts cells.join(options[:delimiter])
+      end
+    end
+  end
+end

data/lib/referral/runner.rb

@@ -0,0 +1,20 @@
+require "referral/filters_tokens"
+require "referral/sorts_tokens"
+require "referral/scans_tokens"
+require "referral/value/result"
+
+module Referral
+  class Runner
+    def call(options)
+      Value::Result.new(
+        tokens: SortsTokens.new.call(
+          FiltersTokens.new.call(
+            ScansTokens.new.call(files: options.files),
+            options
+          ),
+          options
+        )
+      )
+    end
+  end
+end

data/lib/referral/scans_tokens.rb

@@ -0,0 +1,41 @@
+require "referral/translates_node_to_token"
+require "referral/expands_directories"
+require "referral/tokenizes_identifiers"
+
+module Referral
+  class ScansTokens
+    def initialize
+      @expands_directories = ExpandsDirectories.new
+      @translates_node_to_token = TranslatesNodeToToken.new
+      @tokenizes_identifiers = TokenizesIdentifiers.new
+    end
+
+    def call(files:, &blk)
+      @expands_directories.call(files).flat_map { |file|
+        begin
+          root = RubyVM::AbstractSyntaxTree.parse_file(file)
+          find_tokens([root], nil, file)
+        rescue SyntaxError => e
+          warn "ERROR: Failed to parse \"#{file}\": #{e.message} (#{e.class})"
+        rescue SystemCallError => e
+          warn "ERROR: Failed to read \"#{file}\": #{e.message} (#{e.class})"
+        end
+      }.compact
+    end
+
+    private
+
+    def find_tokens(nodes, parent, file)
+      nodes.flat_map { |node|
+        next unless node.is_a?(RubyVM::AbstractSyntaxTree::Node)
+
+        if (token = @translates_node_to_token.call(node, parent, file))
+          @tokenizes_identifiers.call(node, token)
+          [token, *find_tokens(node.children[1..-1], token, file)]
+        else
+          find_tokens(node.children, parent, file)
+        end
+      }.compact
+    end
+  end
+end

data/lib/referral/sorts_tokens.rb

@@ -0,0 +1,46 @@
+require "referral/git_store"
+
+module Referral
+  SORT_FUNCTIONS = {
+    file: ->(tokens) {
+      tokens.sort_by { |token|
+        [token.file, token.line, token.column, token.id]
+      }
+    },
+    scope: ->(tokens) {
+      max_length = tokens.map { |t| t.fully_qualified.size }.max
+      tokens.sort_by { |token|
+        names = token.fully_qualified.map { |fq| fq.name.to_s }
+        [
+          *names.fill("", names.size...max_length),
+          token.file, token.line, token.column, token.id,
+        ]
+      }
+    },
+    least_recent_commit: ->(tokens) {
+      tokens.sort_by do |token|
+        [
+          GitStore.time(token.file, token.line).to_i,
+          token.file, token.line, token.column, token.id,
+        ]
+      end
+    },
+    most_recent_commit: ->(tokens) {
+      tokens.sort_by { |token|
+        [
+          -1 * GitStore.time(token.file, token.line).to_i,
+          token.file, token.line, token.column, token.id,
+        ]
+      }
+    },
+  }
+  class SortsTokens
+    def call(tokens, options)
+      if (sort_func = SORT_FUNCTIONS[options[:sort].to_sym])
+        sort_func.call(tokens)
+      else
+        raise Referral::Error.new("Sort '#{options[:sort]} not found in Referral::SORT_FUNCTIONS")
+      end
+    end
+  end
+end

data/lib/referral/token_types.rb

@@ -0,0 +1,186 @@
+require "referral/value/node_type"
+
+module Referral
+  JOIN_SEPARATORS = {
+    double_colon: "::",
+    dot: ".",
+    hash: "#",
+    tilde: "~",
+  }
+  TOKEN_TYPES = {
+    module: Value::NodeType.new(
+      name: :module,
+      ast_type: :MODULE,
+      join_separator: JOIN_SEPARATORS[:double_colon],
+      token_type: :definition,
+      reverse_identifiers: false,
+      good_parent: true,
+      name_finder: ->(node) { nil }
+    ),
+    class: Value::NodeType.new(
+      name: :class,
+      ast_type: :CLASS,
+      join_separator: JOIN_SEPARATORS[:double_colon],
+      token_type: :definition,
+      reverse_identifiers: false,
+      good_parent: true,
+      name_finder: ->(node) { nil }
+    ),
+    constant_def: Value::NodeType.new(
+      name: :constant_declaration,
+      ast_type: :CDECL,
+      join_separator: JOIN_SEPARATORS[:double_colon],
+      token_type: :definition,
+      reverse_identifiers: false,
+      good_parent: true,
+      name_finder: ->(node) {
+        possible_name = node.children[0]
+        possible_name.is_a?(Symbol) ? possible_name : nil
+      }
+    ),
+    class_method: Value::NodeType.new(
+      name: :class_method,
+      ast_type: :DEFS,
+      join_separator: JOIN_SEPARATORS[:dot],
+      token_type: :definition,
+      reverse_identifiers: false,
+      good_parent: true,
+      name_finder: ->(node) { node.children[1] }
+    ),
+    instance_method: Value::NodeType.new(
+      name: :instance_method,
+      ast_type: :DEFN,
+      join_separator: JOIN_SEPARATORS[:hash],
+      token_type: :definition,
+      reverse_identifiers: false,
+      good_parent: true,
+      name_finder: ->(node) { node.children[0] }
+    ),
+    local_var_assign: Value::NodeType.new(
+      name: :local_var_assign,
+      ast_type: :LASGN,
+      join_separator: JOIN_SEPARATORS[:tilde],
+      token_type: :definition,
+      reverse_identifiers: false,
+      good_parent: false,
+      name_finder: ->(node) { node.children[0] }
+    ),
+    instance_var_assign: Value::NodeType.new(
+      name: :instance_var_assign,
+      ast_type: :IASGN,
+      join_separator: JOIN_SEPARATORS[:tilde],
+      token_type: :definition,
+      reverse_identifiers: false,
+      good_parent: false,
+      name_finder: ->(node) { node.children[0] }
+    ),
+    class_var_assign: Value::NodeType.new(
+      name: :class_var_assign,
+      ast_type: :CVASGN,
+      join_separator: JOIN_SEPARATORS[:tilde],
+      token_type: :definition,
+      reverse_identifiers: false,
+      good_parent: false,
+      name_finder: ->(node) { node.children[0] }
+    ),
+    global_var_assign: Value::NodeType.new(
+      name: :global_var_assign,
+      ast_type: :GASGN,
+      join_separator: JOIN_SEPARATORS[:tilde],
+      token_type: :definition,
+      reverse_identifiers: false,
+      good_parent: false,
+      name_finder: ->(node) { node.children[0] }
+    ),
+    attr_assign: Value::NodeType.new(
+      name: :attr_assign,
+      ast_type: :ATTRASGN,
+      join_separator: JOIN_SEPARATORS[:dot],
+      token_type: :definition,
+      reverse_identifiers: false,
+      good_parent: false,
+      name_finder: ->(node) { node.children[1] }
+    ),
+    call: Value::NodeType.new(
+      name: :call,
+      ast_type: :CALL,
+      join_separator: JOIN_SEPARATORS[:dot],
+      token_type: :reference,
+      reverse_identifiers: true,
+      good_parent: true,
+      name_finder: ->(node) { node.children[1] }
+    ),
+    function_call: Value::NodeType.new(
+      name: :function_call,
+      ast_type: :FCALL,
+      join_separator: JOIN_SEPARATORS[:dot],
+      token_type: :reference,
+      reverse_identifiers: true,
+      good_parent: false,
+      name_finder: ->(node) { node.children[0] }
+    ),
+    local_var: Value::NodeType.new(
+      name: :local_var,
+      ast_type: :LVAR,
+      join_separator: JOIN_SEPARATORS[:tilde],
+      token_type: :reference,
+      reverse_identifiers: true,
+      good_parent: false,
+      name_finder: ->(node) { node.children[0] }
+    ),
+    instance_var: Value::NodeType.new(
+      name: :instance_var,
+      ast_type: :IVAR,
+      join_separator: JOIN_SEPARATORS[:tilde],
+      token_type: :reference,
+      reverse_identifiers: true,
+      good_parent: false,
+      name_finder: ->(node) { node.children[0] }
+    ),
+    class_var: Value::NodeType.new(
+      name: :class_var,
+      ast_type: :CVAR,
+      join_separator: JOIN_SEPARATORS[:tilde],
+      token_type: :reference,
+      reverse_identifiers: true,
+      good_parent: false,
+      name_finder: ->(node) { node.children[0] }
+    ),
+    global_var: Value::NodeType.new(
+      name: :global_var,
+      ast_type: :GVAR,
+      join_separator: JOIN_SEPARATORS[:tilde],
+      token_type: :reference,
+      reverse_identifiers: true,
+      good_parent: false,
+      name_finder: ->(node) { node.children[0] }
+    ),
+    constant_ref: Value::NodeType.new(
+      name: :constant,
+      ast_type: :CONST,
+      join_separator: JOIN_SEPARATORS[:double_colon],
+      token_type: :reference,
+      reverse_identifiers: true,
+      good_parent: false,
+      name_finder: ->(node) { node.children[0] }
+    ),
+    double_colon: Value::NodeType.new(
+      name: :constant,
+      ast_type: :COLON2,
+      join_separator: JOIN_SEPARATORS[:double_colon],
+      token_type: :reference,
+      reverse_identifiers: true,
+      good_parent: false,
+      name_finder: ->(node) { node.children[1] }
+    ),
+    triple_colon: Value::NodeType.new(
+      name: :constant,
+      ast_type: :COLON3,
+      join_separator: JOIN_SEPARATORS[:double_colon],
+      token_type: :reference,
+      reverse_identifiers: true,
+      good_parent: false,
+      name_finder: ->(node) { node.children[0] }
+    ),
+  }
+end

data/lib/referral/tokenizes_identifiers.rb

@@ -0,0 +1,32 @@
+require "referral/translates_node_to_token"
+
+module Referral
+  class TokenizesIdentifiers
+    def initialize
+      @translates_node_to_token = TranslatesNodeToToken.new
+    end
+
+    def call(root_node, root_token)
+      find_names(root_node, root_token)
+        .reject { |identifier_token| identifier_token.name.nil? }
+        .tap do |identifiers|
+        root_token.identifiers = identifiers
+
+        if identifiers.any? { |id| id.node_type == TOKEN_TYPES[:triple_colon] }
+          root_token.parent = nil
+        end
+      end
+    end
+
+    private
+
+    def find_names(node, parent)
+      return [] unless node.is_a?(RubyVM::AbstractSyntaxTree::Node)
+
+      [
+        *find_names(node.children.first, parent),
+        @translates_node_to_token.call(node, parent, parent.file),
+      ].compact
+    end
+  end
+end

data/lib/referral/translates_node_to_token.rb

@@ -0,0 +1,29 @@
+require "referral/token_types"
+require "referral/value/token"
+
+module Referral
+  class TranslatesNodeToToken
+    def call(node, parent, file)
+      return unless (type = TOKEN_TYPES.values.find { |d| node.type == d.ast_type })
+
+      Value::Token.new(
+        name: type.name_finder.call(node),
+        node_type: type,
+        parent: parent_unless_bad_parent(parent, type),
+        file: file,
+        line: node.first_lineno,
+        column: node.first_column
+      )
+    end
+
+    private
+
+    def parent_unless_bad_parent(parent_token, type)
+      return parent_token if parent_token&.node_type&.good_parent
+
+      # puts "should a #{parent_token.node_type.name} parent a #{type.name}?"
+      # return if parent_token.node_type.name == :local_var_assign
+      # parent_token
+    end
+  end
+end

data/lib/referral/value/node_type.rb

@@ -0,0 +1,15 @@
+module Referral
+  module Value
+    class NodeType < Struct.new(
+      :name,
+      :ast_type,
+      :join_separator,
+      :name_finder,
+      :token_type,
+      :reverse_identifiers,
+      :good_parent,
+      keyword_init: true
+    )
+    end
+  end
+end

data/lib/referral/value/options.rb

@@ -0,0 +1,38 @@
+module Referral
+  module Value
+    class Options < Struct.new(
+      :files,
+      # Filtering
+      :name,
+      :exact_name,
+      :full_name,
+      :pattern,
+      :type,
+      :include_unnamed,
+      # Sorting
+      :sort,
+      # Printing
+      :print_headers,
+      :columns,
+      :delimiter,
+      keyword_init: true
+    )
+
+      def self.default(overrides = {})
+        DEFAULT.merge({files: Dir["**/*.rb"]}.merge(overrides))
+      end
+
+      DEFAULT = new(
+        columns: ["location", "type", "scope", "name"],
+        delimiter: " ",
+        include_unnamed: false,
+        sort: "file",
+        print_headers: false,
+      ).freeze
+
+      def merge(new_opts)
+        self.class.new(to_h.merge(new_opts))
+      end
+    end
+  end
+end

data/lib/referral/value/result.rb

@@ -0,0 +1,6 @@
+module Referral
+  module Value
+    class Result < Struct.new(:tokens, keyword_init: true)
+    end
+  end
+end

data/lib/referral/value/token.rb

@@ -0,0 +1,68 @@
+require "digest/sha1"
+
+module Referral
+  module Value
+    class Token < Struct.new(
+      :name, :identifiers, :node_type, :parent, :file, :line, :column,
+      keyword_init: true
+    )
+
+      def fully_qualified
+        [
+          *parent&.fully_qualified,
+          *identity_components,
+        ].compact
+      end
+
+      def full_name
+        join_names(fully_qualified)
+      end
+
+      def scope
+        return "" unless parent
+        parent.full_name
+      end
+
+      def literal_name
+        if identifiers.empty?
+          name.to_s
+        else
+          join_names(identifiers)
+        end
+      end
+
+      def type_name
+        node_type.name.to_s
+      end
+
+      def id
+        Digest::SHA1.hexdigest(to_h.merge(
+          parent: nil,
+          identifiers: identifiers&.map(&:id),
+          node_type: node_type.name
+        ).inspect)[0..6]
+      end
+
+      protected
+
+      def join_names(tokens)
+        tokens.reduce("") { |s, token|
+          next s unless token.name
+          if s.empty?
+            token.name.to_s
+          else
+            "#{s}#{token.node_type.join_separator}#{token.name}"
+          end
+        }
+      end
+
+      def identity_components
+        if identifiers && !identifiers.empty?
+          identifiers
+        else
+          [self]
+        end
+      end
+    end
+  end
+end

data/lib/referral/version.rb

@@ -0,0 +1,3 @@
+module Referral
+  VERSION = "0.0.1"
+end

data/lib/referral.rb

@@ -0,0 +1,11 @@
+require "referral/runner"
+
+require "referral/version"
+require "referral/error"
+require "referral/cli"
+
+module Referral
+  def self.run(*args, **kwargs, &blk)
+    Runner.new.call(*args, **kwargs, &blk)
+  end
+end

data/referral.gemspec

@@ -0,0 +1,29 @@
+lib = File.expand_path("lib", __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "referral/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "referral"
+  spec.version       = Referral::VERSION
+  spec.authors       = ["Justin Searls"]
+  spec.email         = ["searls@gmail.com"]
+
+  spec.summary       = "Scan for definitions and references in your Ruby code."
+  spec.homepage      = "https://github.com/testdouble/referral"
+  spec.license       = "MIT"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+
+  spec.files = Dir.chdir(File.expand_path("..", __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.17.3"
+  spec.add_development_dependency "rake", "~> 12.3"
+  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "standard"
+end