referral 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 9deae17c8a8b085401ea981eb523db328a3d838ef3d0b6b84ea37eeccb333122
+  data.tar.gz: f89b3f60756982c430d0adb24a8c9c2617ee1c96f3e01465e9f042ae1ff107e1
+SHA512:
+  metadata.gz: 21a98b3027be2a52f5fbd52942a638074c3b2aba4c176916af9587d17467fc8d7351887eccfe8ef29a83fcf597c682de25b8681b29ad4f4c5efe31a84c8bb6f8
+  data.tar.gz: 4a7ef8a0a92c9faabb345159d5837fcd15917b3146f949a612ee60cd390266f9681d836d381898f2f1aa7bc9722c813843e956c01679cbdcf6ca8727613ae92a

data/.circleci/config.yml

@@ -0,0 +1,32 @@
+# Ruby CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-ruby/ for more details
+#
+version: 2
+jobs:
+  build:
+    docker:
+      - image: circleci/ruby:2.6
+
+    working_directory: ~/repo
+
+    steps:
+      - checkout
+
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum "Gemfile.lock" }}
+            - v1-dependencies-
+
+      - run:
+          name: install dependencies
+          command: |
+            bundle install --jobs=4 --retry=3 --path vendor/bundle
+
+      - save_cache:
+          paths:
+            - ./vendor/bundle
+          key: v1-dependencies-{{ checksum "Gemfile.lock" }}
+
+      - run: bundle exec rake
+

data/.gitignore

@@ -0,0 +1,8 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.standard.yml

@@ -0,0 +1,3 @@
+ignore:
+  - "test/fixture/**/*"
+

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,42 @@
+PATH
+  remote: .
+  specs:
+    referral (0.0.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.0)
+    jaro_winkler (1.5.3)
+    minitest (5.11.3)
+    parallel (1.17.0)
+    parser (2.6.3.0)
+      ast (~> 2.4.0)
+    psych (3.1.0)
+    rainbow (3.0.0)
+    rake (12.3.2)
+    rubocop (0.67.2)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.5, != 2.5.1.1)
+      psych (>= 3.1.0)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 1.6)
+    ruby-progressbar (1.10.1)
+    standard (0.0.41)
+      rubocop (~> 0.67.1)
+    unicode-display_width (1.5.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.17.3)
+  minitest (~> 5.0)
+  rake (~> 12.3)
+  referral!
+  standard
+
+BUNDLED WITH
+   1.17.3

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2019 Test Double, LLC
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,249 @@
+# referral
+
+Referral is a CLI toolkit for helping you undertake complex analysis and
+refactoring of Ruby codebases. It finds, filters, and sorts the definitions & references of most of the
+identifiers (e.g. classes, methods, and variables) throughout your code.
+
+Think of `referral` as a toolkit for tracking down references to the code that
+you want to change, offering a number of command-line options to quickly enable
+you to do things like:
+
+* Size up a codebase by gathering basic statistics and spotting usage hotspots
+* Build a to-do list to help you manage a large or complex refactor
+* Get a sense for how many callers would be impacted if you deleted a method
+* Before renaming a module, verify there aren't any already other modules with
+  the new name
+* Verify that you removed every reference to a deleted class before you merge
+* Identify dead code, like method definitions that aren't invoked anywhere
+* Catch references that haven't been updated since a change that affected them
+  (via `git-blame`)
+* Rather than wait for warnings at runtime, quickly make a list of every call to
+  deprecated methods
+
+Because Referral is powered by the introspection made possible by Ruby 2.6's
+[RubyVM::AbstractSyntaxTree](https://ruby-doc.org/core-2.6.3/RubyVM/AbstractSyntaxTree.html)
+API, it requires Ruby 2.6, but can often analyze codebases that target older
+Rubies.
+
+## Install
+
+From the command line:
+
+```
+$ gem install referral
+```
+
+Or in your `Gemfile`
+
+```ruby
+gem "referral", require: false, group: :development
+```
+
+## Usage
+
+### Basic usage
+
+At its most basic, you can just run `referral` and it'll scan `**/*.rb` from the
+current working directory and print everything:
+
+```
+$ referral
+app/channels/application_cable/channel.rb:1:0: module  ApplicationCable
+app/channels/application_cable/channel.rb:2:2: class ApplicationCable Channel
+app/channels/application_cable/channel.rb:2:18: constant ApplicationCable::Channel ActionCable::Channel::Base
+# … and then another 2400 lines (which you can easily count with `referral | wc -l`)
+```
+
+By default, Referral will sort entries by file, line, and column. Default output
+is broken into 4 columns: `location`, `type`, `scope`, and `name`.
+
+Everything above can be custom-tailored to your purposes, so let's work through
+some examples below.
+
+### Build a refactoring to-do spreadsheet
+
+When I'm undergoing a large refactor, I like to start by grepping around for all
+the obvious definitions and references that might be affected. Suppose I'm
+going to make major changes to my `User` class, I might search with the
+`--exact-name` filter like this:
+
+```
+referral --exact-name User,user,@user,@current_user
+```
+
+[Fun fact: if I'd have wanted to match on partial names, I could have used the looser
+`--name`, or for fully-qualified names (e.g. `API::User`), the stricter
+`--full-name` option.]
+
+Next, I usually find it easiest to work through a large refactor file-by-file,
+but in certain cases where I'm looking for a specific type of reference, it
+makes more sense to sort by the fully-qualified scope, which can be done with
+`--sort scope`:
+
+```
+referral --exact-name User,user,@user,@current_user --sort scope
+```
+
+Of course, if we want a checklist, the default output could be made a lot nicer
+for export to a spreadsheet app like [Numbers](https://www.apple.com/numbers/).
+
+Here's what that might look like:
+
+```
+referral --exact-name User,user,@user,@current_user --sort scope --print-headers --delimiter "\t" > user_refs.tsv
+```
+
+Where `--print-headers` prints an initial row of the selected column names, and `--delimiter
+"\t"` separates each field by a tab (making it easier to ingest for a
+spreadsheet app like Excel or Numbers), before being redirected to the file
+`user_refs.tsv`.
+
+Now, to open it in Numbers, I'd run:
+
+```
+open -a Numbers user_refs.tsv
+```
+
+And be immediately greeted by a spreadsheet. Heck, why not throw a checkbox on
+there while we're at it:
+
+<img width="1272" alt="Screen Shot 2019-06-27 at 1 27 42 PM" src="https://user-images.githubusercontent.com/79303/60287234-64560a00-98df-11e9-9fed-46c68fdaac58.png">
+
+### Detect references you forgot to update
+
+When working in a large codebase, it can be really tough to figure out if you
+remembered to update every reference to a class or method across thousands of
+files, so Referral ships with the ability to get some basic information from
+`git-blame`, like this:
+
+```
+referral --column file,line,git_sha,git_author,git_commit_at,full_name
+```
+
+By setting `--column` to a comma-separated array that includes the above,
+Referral will print results that look like these:
+
+```
+test/lib/splits_furigana_test.rb 56 634edc04 searls@gmail.com 2017-09-04T13:34:09Z SplitsFuriganaTest#test_nasty_edge_cases.assert_equal
+test/lib/splits_furigana_test.rb 56 634edc04 searls@gmail.com 2017-09-04T13:34:09Z h
+test/lib/splits_furigana_test.rb 56 634edc04 searls@gmail.com 2017-09-04T13:34:09Z @subject.call
+```
+
+[Warning: running `git-blame` on each file is, of course, a bit slow. Running
+this command on the [KameSame](https://kamesame.com) codebase took 3 seconds of
+wall-time, compared to 0.7 seconds by default.]
+
+And it gets better! Since we're already running blame, why not sort every line
+by its most and least recent commit time?!
+
+You can see your least-recently updated references first by adding `--sort
+least_recent_commit`, which does just what it says on the tin:
+
+```
+referral --column file,line,git_sha,git_author,git_commit_at,full_name --sort least_recent_commit
+```
+
+And I'll see that my least-recently-updated Ruby reference is:
+
+```
+app/channels/application_cable/channel.rb 1  searls@gmail.com 2017-08-20T14:59:35Z ApplicationCable
+```
+
+The inclusion of `git-blame` fields and sorting can be a powerful tool to
+spot-check a large refactor before deciding to merge it in.
+
+### Search for a regex pattern and print the source
+
+Once in a while, I'll want to scan line-by-line in a codebase for lines that
+match a given pattern, and in those cases, the `--pattern` option and `source`
+column can be a big help.
+
+Suppose I'm trying to size up a codebase by looking for how many methods appear
+to have a lot of arguments. While _definitely imperfect and regex cannot parse
+context-free grammars_, I can get a rough gist by searching for any lines that
+have 4 or more commas on them:
+
+```
+referral --pattern "/^([^,]*,){4,}[^,]*$/" -c location,source
+```
+
+Which would yield results like this one:
+
+```
+app/lib/card.rb:22:2:   def self.from_everything(id:, lesson_type:, item:, assignment:, meaning:)
+```
+
+Naturally, other programs like `find` could do this just as well, but the added
+ability to see & sort by when these lines were last updated in git might be
+interesting. Additionally, suppose you only wanted to find method _definitions_
+with a lot of (apparent) arguments? You could filter the matches down with
+`--type instance_method,class_method`, too, like this:
+
+```
+referral --pattern "/^([^,]*,){4,}[^,]*$/" -c location,git_commit_at,source -s most_recent_commit --type instance_method,class_method
+```
+
+And I can see that as recently as June 6th, I apparently wrote a very long
+method definition. `find` can't do that (I think)!
+
+```
+app/lib/presents_review_result.rb:60:2: 2019-06-02T02:38:01Z   def item_result(study_card_identifier, user, answer, item, learning, judgment, reward)
+```
+
+## Options
+
+The help output of `referral --help` will print out the available options and
+defaults:
+
+```
+Usage: referral [options] files
+    -v, --version                    Prints the version
+    -h, --help                       Prints this help
+    -n, --name [NAME]                Partial or complete name(s) to filter
+        --exact-name [NAME]          Exact name(s) to filter
+        --full-name [NAME]           Exact, fully-qualified name(s) to filter
+    -p, --pattern [PATTERN]          Regex pattern to filter
+    -t, --type [TYPES]              Include only certain types. See Referral::TOKEN_TYPES.
+        --include-unnamed            Include reference without identifiers (default: false)
+    -s, --sort {file,scope}          (default: file). See Referral::SORT_FUNCTIONS
+        --print-headers              Print header names (default: false)
+    -c, --columns [COL1,COL2,COL3]   (default: location,type,scope,name). See Referral::COLUMN_FUNCTIONS
+    -d, --delimiter [DELIM]          String separating columns (default: ' ')
+```
+
+A few things to note:
+
+* Each of `--name`, `--exact-name`, `--full-name`, `--type`, and `--columns`
+  accept comma-separated arrays (e.g. `-n foo,bar,baz`)
+
+* You can browse available sort functions [in
+  Refferral::SORT_FUNCTIONS](/lib/referral/sorts_tokens.rb) for use with
+  `--sort`. Each key is the name to be specified on the command line. (If you're
+  feeling adventurous, we've left the hash unfrozen so you can define your own
+  custom sorts dynamically, but YMMV.)
+
+* Just like sort functions, you can find the available column types [in
+  Refferral::COLUMN_FUNCTIONS](/lib/referral/prints_results.rb) when passing a
+  comma-separated list to `--column`. (This hash has
+  also been left mutable for you, dear user.)
+
+* The types of AST nodes that Referral supports can be found [in
+  Refferral::TOKEN_TYPES](/lib/referral/token_types.rb) when filtering to
+  certain `--type`
+
+* Note that the columns `git_sha`, `git_author`, `git_commit_at` and the sort
+  functions `most_recent_commit` and `least_recent_commit` will incur a
+  `git-blame` invocation for each file counted among the filtered results
+
+* Note that the `source` column and `--pattern` options will read each file in
+  the result set twice: once when parsing the AST, and again when printing
+  results
+
+## Code of Conduct
+
+This project follows Test Double's [code of
+conduct](https://testdouble.com/code-of-conduct) for all community interactions,
+including (but not limited to) one-on-one communications, public posts/comments,
+code reviews, pull requests, and GitHub issues. If violations occur, Test Double
+will take any action they deem appropriate for the infraction, up to and
+including blocking a user from the organization's repositories.

data/Rakefile

@@ -0,0 +1,16 @@
+begin
+  require "bundler/gem_tasks"
+rescue TypeError
+  # Support Gel
+end
+
+require "rake/testtask"
+require "standard/rake"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+task default: [:test, "standard:fix"]

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "referral"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/referral

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift("#{__dir__}/../lib")
+
+require "referral"
+
+Referral::Cli.new(ARGV.dup).call

data/lib/referral/cli.rb

@@ -0,0 +1,18 @@
+require "referral/parses_options"
+require "referral/prints_results"
+
+module Referral
+  class Cli
+    def initialize(argv)
+      @options = ParsesOptions.new.call(argv)
+    end
+
+    def call
+      PrintsResults.new.call(Runner.new.call(@options), @options)
+    rescue => e
+      warn "FATAL ERROR: #{e.message}"
+      warn e.backtrace
+      exit 1
+    end
+  end
+end

data/lib/referral/error.rb

@@ -0,0 +1,3 @@
+module Referral
+  class Error < StandardError; end
+end

data/lib/referral/expands_directories.rb

@@ -0,0 +1,13 @@
+module Referral
+  class ExpandsDirectories
+    def call(files_and_directories)
+      files_and_directories.flat_map { |f_or_d|
+        if File.directory?(f_or_d)
+          Dir["#{f_or_d}/**/*.rb"]
+        else
+          f_or_d
+        end
+      }
+    end
+  end
+end

data/lib/referral/file_store.rb

@@ -0,0 +1,13 @@
+module Referral
+  class FileStore
+    GROSS_FILE_CACHE = {}
+
+    def self.read_line(file, line)
+      read_file(file).split("\n")[line - 1]
+    end
+
+    def self.read_file(file)
+      GROSS_FILE_CACHE[file] ||= File.read(file)
+    end
+  end
+end

data/lib/referral/filters_tokens.rb

@@ -0,0 +1,51 @@
+require "referral/matches_token_names"
+require "referral/value/result"
+require "referral/file_store"
+
+module Referral
+  FILTER_FUNCTIONS = {
+    name: ->(token, names) {
+      names.any? { |name| token.full_name.include?(name) }
+    },
+    exact_name: ->(token, exact_names) {
+      exact_names.any? { |query|
+        MatchesTokenNames.subset(token, query)
+      }
+    },
+    full_name: ->(token, exact_names) {
+      exact_names.any? { |query|
+        MatchesTokenNames.entirely(token, query)
+      }
+    },
+    pattern: ->(token, regex) {
+      regex.match(token.full_name) || regex.match(FileStore.read_line(token.file, token.line))
+    },
+    type: ->(token, types) {
+      types.include?(token.node_type.name.to_s)
+    },
+    include_unnamed: ->(token, opt_val) {
+      if !opt_val
+        /\w/ =~ token.full_name
+      else
+        true
+      end
+    },
+  }
+  class FiltersTokens
+    def call(tokens, options)
+      filters = options.to_h.select { |opt_name, opt_val|
+        FILTER_FUNCTIONS.key?(opt_name) && !opt_val.nil?
+      }
+
+      if !filters.empty?
+        tokens.filter { |token|
+          filters.all? { |(opt_name, opt_val)|
+            FILTER_FUNCTIONS[opt_name].call(token, opt_val)
+          }
+        }
+      else
+        result
+      end
+    end
+  end
+end

data/lib/referral/git_store.rb

@@ -0,0 +1,42 @@
+require "open3"
+require "time"
+
+module Referral
+  class GitStore
+    GROSS_BLAME_CACHE = {}
+
+    def self.sha(file, line)
+      return unless (output = blame_line(file, line))
+      return unless (match = output.match(/^(\w+)/))
+      match[1]
+    end
+
+    def self.author(file, line)
+      return unless (output = blame_line(file, line))
+      return unless (match = output.match(/\(<([^>]*?)>/))
+      match[1]
+    end
+
+    def self.time(file, line)
+      return unless (output = blame_line(file, line))
+      return unless (match = output.match(/\(<.*?>\s+(\d+)\s+/))
+      Time.at(Integer(match[1]))
+    end
+
+    def self.blame_line(file, line)
+      return unless (output = blame(file))
+      output.split("\n")[line - 1]
+    end
+
+    # This format will look like:
+    # a50eb722 (<searls@gmail.com> 1561643971 -0400 2) class FirstThing
+    # or
+    # a50eb722 old/file/path.rb (<searls@gmail.com> 1561643971 -0400 2) class FirstThing
+    def self.blame(file)
+      GROSS_BLAME_CACHE[file] ||= begin
+        out, _, status = Open3.capture3("git blame -e -t \"#{file}\"")
+        status.success? ? out : ""
+      end
+    end
+  end
+end

data/lib/referral/matches_token_names.rb

@@ -0,0 +1,22 @@
+module Referral
+  class MatchesTokenNames
+    def self.subset(token, query)
+      token_tokens = names_from_token(token)
+      query_tokens = names_from_query(query)
+
+      token_tokens & query_tokens == query_tokens
+    end
+
+    def self.entirely(token, query)
+      names_from_token(token) == names_from_query(query)
+    end
+
+    def self.names_from_token(token)
+      token.fully_qualified.reject { |t| t.name.nil? }.map { |t| t.name.to_s }
+    end
+
+    def self.names_from_query(query)
+      query.split(Regexp.union(JOIN_SEPARATORS.values))
+    end
+  end
+end

data/lib/referral/parses_options.rb

@@ -0,0 +1,64 @@
+require "optparse"
+require "referral/value/options"
+
+module Referral
+  class ParsesOptions
+    def call(argv)
+      options = snake_case(run_optparse(argv))
+      Value::Options.default.merge(
+        merge_files(options, argv)
+      ).freeze
+    end
+
+    private
+
+    def run_optparse(argv)
+      {}.tap do |options|
+        op = OptionParser.new
+        op.banner += " files"
+        op.version = Referral::VERSION
+        version!(op)
+        help!(op)
+        op.on("-n", "--name [NAME]", Array, "Partial or complete name(s) to filter")
+        op.on("--exact-name [NAME]", Array, "Exact name(s) to filter")
+        op.on("--full-name [NAME]", Array, "Exact, fully-qualified name(s) to filter")
+        op.on("-p", "--pattern [PATTERN]", Regexp, "Regex pattern to filter")
+        op.on("-t", "--type [TYPES]", Array, "Include only certain types. See Referral::TOKEN_TYPES.")
+        op.on("--include-unnamed", TrueClass, "Include reference without identifiers (default: false)")
+        op.on("-s", "--sort {file,scope}", "(default: file). See Referral::SORT_FUNCTIONS")
+        op.on("--print-headers", TrueClass, "Print header names (default: false)")
+        op.on("-c", "--columns [COL1,COL2,COL3]", Array, "(default: location,type,scope,name). See Referral::COLUMN_FUNCTIONS")
+        op.on("-d", "--delimiter [DELIM]", "String separating columns (default: ' ')") do |v|
+          "\"#{v}\"".undump
+        end
+        op.parse!(argv, into: options)
+      end
+    end
+
+    def snake_case(options)
+      options.transform_keys { |k|
+        k.to_s.tr("-", "_").to_sym
+      }
+    end
+
+    def merge_files(options, argv)
+      options.merge(
+        files: argv.empty? ? Dir["**/*.rb"] : argv.dup
+      )
+    end
+
+    def version!(op)
+      op.on("-v", "--version", "Prints the version") do
+        puts VERSION
+        exit 0
+      end
+    end
+
+    def help!(op)
+      op.on("-h", "--help", "Prints this help") do
+        puts op
+        exit 0
+      end
+    end
+  end
+end