what_to_run 0.1.0 → 1.0.2

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    NWI3ZjRiMTMzYmQ5YzA0NmM4NjJkNTViM2VkOTJkZTI2NDQxNmE4MQ==
-  data.tar.gz: !binary |-
-    MmE2Yzc1YWVhMjA3NDliZjk4YTQ5NzdlNjljNTg4NzZjMjBjODM1Mg==
+SHA1:
+  metadata.gz: 5628bef322e5ffe04df75062217228f1c462251a
+  data.tar.gz: a3a0beabaabfb83223a0b588acb713a0e56a682f
 SHA512:
-  metadata.gz: !binary |-
-    ZmY3Yzk3ZmY1ZmMwMTMzNTc3ZWYyYzRjNjU5NjVhODY0NmNkODI2MTM4YzJi
-    NGNjYTRhNTg5MjRkMTdhY2NhNWNhYjBhZDE4M2VjYjEwOTI1MTJhNWFiNWRh
-    NDAxNjFiMjc1NDUyYTI0YTllYzBlNTk0NmQ2MWQ2MWYyM2NiMmQ=
-  data.tar.gz: !binary |-
-    ZGZmMzBiYjZkZmUzMzcxNDZhZWU5MjgxZTAwMDQwMjAwOGY0MTQwZTliMDdm
-    MmNjNjczNTdlN2YzNjg5YWUyZWFlYzkwZTlhMTIwMWY0YWQ1MDk1MjRhNjY0
-    MmIxYzM3M2NjYWYxYTM3OTM4ZTgzOTNhOGRlOWRkY2JhYjQzNjI=
+  metadata.gz: bbb66e545238e59c45d64b927b518a9b1b25bfb6cd7e24c17125ac5e00b32f3ead0bbf2346052cbeb7f5be69b10687e2106153303c5377ad7f73924d3a6a38f8
+  data.tar.gz: ff854a098d8f76780a4c7977cdb17ad57d3adfc1a1a1253910e4fca1a3bd2b71b3a2c3864c8145ef4a384830370900cda21ebb8e63b29ff006cc4aace266e702

data/README.md

@@ -1,14 +1,30 @@
 # What To Run
 
+[![Join the chat at https://gitter.im/DyegoCosta/what_to_run](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/DyegoCosta/what_to_run?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+
 [![Build Status](https://travis-ci.org/DyegoCosta/what_to_run.svg?branch=master)](https://travis-ci.org/DyegoCosta/what_to_run)
 
-What To Run is a lib for regression test selection, use it to predict which tests you should run when you make any modification on your codebase.
+What To Run is a lib for regression test selection for Ruby projects, use it to predict which tests you should run when you make any modification on your codebase.
+
+This lib was inspired by [@tenderlove](https://github.com/tenderlove) [blog post](tenderlove-post). Make sure to check it out.
+
 
-This lib is based on [@tenderlove](https://github.com/tenderlove) idea and guidance, make sure to read his [blog post](http://tenderlovemaking.com/2015/02/13/predicting-test-failues.html) on the subject.
+From the _[An Empirical Study of Regression Test Selection Techniques](rts-article)_ article:
+
+> Regression testing is the process of validating modified software to detect whether new errors
+have been introduced into previously tested code and to provide confidence that modifications
+are correct. Since regression testing is an expensive process, researchers have proposed
+regression test selection techniques as a way to reduce some of this expense. These techniques
+attempt to reduce costs by selecting and running only a subset of the test cases in a program’s
+existing test suite.
+
+[rts-article]: https://www.cs.umd.edu/~aporter/Docs/p184-graves.pdf
+[tenderlove-post]: http://tenderlovemaking.com/2015/02/13/predicting-test-failues.html
 
 ## Requirements
 
 - Project must be inside a Git repository
+- CMake to build the gem
 
 ## Installation
 
@@ -27,12 +43,12 @@ $ bundle
 Or install it yourself as:
 
 ```
-gem install what_to_run
+$ gem install what_to_run
 ```
 
 ## Usage
 
-Require the lib with:
+Require it after requiring your test framework and before load your files to be tested and your test suite config:
 
 Minitest
 
@@ -46,29 +62,41 @@ RSpec
 require 'what_to_run/rspec'
 ```
 
-Run your tests on a clean git branch
+Run your full tests suite with COLLECT=1 on a **clean git branch**
 
 Minitest
 
 ```
-$ COLLECTION=1 bundle exec rake test
+$ COLLECT=1 bundle exec rake test
 ```
 
 RSpec
 
 ```
-$ COLLECTION=1 bundle exec rspec
+$ COLLECT=1 bundle exec rspec
 ```
 
 This will create the initial coverage information. Then make your desired modifications on your code.
 
-Now to predict which tests is likely fail, run this:
+Now to run the tests that could reveal faults do the following
+
+```
+$ what_to_run <framework> [options]
+```
+
+Supported frameworks are:
 
 ```
-$ what_to_run
+rspec
+minitest
 ```
 
-:warning: A `run_log.json` file will be created in the current directory, you might want to include it in your `.gitignore`.
+Options are:
+
+```
+-e, --exec EXECUTABLE            Alternate test runner executable
+-h, --help                       Prints this help
+```
 
 ## Contributing
 

data/bin/what_to_run

@@ -2,4 +2,5 @@
 
 require 'what_to_run'
 
-puts WhatToRun.predict
+WhatToRun::CLI.new.run ARGV
+

data/lib/what_to_run.rb

@@ -1,91 +1,63 @@
-require 'json'
-require 'rugged'
 require 'set'
+require 'rugged'
+
+require_relative 'what_to_run/tracker'
 
 module WhatToRun
-  extend self
+  autoload :CLI, 'what_to_run/cli'
+  autoload :VERSION, 'what_to_run/version'
 
-  def predict
-    lines_to_run.inject([]) do |tests, (file, line)|
-      path = File.expand_path(file)
-      tests += Array cov_map[path][line]
+  class << self
+    def predict
+      lines_to_run.inject(Set.new) do |tests, (file, line)|
+        tests += Array cov_map[file][line]
+      end
     end
-  end
-
-  def lines_to_run
-    repo = Rugged::Repository.new('.')
-    lines_to_run = Set.new
 
-    repo.index.diff.each_patch do |patch|
-      file = patch.delta.old_file[:path]
-
-      patch.each_hunk do |hunk|
-        hunk.each_line do |line|
-          case line.line_origin
-          when :addition
-            lines_to_run << [file, line.new_lineno]
-          when :deletion
-            lines_to_run << [file, line.old_lineno]
-          when :context
-            # do nothing
+    def lines_to_run
+      repository = Rugged::Repository.discover('.')
+      repository_root = File.expand_path("..", repository.path)
+      lines_to_run = Set.new
+
+      repository.index.diff.each_patch do |patch|
+        file = patch.delta.old_file[:path]
+        file_path = File.join(repository_root, file)
+
+        patch.each_hunk do |hunk|
+          hunk.each_line do |line|
+            case line.line_origin
+            when :addition
+              lines_to_run << [file_path, line.new_lineno]
+            when :deletion
+              lines_to_run << [file_path, line.old_lineno]
+            when :context
+              # do nothing
+            end
           end
         end
       end
-    end
-
-    lines_to_run
-  end
-
-  def cov_delta(before, after)
-    after.each_with_object({}) do |(file_name, lines_cov), delta|
-      before_lines_cov = before[file_name]
-
-      # skip arrays that are exactly the same
-      next if before_lines_cov == lines_cov
 
-      # subtract the old coverage from the new coverage
-      cov = lines_cov_delta(before_lines_cov, lines_cov)
-
-      # add the "diffed" coverage to the hash
-      delta[file_name] = cov
+      lines_to_run
     end
-  end
-
-  def cov_map
-    cov_map = Hash.new { |h, file| h[file] = Hash.new { |i, line| i[line] = [] } }
 
-    File.open('run_log.json') do |f|
-      JSON.parse(f.read).each do |cov_info|
-        before, after = cov_info.last(2)
-        desc = build_test_desc(cov_info)
+    def cov_map
+      schema = Hash.new { |h, file| h[file] = Hash.new { |i, line| i[line] = [] } }
 
-        delta = cov_delta(before, after)
-
-        delta.each_pair do |file, lines|
+      @cov_map ||= Tracker.read.inject(schema) do |cov_map, (desc, cov_delta)|
+        cov_delta.each_pair do |file, lines|
           file_map = cov_map[file]
 
-          lines.each_with_index do |val, i|
-            file_map[i + 1] << desc if line_executed?(val)
+          lines.each_with_index do |line, i|
+            file_map[i + 1] << desc if line_executed?(line)
           end
         end
+
+        cov_map
       end
     end
 
-    cov_map
-  end
-
-  def build_test_desc(cov_info)
-    using_minitest = cov_info.length == 4
-    using_minitest ? cov_info.first(2).join('#') : cov_info.first
-  end
-
-  def line_executed?(line)
-    line.to_i > 0
-  end
-
-  def lines_cov_delta(before_lines_cov, after_lines_cov)
-    after_lines_cov.zip(before_lines_cov).map do |lines_after, lines_before|
-      lines_after ? lines_after - lines_before : lines_after
+    def line_executed?(line)
+      line.to_i > 0
     end
   end
 end

data/lib/what_to_run/cli.rb

@@ -0,0 +1,52 @@
+require 'optparse'
+require 'shellwords'
+
+module WhatToRun
+  class CLI
+    def run(argv)
+      framework = Array(argv)[0]
+
+      abort 'Must specify a test framework' unless framework
+
+      options = parse_options!(argv)
+
+      begin
+        runner = load_runner framework.downcase
+        runner.new(options).run
+      rescue LoadError
+        abort "Unsupported test framework: #{framework}"
+      end
+    end
+
+    private
+
+    def parse_options!(argv)
+      options = {}
+
+      parser = OptionParser.new do |opts|
+        opts.banner = 'Usage: what_to_run <framework> [options]'
+
+        opts.on('-e', '--exec EXECUTABLE', 'Alternate test runner executable') do |e|
+          options[:exec] = e
+        end
+
+        opts.on('-h', '--help', 'Prints this help') do
+          puts opts
+          exit
+        end
+      end
+
+      parser.parse!(argv)
+
+      options
+    end
+
+    def load_runner(framework)
+      require "what_to_run/#{framework}/runner"
+      klass_name = "WhatToRun::#{RUNNERS[framework]}::Runner"
+      klass_name.split('::').inject(Object) {|x, y| x.const_get(y.to_sym)}
+    end
+
+    RUNNERS = {'rspec' => 'RSpec', 'minitest' => 'Minitest'}
+  end
+end

data/lib/what_to_run/differ.rb

@@ -0,0 +1,77 @@
+module WhatToRun
+  class Differ
+    class << self
+      ##
+      # Gives the delta beteween the coverage result
+      # before and after a test run and before starting
+      # the test suite
+      #
+      # Results in the lines that may trigger the test
+      # that gave the after result
+      def coverage_delta(cov_before, cov_after, cov_before_suite)
+        cov_after.each_with_object({}) do |(file_name, lines_cov_after), delta|
+          lines_cov_before = cov_before[file_name]
+          lines_cov_before_suite = cov_before_suite[file_name]
+
+          next unless file_covered?(lines_cov_before, lines_cov_after)
+
+          lines_delta = lines_cov_delta \
+            lines_cov_before_suite, lines_cov_before, lines_cov_after
+
+          delta[file_name] = lines_delta
+        end
+      end
+
+      ##
+      # It needs to diff the diff of before and after
+      # with the coverage before the test suite in order
+      # to include the uncovered lines to all tests that
+      # touch this file
+      def lines_cov_delta(before_suite, before, after)
+        delta = diff before_suite, diff(before, after)
+        delta.map(&method(:normalize_cov_result))
+      end
+
+      def diff(before, after)
+        after = Array(after)
+        before = Array(before)
+
+        after.zip(before).map do |lines_after, lines_before|
+          lines_after ? lines_after - lines_before.to_i : lines_after
+        end
+      end
+
+      ##
+      # The possible param value that this method might receive
+      # are the following
+      #
+      # @param result positive => should run the test
+      # @param result negative => should run the test
+      # @param result nil      => should run the test
+      # @param result zero     => should not run the test
+      #
+      # This method will convert negative and nil values to 1,
+      # which will make them represent lines that should be run.
+      # The positive lines can be kept as they are since this mean
+      # they will be run.
+      #
+      # The only exception case is the 0 result which will be kept
+      # as it is so we don't run lines within the not called methods
+      #
+      # This introduces false positives to avoid missing tests that
+      # depends on lines that are evaluated when the file is required
+      #
+      # @return 1 to represent that this line should trigger the current test
+      # @return 0 to represent that this line should not trigger the current test
+      def normalize_cov_result(result)
+        result.nil? || result.to_i < 0 ? 1 : result
+      end
+
+      def file_covered?(cov_before, cov_after)
+        cov_before != cov_after
+      end
+    end
+
+    private_class_method :lines_cov_delta, :file_covered?
+  end
+end

data/lib/what_to_run/minitest.rb

@@ -1,26 +1,28 @@
-require 'json'
-require 'coverage'
-require 'coverage_peeker'
+configure = -> do
+  require 'coverage'
+  require 'coverage_peeker'
+  require 'what_to_run/tracker'
 
-Coverage.start
+  Coverage.start
 
-require 'minitest'
+  require 'minitest'
 
-class Minitest::Runnable
-  LOGS = []
+  WhatToRun::Tracker.start
 
-  Minitest.after_run {
-    File.open('run_log.json', 'w') { |f| f.write JSON.dump LOGS }
-  }
+  class Minitest::Runnable
+    Minitest.after_run {WhatToRun::Tracker.finish}
 
-  class << self
-    alias :old_run_one_method :run_one_method
+    class << self
+      alias :old_run_one_method :run_one_method
 
-    def run_one_method klass, method_name, reporter
-      before = CoveragePeeker.peek_result
-      old_run_one_method klass, method_name, reporter
-      after = CoveragePeeker.peek_result
-      LOGS << [ klass.name, method_name.to_s, before, after ]
+      def run_one_method(klass, method_name, reporter)
+        before = CoveragePeeker.peek_result
+        old_run_one_method klass, method_name, reporter
+        after = CoveragePeeker.peek_result
+        WhatToRun::Tracker.track "#{klass.name}##{method_name}", before, after
+      end
     end
   end
 end
+
+configure.call if ENV['COLLECT'] && ENV['COLLECT'] != 'false'

data/lib/what_to_run/minitest/runner.rb

@@ -0,0 +1,28 @@
+require 'what_to_run/runner'
+require 'shellwords'
+
+module WhatToRun
+  module Minitest
+    class Runner < WhatToRun::Runner
+      DEFAULT_EXECUTABLE = 'bundle exec rake test'.freeze
+
+      def initialize(opts = {})
+        super({exec: DEFAULT_EXECUTABLE}.merge(opts))
+      end
+
+      private
+
+      def predicted_example_args
+        patterns = predicted_examples.flat_map do |example|
+          "^#{example}$"
+        end
+
+        examples = patterns.join('|')
+
+        name_arg = Shellwords.escape("--name=/#{examples}/")
+
+        "TESTOPTS=\"#{name_arg}\""
+      end
+    end
+  end
+end

data/lib/what_to_run/rspec.rb

@@ -1,21 +1,27 @@
-require 'json'
-require 'rspec'
+configure = -> do
+  require 'coverage'
+  require 'coverage_peeker'
+  require 'what_to_run/tracker'
 
-require 'coverage'
-require 'coverage_peeker'
+  Coverage.start
 
-LOGS = []
-Coverage.start
+  RSpec.configuration.before(:suite) do
+    WhatToRun::Tracker.start
+  end
 
-RSpec.configuration.after(:suite) {
-  File.open('run_log.json', 'w') { |f| f.write JSON.dump LOGS }
-}
+  RSpec.configuration.after(:suite) do
+    WhatToRun::Tracker.finish
+    Coverage.result
+  end
 
-RSpec.configuration.around(:example) do |example|
-  before = CoveragePeeker.peek_result
-  puts 'before:' + before.object_id.to_s
-  example.call
-  after = CoveragePeeker.peek_result
-  puts 'after:' + after.object_id.to_s
-  LOGS << [ example.full_description, before, after ]
+  RSpec.configuration.around(:each) do |example|
+    before = CoveragePeeker.peek_result
+    example.call
+    after = CoveragePeeker.peek_result
+
+    WhatToRun::Tracker.track \
+      example.metadata[:full_description], before, after
+  end
 end
+
+configure.call if ENV['COLLECT'] && ENV['COLLECT'] != 'false'

data/lib/what_to_run/rspec/runner.rb

@@ -0,0 +1,21 @@
+require 'what_to_run/runner'
+require 'shellwords'
+
+module WhatToRun
+  module RSpec
+    class Runner < WhatToRun::Runner
+      DEFAULT_EXECUTABLE = 'bundle exec rspec'.freeze
+
+      def initialize(opts = {})
+        super({exec: DEFAULT_EXECUTABLE}.merge(opts))
+      end
+
+      private
+
+      def predicted_example_args
+        args = predicted_examples.flat_map { |example| ['-e', example] }
+        Shellwords.join(args)
+      end
+    end
+  end
+end

data/lib/what_to_run/runner.rb

@@ -0,0 +1,35 @@
+require 'what_to_run'
+
+module WhatToRun
+  ##
+  # Abstract base spec runner
+  class Runner
+    attr_reader :executable, :collect
+
+    def initialize(opts = {})
+      @executable = opts.fetch(:exec)
+    end
+
+    def run
+      if predicted_examples.empty?
+        exit 0
+      else
+        Kernel.exec command
+      end
+    end
+
+    private
+
+    def command
+      "#{executable} #{predicted_example_args}"
+    end
+
+    def predicted_example_args
+      fail NotImplementedError, 'Subclass must override #predicted_example_args'
+    end
+
+    def predicted_examples
+      @predicted_examples ||= WhatToRun.predict
+    end
+  end
+end

data/lib/what_to_run/tracker.rb

@@ -0,0 +1,74 @@
+require 'sqlite3'
+require 'fileutils'
+
+require 'coverage_peeker'
+require_relative 'differ'
+
+module WhatToRun
+  class Tracker
+    FileUtils.mkdir_p('.what_to_run')
+
+    DB_NAME = 'run_log'.freeze
+
+    DB = SQLite3::Database.open \
+      ".what_to_run/#{DB_NAME}#{ENV['TEST_ENV_NUMBER']}.db"
+
+    class << self
+      def start
+        DB.execute 'drop table if exists coverage'
+
+        DB.execute <<-SQL
+          create table if not exists coverage (
+            description varchar,
+            log blob
+          )
+        SQL
+
+        @before_suite = CoveragePeeker.peek_result
+      end
+
+      def track(description, before, after)
+        coverage = Marshal.dump \
+          Differ.coverage_delta(before, after, @before_suite)
+
+        DB.execute 'insert into coverage VALUES(?, ?)',
+          [description, SQLite3::Blob.new(coverage)]
+      end
+
+      def finish
+        DB.close
+      end
+
+      def read
+        query = 'select description, log from coverage'
+
+        unless additional_databases.empty?
+          attach_databases!
+          query += union_query
+        end
+
+        DB.execute(query).map { |row| [row[0], Marshal.load(row[1])] }
+      end
+
+      def additional_databases
+        count = Dir['.what_to_run/*.db'].length
+        (2..count).map { |n| "#{DB_NAME}#{n}" }
+      end
+
+      def attach_databases!
+        additional_databases.each do |db|
+          DB.execute "attach '.what_to_run/#{db}.db' as #{db}"
+        end
+      end
+
+      def union_query
+        additional_databases.inject('') do |query, db|
+          query += " union select description, log from #{db}.coverage"
+        end
+      end
+    end
+
+    private_class_method :additional_databases,
+      :attach_databases!, :union_query
+  end
+end

data/lib/what_to_run/version.rb

@@ -0,0 +1,3 @@
+module WhatToRun
+  VERSION = '1.0.2'
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: what_to_run
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 1.0.2
 platform: ruby
 authors:
 - Aaron Patterson
@@ -9,48 +9,68 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-30 00:00:00.000000000 Z
+date: 2015-04-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake-compiler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.9.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.9.0
 - !ruby/object:Gem::Dependency
   name: rugged
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.21'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.21.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.21'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.21.0
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.10
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.10
 description: Predict which tests are likely to fail after you’ve changed the code
 email: dyego@dyegocosta.com
 executables:
@@ -66,8 +86,15 @@ files:
 - ext/coverage_peeker/extconf.rb
 - lib/coverage_peeker.rb
 - lib/what_to_run.rb
+- lib/what_to_run/cli.rb
+- lib/what_to_run/differ.rb
 - lib/what_to_run/minitest.rb
+- lib/what_to_run/minitest/runner.rb
 - lib/what_to_run/rspec.rb
+- lib/what_to_run/rspec/runner.rb
+- lib/what_to_run/runner.rb
+- lib/what_to_run/tracker.rb
+- lib/what_to_run/version.rb
 homepage: https://github.com/DyegoCosta/what_to_run
 licenses:
 - MIT
@@ -78,12 +105,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []