covet 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f7fd3d912a0b3a182a3b8004c0d4bd4b7a75c313
+  data.tar.gz: 090be05aac594fac1273ef2ea6c90a732367d021
+SHA512:
+  metadata.gz: 90adde7bc36fbfe9550e16212bdd52da65161adb3de5c003227b9a5782990762ce9b6c552148748652d6a559f48d7f190b6084a14e107c24d8dcc8c10b75a726
+  data.tar.gz: a3ddfd5fd2997c04542bb35e3e7c2ce34ba29f6bf1c6853d0502b84eb2dcc7e482b3b67ef6ab71c19c75a965fedd2ff924a30d4ff8820fc6030c0d3c2cce1f25

data/Gemfile

@@ -0,0 +1,12 @@
+source 'https://rubygems.org'
+gemspec
+
+group :test, :development do
+  gem 'rspec'
+  gem 'minitest'
+  #if RUBY_VERSION.to_i < 2
+    #gem 'debugger'
+  #else
+    #gem 'byebug'
+  #end
+end

data/README.md

@@ -0,0 +1,82 @@
+Covet
+=====
+
+What Is It?
+-----------
+
+It's a regression test selection tool for ruby, an implementation
+of some ideas by Mr. Tenderlove expressed
+[here](http://tenderlovemaking.com/2015/02/13/predicting-test-failues.html).
+
+How Does It Work?
+-----------------
+
+Reading the article will give you a good idea, but here's a short summary:
+
+1) Gather coverage information during test runs, before and after
+each test method in order to know which test methods ran which
+files and lines of the tested application code.
+
+2) Change the application code.
+
+3) Covet shows you which tests to run based on the coverage information
+gathered in step 1, and the fact that `git` knows that you changed
+certain lines of the application code.
+
+Usage
+-----
+
+Coverage Collection:
+
+Run your test suite with coverage collection on. To enable this,
+add `require 'covet'` before any tests run (in a test helper file or similar),
+and run your suite with: `covet -c $CMD`, where $CMD is the command to run your
+test suite. Example:
+
+    $ covet -c "rake test"
+
+Covet should output a message before any other message:
+
+    Collecting coverage information for each test method...
+
+By default, `covet` hooks into `minitest` and collects coverage before
+and after each method. If you're using `rspec`, make sure to pass the `-t`
+option:
+
+    $ covet -t rspec -c "rake test"
+
+After this, you should have 2 new files: `run_log.json`, and
+`run_log_index.json`.
+
+Now, by default the `covet` command will print out which test
+files should be run based off the changes in your git repo since
+the last commit.
+
+For example:
+
+    $ covet
+
+    You need to run:
+    - /home/luke/Desktop/code/rails/activesupport/test/test\_case\_test.rb
+
+To execute the run list, simply:
+
+    $ covet -e
+
+Testing Gems
+------------
+
+By default, `covet` removes all standard library and gem files from the `run_log`, because
+it assumes you're testing your own library code. In order to test a gem, you need to add the
+`--whitelist-gems` option. For example:
+
+    $ covet -c "rake test" --whitelist-gems "activesupport,rails"
+
+Caveats/Bugs
+------------
+
+1) It's not tested thoroughly enough.
+
+2) Don't rely on this library to be correct yet (ie: don't forgo full test
+suite runs before committing to a repository). It's still early days.
+Please contribute code, docs, or ideas, though!

data/Rakefile

@@ -0,0 +1,37 @@
+require 'rake/testtask'
+require 'rake/extensiontask'
+require 'rspec/core/rake_task'
+require 'rake/clean'
+
+task :default => [:tests_and_specs]
+
+Rake::TestTask.new(:test) do |t|
+  t.test_files = FileList['test/**/*_test.rb'].to_a
+  t.verbose = true
+  if t.respond_to?(:description=)
+    t.description = 'Run all minitest tests'
+  end
+end
+
+desc 'Run all rspec specs'
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.pattern = Dir.glob('spec/**/*_spec.rb')
+  t.verbose = true
+end
+
+desc 'Run minitest and rspec tests (default)'
+task :tests_and_specs => [:test, :spec]
+
+desc 'compile internal coverage C extension'
+Rake::ExtensionTask.new('covet_coverage')
+
+desc 'recompile internal coverage C extension'
+task :recompile => [:clobber, :compile, :tests_and_specs]
+
+# for rake:clobber
+CLOBBER.include(
+  'run_log.json',
+  'run_log_index.json',
+  'ext/covet_coverage/*.{so,o}',
+  'ext/covet_coverage/Makefile',
+)

data/bin/covet

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require_relative '../lib/covet'
+Covet::CLI.new(ARGV.dup).run

data/ext/covet_coverage/covet_coverage.c

@@ -0,0 +1,153 @@
+/************************************************
+
+  coverage.c -
+
+  $Author: $
+
+  Copyright (c) 2008 Yusuke Endoh
+
+************************************************/
+
+#include "ruby.h"
+#include "extconf.h"
+
+extern VALUE rb_get_coverages(void);
+extern void rb_set_coverages(VALUE);
+extern void rb_reset_coverages(void);
+
+#ifdef HAVE_RB_OBJ_HIDE
+extern VALUE rb_obj_hide(VALUE);
+static void hide_obj(VALUE val)
+{
+    rb_obj_hide(val);
+}
+#else
+static void hide_obj(VALUE val)
+{
+    RBASIC(val)->klass = 0;
+}
+#endif
+
+static VALUE rb_coverages = Qundef;
+
+
+/*
+ * call-seq:
+ *    CovetCoverage.start  => nil
+ * * Enables coverage measurement.
+ */
+static VALUE
+rb_coverage_start(void)
+{
+    if (!RTEST(rb_get_coverages())) {
+        if (rb_coverages == Qundef) {
+            rb_coverages = rb_hash_new();
+            hide_obj(rb_coverages);
+        }
+        rb_set_coverages(rb_coverages);
+    }
+    return Qnil;
+}
+
+static int
+coverage_clear_result_i(st_data_t key, st_data_t val, st_data_t h)
+{
+    VALUE coverage = (VALUE)val;
+    rb_ary_clear(coverage);
+    return ST_CONTINUE;
+}
+
+static int
+coverage_peek_result_i(st_data_t key, st_data_t val, st_data_t h)
+{
+    VALUE path = (VALUE)key;
+    VALUE coverage = (VALUE)val;
+    VALUE coverages = (VALUE)h;
+    coverage = rb_ary_dup(coverage);
+    rb_ary_freeze(coverage);
+    rb_hash_aset(coverages, path, coverage);
+    return ST_CONTINUE;
+}
+
+/*
+ *  call-seq:
+ *     CovetCoverage.peek_result  => hash
+ *
+ * Returns a hash that contains filename as key and coverage array as value.
+ */
+static VALUE
+rb_coverage_peek_result(void)
+{
+    VALUE coverages = rb_get_coverages();
+    VALUE ncoverages = rb_hash_new();
+    if (!RTEST(coverages)) {
+        rb_raise(rb_eRuntimeError, "coverage measurement is not enabled (covet)");
+    }
+    st_foreach(RHASH_TBL(coverages), coverage_peek_result_i, ncoverages);
+    rb_hash_freeze(ncoverages);
+    return ncoverages;
+}
+
+static int
+coverage_result_i(st_data_t key, st_data_t val, st_data_t h)
+{
+    VALUE path = (VALUE)key;
+    VALUE coverage = (VALUE)val;
+    VALUE coverages = (VALUE)h;
+    coverage = rb_ary_dup(coverage);
+    rb_ary_clear((VALUE)val);
+    rb_ary_freeze(coverage);
+    rb_hash_aset(coverages, path, coverage);
+    return ST_CONTINUE;
+}
+
+/*
+ *  call-seq:
+ *     CovetCoverage.result  => hash
+ *
+ * Returns a hash that contains filename as key and coverage array as value
+ * and disables coverage measurement.
+ */
+static VALUE
+rb_coverage_result(void)
+{
+    VALUE coverages = rb_get_coverages();
+    VALUE ncoverages = rb_hash_new();
+    if (!RTEST(coverages)) {
+        rb_raise(rb_eRuntimeError, "coverage measurement is not enabled");
+    }
+    st_foreach(RHASH_TBL(coverages), coverage_result_i, ncoverages);
+    rb_hash_freeze(ncoverages);
+    rb_reset_coverages();
+    return ncoverages;
+}
+
+static VALUE
+rb_coverage_stop(void) {
+    if (!RTEST(rb_get_coverages())) {
+        rb_raise(rb_eRuntimeError, "coverage measurement is not enabled (covet)");
+    }
+    rb_reset_coverages();
+    return Qnil;
+}
+
+static VALUE
+rb_coverage_enabled_p(void) {
+    if (RTEST(rb_get_coverages())) {
+        return Qtrue;
+    } else {
+        return Qfalse;
+    }
+}
+
+void
+Init_covet_coverage(void)
+{
+    VALUE rb_mCoverage = rb_define_module("CovetCoverage");
+    rb_define_module_function(rb_mCoverage, "start", rb_coverage_start, 0);
+    rb_define_module_function(rb_mCoverage, "stop",  rb_coverage_stop, 0);
+    rb_define_module_function(rb_mCoverage, "result", rb_coverage_result, 0);
+    rb_define_module_function(rb_mCoverage, "peek_result", rb_coverage_peek_result, 0);
+    rb_define_module_function(rb_mCoverage, "enabled?", rb_coverage_enabled_p, 0);
+    rb_gc_register_address(&rb_coverages);
+}

data/ext/covet_coverage/extconf.rb

@@ -0,0 +1,7 @@
+require 'mkmf'
+
+have_func('rb_obj_hide')
+
+extension_name = 'covet_coverage'
+create_header
+create_makefile(extension_name)

data/lib/covet.rb

@@ -0,0 +1,206 @@
+require_relative 'covet/version'
+if defined?(Coverage) && Coverage.respond_to?(:peek_result)
+  CovetCoverage = Coverage
+else
+  # TODO: error out if non-mri ruby
+  begin
+    require_relative 'covet_coverage.so'
+  rescue Exception # re-raised
+    $stderr.puts "Error loading 'covet' C extension.\n" \
+      "Please report this bug along with a backtrace. Thanks :)"
+    raise
+  end
+end
+require_relative 'covet/collection_filter'
+require_relative 'covet/collection_compressor'
+require_relative 'covet/line_changes_vcs'
+require_relative 'covet/log_collection'
+require_relative 'covet/cli'
+
+# just for testing purposes
+if ENV['COVET_DEBUG']
+  if RUBY_VERSION < '2.0'
+    gem 'debugger'
+    require 'debugger'
+  else
+    gem 'byebug'
+    require 'byebug'
+  end
+else
+  if !defined?(debugger)
+    def debugger; end
+  end
+end
+
+module Covet
+  BASE_COVERAGE = {}
+
+  # @return Hash (frozen)
+  def self.options
+    CLI.options || Options::DEFAULTS
+  end
+
+  def self.log_collection
+    @log_collection
+  end
+
+  # TODO: filename should depend on covet options and there should
+  # be multiple run logs in a .covet_run_logs directory or something
+  @log_collection = LogCollection.new(
+    :filename => File.join(Dir.pwd, 'run_log.json'),
+    :bufsize => 50,
+  )
+
+  def self.vcs=(vcs)
+    @vcs = vcs.intern
+    if @vcs != :git
+      raise NotImplementedError, "Can only use git as the VCS for now."
+    end
+  end
+  def self.vcs; @vcs; end
+
+  self.vcs = :git # default
+
+  def self.test_runner=(runner)
+    @test_runner = runner.intern
+    require_relative "covet/test_runners/#{runner}"
+  rescue LoadError
+    raise ArgumentError, "invalid test runner given: '#{runner}'. " \
+      "Expected 'rspec' or 'minitest'"
+  end
+  def self.test_runner; @test_runner; end
+
+  if (runner = ENV['COVET_TEST_RUNNER'])
+    self.test_runner = runner
+  else
+    self.test_runner = :minitest # default
+  end
+
+  VALID_TEST_ORDERS = [:random_seeded, :random, :ordered].freeze
+  def self.test_order=(order)
+    unless VALID_TEST_ORDERS.include?(order.intern)
+      raise ArgumentError, "Invalid test order given. Expected one of " \
+        "#{VALID_TEST_ORDERS.map(&:inspect).join(", ")} - #{order.intern.inspect} given"
+    end
+    @test_order = order
+  end
+  def self.test_order; @test_order; end
+
+  self.test_order = :random_seeded # default
+
+  def self.test_directories=(*dirs)
+    dirs = dirs.flatten
+    dirs.each do |dir|
+      unless Dir.exist?(dir)
+        raise Errno::ENOENT, %Q(invalid directory given: "#{dir}" ) +
+          %Q{("#{File.join(Dir.pwd, dir)}")}
+      end
+    end
+    @test_directories = dirs
+  end
+  def self.test_directories; @test_directories.dup; end
+
+  # FIXME: make this configurable, the test directory could be something else
+  self.test_directories = []
+  if Dir.exist?('test')
+    self.test_directories = self.test_directories + ['test']
+  end
+  if Dir.exist?('spec')
+    self.test_directories = self.test_directories + ['spec']
+  end
+
+  def self.register_coverage_collection!
+    # stdlib Coverage can't run at the same time as CovetCoverage or
+    # bad things will happen
+    if defined?(Coverage) && !Coverage.respond_to?(:peek_result)
+      # There's no way to tell if coverage is enabled or not, and
+      # if we try stopping the coverage and it's not enabled, it raises
+      # a RuntimeError.
+      Coverage.stop rescue nil
+    end
+    CovetCoverage.start # needs to be called before any application code gets required
+    Covet::TestRunners.const_get(
+      @test_runner.to_s.capitalize
+    ).hook_into_test_methods!
+  end
+
+  # @return String
+  def self.cmdline_for_run_list(run_list)
+    Covet::TestRunners.const_get(
+      @test_runner.to_s.capitalize
+    ).cmdline_for_run_list(run_list)
+  end
+
+  # Returns coverage information for before `block` ran, and after `block` ran
+  # for the codebase in its current state.
+  # @return Array
+  def self.coverage_before_and_after # yields
+    before = CovetCoverage.peek_result
+    yield if block_given?
+    after = CovetCoverage.peek_result
+    before = normalize_coverage_info(before)
+    if Covet::BASE_COVERAGE.any?
+      before = diff_coverages(Covet::BASE_COVERAGE, before)
+    end
+    after = normalize_coverage_info(after)
+    after = diff_coverages(before, after)
+    [before, after]
+  end
+
+  def self.normalize_coverage_info(coverage_info)
+    filtered = CollectionFilter.filter(coverage_info)
+    CollectionCompressor.compress(filtered)
+  end
+
+  # Generates a mapping of filenames to the lines and test methods that
+  # caused the changes.
+  # @return Hash, example:
+  #   { "/home/me/workspace/myproj/myproj.rb" => { 1 => ['test_method_that_caused_changed']} }
+  def self.generate_run_list_for_method(before, after, options = {})
+    cov_map = Hash.new { |h, file| h[file] = Hash.new { |i, line| i[line] = [] } }
+    after.each do |file, lines_hash|
+      file_map = cov_map[file]
+
+      lines_hash.each do |lineno, exec_times|
+        # add the test name to the map. Multiple tests can execute the same
+        # line, so we need to use an array.
+        file_map[lineno] << (options[:method_name] || '???').to_s
+      end
+    end
+    cov_map
+  end
+
+  # @return Hash
+  def self.diff_coverages(before, after)
+    ret = after.each_with_object({}) do |(file_name, after_line_cov), res|
+      before_line_cov = before[file_name] || {}
+      next if before_line_cov == after_line_cov
+
+      cov = {}
+
+      after_line_cov.each do |lineno, exec_times|
+        # no change
+        if (before_exec_times = before_line_cov[lineno]) == exec_times
+          next
+        end
+
+        # execution of previous line number
+        if before_exec_times && exec_times
+          cov[lineno] = exec_times - before_exec_times
+        elsif exec_times
+          cov[lineno] = exec_times
+        else
+          raise "shouldn't get here"
+        end
+      end
+
+      # add the "diffed" coverage to the hash
+      res[file_name] = cov
+    end
+    ret
+  end
+end
+
+if ENV['COVET_COLLECT'] == '1'
+  Covet.register_coverage_collection!
+end