covet 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/Gemfile +12 -0
- data/README.md +82 -0
- data/Rakefile +37 -0
- data/bin/covet +3 -0
- data/ext/covet_coverage/covet_coverage.c +153 -0
- data/ext/covet_coverage/extconf.rb +7 -0
- data/lib/covet.rb +206 -0
- data/lib/covet/cli.rb +254 -0
- data/lib/covet/collection_compressor.rb +20 -0
- data/lib/covet/collection_filter.rb +113 -0
- data/lib/covet/line_changes_vcs.rb +13 -0
- data/lib/covet/log_collection.rb +51 -0
- data/lib/covet/log_file.rb +164 -0
- data/lib/covet/test_runners/minitest.rb +159 -0
- data/lib/covet/test_runners/rspec.rb +92 -0
- data/lib/covet/vcs/git.rb +62 -0
- data/lib/covet/version.rb +3 -0
- data/lib/covet_collect.rb +24 -0
- metadata +92 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: f7fd3d912a0b3a182a3b8004c0d4bd4b7a75c313
|
4
|
+
data.tar.gz: 090be05aac594fac1273ef2ea6c90a732367d021
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 90adde7bc36fbfe9550e16212bdd52da65161adb3de5c003227b9a5782990762ce9b6c552148748652d6a559f48d7f190b6084a14e107c24d8dcc8c10b75a726
|
7
|
+
data.tar.gz: a3ddfd5fd2997c04542bb35e3e7c2ce34ba29f6bf1c6853d0502b84eb2dcc7e482b3b67ef6ab71c19c75a965fedd2ff924a30d4ff8820fc6030c0d3c2cce1f25
|
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,82 @@
|
|
1
|
+
Covet
|
2
|
+
=====
|
3
|
+
|
4
|
+
What Is It?
|
5
|
+
-----------
|
6
|
+
|
7
|
+
It's a regression test selection tool for ruby, an implementation
|
8
|
+
of some ideas by Mr. Tenderlove expressed
|
9
|
+
[here](http://tenderlovemaking.com/2015/02/13/predicting-test-failues.html).
|
10
|
+
|
11
|
+
How Does It Work?
|
12
|
+
-----------------
|
13
|
+
|
14
|
+
Reading the article will give you a good idea, but here's a short summary:
|
15
|
+
|
16
|
+
1) Gather coverage information during test runs, before and after
|
17
|
+
each test method in order to know which test methods ran which
|
18
|
+
files and lines of the tested application code.
|
19
|
+
|
20
|
+
2) Change the application code.
|
21
|
+
|
22
|
+
3) Covet shows you which tests to run based on the coverage information
|
23
|
+
gathered in step 1, and the fact that `git` knows that you changed
|
24
|
+
certain lines of the application code.
|
25
|
+
|
26
|
+
Usage
|
27
|
+
-----
|
28
|
+
|
29
|
+
Coverage Collection:
|
30
|
+
|
31
|
+
Run your test suite with coverage collection on. To enable this,
|
32
|
+
add `require 'covet'` before any tests run (in a test helper file or similar),
|
33
|
+
and run your suite with: `covet -c $CMD`, where $CMD is the command to run your
|
34
|
+
test suite. Example:
|
35
|
+
|
36
|
+
$ covet -c "rake test"
|
37
|
+
|
38
|
+
Covet should output a message before any other message:
|
39
|
+
|
40
|
+
Collecting coverage information for each test method...
|
41
|
+
|
42
|
+
By default, `covet` hooks into `minitest` and collects coverage before
|
43
|
+
and after each method. If you're using `rspec`, make sure to pass the `-t`
|
44
|
+
option:
|
45
|
+
|
46
|
+
$ covet -t rspec -c "rake test"
|
47
|
+
|
48
|
+
After this, you should have 2 new files: `run_log.json`, and
|
49
|
+
`run_log_index.json`.
|
50
|
+
|
51
|
+
Now, by default the `covet` command will print out which test
|
52
|
+
files should be run based off the changes in your git repo since
|
53
|
+
the last commit.
|
54
|
+
|
55
|
+
For example:
|
56
|
+
|
57
|
+
$ covet
|
58
|
+
|
59
|
+
You need to run:
|
60
|
+
- /home/luke/Desktop/code/rails/activesupport/test/test\_case\_test.rb
|
61
|
+
|
62
|
+
To execute the run list, simply:
|
63
|
+
|
64
|
+
$ covet -e
|
65
|
+
|
66
|
+
Testing Gems
|
67
|
+
------------
|
68
|
+
|
69
|
+
By default, `covet` removes all standard library and gem files from the `run_log`, because
|
70
|
+
it assumes you're testing your own library code. In order to test a gem, you need to add the
|
71
|
+
`--whitelist-gems` option. For example:
|
72
|
+
|
73
|
+
$ covet -c "rake test" --whitelist-gems "activesupport,rails"
|
74
|
+
|
75
|
+
Caveats/Bugs
|
76
|
+
------------
|
77
|
+
|
78
|
+
1) It's not tested thoroughly enough.
|
79
|
+
|
80
|
+
2) Don't rely on this library to be correct yet (ie: don't forgo full test
|
81
|
+
suite runs before committing to a repository). It's still early days.
|
82
|
+
Please contribute code, docs, or ideas, though!
|
data/Rakefile
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'rake/testtask'
|
2
|
+
require 'rake/extensiontask'
|
3
|
+
require 'rspec/core/rake_task'
|
4
|
+
require 'rake/clean'
|
5
|
+
|
6
|
+
task :default => [:tests_and_specs]
|
7
|
+
|
8
|
+
Rake::TestTask.new(:test) do |t|
|
9
|
+
t.test_files = FileList['test/**/*_test.rb'].to_a
|
10
|
+
t.verbose = true
|
11
|
+
if t.respond_to?(:description=)
|
12
|
+
t.description = 'Run all minitest tests'
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
desc 'Run all rspec specs'
|
17
|
+
RSpec::Core::RakeTask.new(:spec) do |t|
|
18
|
+
t.pattern = Dir.glob('spec/**/*_spec.rb')
|
19
|
+
t.verbose = true
|
20
|
+
end
|
21
|
+
|
22
|
+
desc 'Run minitest and rspec tests (default)'
|
23
|
+
task :tests_and_specs => [:test, :spec]
|
24
|
+
|
25
|
+
desc 'compile internal coverage C extension'
|
26
|
+
Rake::ExtensionTask.new('covet_coverage')
|
27
|
+
|
28
|
+
desc 'recompile internal coverage C extension'
|
29
|
+
task :recompile => [:clobber, :compile, :tests_and_specs]
|
30
|
+
|
31
|
+
# for rake:clobber
|
32
|
+
CLOBBER.include(
|
33
|
+
'run_log.json',
|
34
|
+
'run_log_index.json',
|
35
|
+
'ext/covet_coverage/*.{so,o}',
|
36
|
+
'ext/covet_coverage/Makefile',
|
37
|
+
)
|
data/bin/covet
ADDED
@@ -0,0 +1,153 @@
|
|
1
|
+
/************************************************
|
2
|
+
|
3
|
+
coverage.c -
|
4
|
+
|
5
|
+
$Author: $
|
6
|
+
|
7
|
+
Copyright (c) 2008 Yusuke Endoh
|
8
|
+
|
9
|
+
************************************************/
|
10
|
+
|
11
|
+
#include "ruby.h"
|
12
|
+
#include "extconf.h"
|
13
|
+
|
14
|
+
extern VALUE rb_get_coverages(void);
|
15
|
+
extern void rb_set_coverages(VALUE);
|
16
|
+
extern void rb_reset_coverages(void);
|
17
|
+
|
18
|
+
#ifdef HAVE_RB_OBJ_HIDE
|
19
|
+
extern VALUE rb_obj_hide(VALUE);
|
20
|
+
static void hide_obj(VALUE val)
|
21
|
+
{
|
22
|
+
rb_obj_hide(val);
|
23
|
+
}
|
24
|
+
#else
|
25
|
+
static void hide_obj(VALUE val)
|
26
|
+
{
|
27
|
+
RBASIC(val)->klass = 0;
|
28
|
+
}
|
29
|
+
#endif
|
30
|
+
|
31
|
+
static VALUE rb_coverages = Qundef;
|
32
|
+
|
33
|
+
|
34
|
+
/*
|
35
|
+
* call-seq:
|
36
|
+
* CovetCoverage.start => nil
|
37
|
+
* * Enables coverage measurement.
|
38
|
+
*/
|
39
|
+
static VALUE
|
40
|
+
rb_coverage_start(void)
|
41
|
+
{
|
42
|
+
if (!RTEST(rb_get_coverages())) {
|
43
|
+
if (rb_coverages == Qundef) {
|
44
|
+
rb_coverages = rb_hash_new();
|
45
|
+
hide_obj(rb_coverages);
|
46
|
+
}
|
47
|
+
rb_set_coverages(rb_coverages);
|
48
|
+
}
|
49
|
+
return Qnil;
|
50
|
+
}
|
51
|
+
|
52
|
+
static int
|
53
|
+
coverage_clear_result_i(st_data_t key, st_data_t val, st_data_t h)
|
54
|
+
{
|
55
|
+
VALUE coverage = (VALUE)val;
|
56
|
+
rb_ary_clear(coverage);
|
57
|
+
return ST_CONTINUE;
|
58
|
+
}
|
59
|
+
|
60
|
+
static int
|
61
|
+
coverage_peek_result_i(st_data_t key, st_data_t val, st_data_t h)
|
62
|
+
{
|
63
|
+
VALUE path = (VALUE)key;
|
64
|
+
VALUE coverage = (VALUE)val;
|
65
|
+
VALUE coverages = (VALUE)h;
|
66
|
+
coverage = rb_ary_dup(coverage);
|
67
|
+
rb_ary_freeze(coverage);
|
68
|
+
rb_hash_aset(coverages, path, coverage);
|
69
|
+
return ST_CONTINUE;
|
70
|
+
}
|
71
|
+
|
72
|
+
/*
|
73
|
+
* call-seq:
|
74
|
+
* CovetCoverage.peek_result => hash
|
75
|
+
*
|
76
|
+
* Returns a hash that contains filename as key and coverage array as value.
|
77
|
+
*/
|
78
|
+
static VALUE
|
79
|
+
rb_coverage_peek_result(void)
|
80
|
+
{
|
81
|
+
VALUE coverages = rb_get_coverages();
|
82
|
+
VALUE ncoverages = rb_hash_new();
|
83
|
+
if (!RTEST(coverages)) {
|
84
|
+
rb_raise(rb_eRuntimeError, "coverage measurement is not enabled (covet)");
|
85
|
+
}
|
86
|
+
st_foreach(RHASH_TBL(coverages), coverage_peek_result_i, ncoverages);
|
87
|
+
rb_hash_freeze(ncoverages);
|
88
|
+
return ncoverages;
|
89
|
+
}
|
90
|
+
|
91
|
+
static int
|
92
|
+
coverage_result_i(st_data_t key, st_data_t val, st_data_t h)
|
93
|
+
{
|
94
|
+
VALUE path = (VALUE)key;
|
95
|
+
VALUE coverage = (VALUE)val;
|
96
|
+
VALUE coverages = (VALUE)h;
|
97
|
+
coverage = rb_ary_dup(coverage);
|
98
|
+
rb_ary_clear((VALUE)val);
|
99
|
+
rb_ary_freeze(coverage);
|
100
|
+
rb_hash_aset(coverages, path, coverage);
|
101
|
+
return ST_CONTINUE;
|
102
|
+
}
|
103
|
+
|
104
|
+
/*
|
105
|
+
* call-seq:
|
106
|
+
* CovetCoverage.result => hash
|
107
|
+
*
|
108
|
+
* Returns a hash that contains filename as key and coverage array as value
|
109
|
+
* and disables coverage measurement.
|
110
|
+
*/
|
111
|
+
static VALUE
|
112
|
+
rb_coverage_result(void)
|
113
|
+
{
|
114
|
+
VALUE coverages = rb_get_coverages();
|
115
|
+
VALUE ncoverages = rb_hash_new();
|
116
|
+
if (!RTEST(coverages)) {
|
117
|
+
rb_raise(rb_eRuntimeError, "coverage measurement is not enabled");
|
118
|
+
}
|
119
|
+
st_foreach(RHASH_TBL(coverages), coverage_result_i, ncoverages);
|
120
|
+
rb_hash_freeze(ncoverages);
|
121
|
+
rb_reset_coverages();
|
122
|
+
return ncoverages;
|
123
|
+
}
|
124
|
+
|
125
|
+
static VALUE
|
126
|
+
rb_coverage_stop(void) {
|
127
|
+
if (!RTEST(rb_get_coverages())) {
|
128
|
+
rb_raise(rb_eRuntimeError, "coverage measurement is not enabled (covet)");
|
129
|
+
}
|
130
|
+
rb_reset_coverages();
|
131
|
+
return Qnil;
|
132
|
+
}
|
133
|
+
|
134
|
+
static VALUE
|
135
|
+
rb_coverage_enabled_p(void) {
|
136
|
+
if (RTEST(rb_get_coverages())) {
|
137
|
+
return Qtrue;
|
138
|
+
} else {
|
139
|
+
return Qfalse;
|
140
|
+
}
|
141
|
+
}
|
142
|
+
|
143
|
+
void
|
144
|
+
Init_covet_coverage(void)
|
145
|
+
{
|
146
|
+
VALUE rb_mCoverage = rb_define_module("CovetCoverage");
|
147
|
+
rb_define_module_function(rb_mCoverage, "start", rb_coverage_start, 0);
|
148
|
+
rb_define_module_function(rb_mCoverage, "stop", rb_coverage_stop, 0);
|
149
|
+
rb_define_module_function(rb_mCoverage, "result", rb_coverage_result, 0);
|
150
|
+
rb_define_module_function(rb_mCoverage, "peek_result", rb_coverage_peek_result, 0);
|
151
|
+
rb_define_module_function(rb_mCoverage, "enabled?", rb_coverage_enabled_p, 0);
|
152
|
+
rb_gc_register_address(&rb_coverages);
|
153
|
+
}
|
data/lib/covet.rb
ADDED
@@ -0,0 +1,206 @@
|
|
1
|
+
require_relative 'covet/version'
|
2
|
+
if defined?(Coverage) && Coverage.respond_to?(:peek_result)
|
3
|
+
CovetCoverage = Coverage
|
4
|
+
else
|
5
|
+
# TODO: error out if non-mri ruby
|
6
|
+
begin
|
7
|
+
require_relative 'covet_coverage.so'
|
8
|
+
rescue Exception # re-raised
|
9
|
+
$stderr.puts "Error loading 'covet' C extension.\n" \
|
10
|
+
"Please report this bug along with a backtrace. Thanks :)"
|
11
|
+
raise
|
12
|
+
end
|
13
|
+
end
|
14
|
+
require_relative 'covet/collection_filter'
|
15
|
+
require_relative 'covet/collection_compressor'
|
16
|
+
require_relative 'covet/line_changes_vcs'
|
17
|
+
require_relative 'covet/log_collection'
|
18
|
+
require_relative 'covet/cli'
|
19
|
+
|
20
|
+
# just for testing purposes
|
21
|
+
if ENV['COVET_DEBUG']
|
22
|
+
if RUBY_VERSION < '2.0'
|
23
|
+
gem 'debugger'
|
24
|
+
require 'debugger'
|
25
|
+
else
|
26
|
+
gem 'byebug'
|
27
|
+
require 'byebug'
|
28
|
+
end
|
29
|
+
else
|
30
|
+
if !defined?(debugger)
|
31
|
+
def debugger; end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
module Covet
|
36
|
+
BASE_COVERAGE = {}
|
37
|
+
|
38
|
+
# @return Hash (frozen)
|
39
|
+
def self.options
|
40
|
+
CLI.options || Options::DEFAULTS
|
41
|
+
end
|
42
|
+
|
43
|
+
def self.log_collection
|
44
|
+
@log_collection
|
45
|
+
end
|
46
|
+
|
47
|
+
# TODO: filename should depend on covet options and there should
|
48
|
+
# be multiple run logs in a .covet_run_logs directory or something
|
49
|
+
@log_collection = LogCollection.new(
|
50
|
+
:filename => File.join(Dir.pwd, 'run_log.json'),
|
51
|
+
:bufsize => 50,
|
52
|
+
)
|
53
|
+
|
54
|
+
def self.vcs=(vcs)
|
55
|
+
@vcs = vcs.intern
|
56
|
+
if @vcs != :git
|
57
|
+
raise NotImplementedError, "Can only use git as the VCS for now."
|
58
|
+
end
|
59
|
+
end
|
60
|
+
def self.vcs; @vcs; end
|
61
|
+
|
62
|
+
self.vcs = :git # default
|
63
|
+
|
64
|
+
def self.test_runner=(runner)
|
65
|
+
@test_runner = runner.intern
|
66
|
+
require_relative "covet/test_runners/#{runner}"
|
67
|
+
rescue LoadError
|
68
|
+
raise ArgumentError, "invalid test runner given: '#{runner}'. " \
|
69
|
+
"Expected 'rspec' or 'minitest'"
|
70
|
+
end
|
71
|
+
def self.test_runner; @test_runner; end
|
72
|
+
|
73
|
+
if (runner = ENV['COVET_TEST_RUNNER'])
|
74
|
+
self.test_runner = runner
|
75
|
+
else
|
76
|
+
self.test_runner = :minitest # default
|
77
|
+
end
|
78
|
+
|
79
|
+
VALID_TEST_ORDERS = [:random_seeded, :random, :ordered].freeze
|
80
|
+
def self.test_order=(order)
|
81
|
+
unless VALID_TEST_ORDERS.include?(order.intern)
|
82
|
+
raise ArgumentError, "Invalid test order given. Expected one of " \
|
83
|
+
"#{VALID_TEST_ORDERS.map(&:inspect).join(", ")} - #{order.intern.inspect} given"
|
84
|
+
end
|
85
|
+
@test_order = order
|
86
|
+
end
|
87
|
+
def self.test_order; @test_order; end
|
88
|
+
|
89
|
+
self.test_order = :random_seeded # default
|
90
|
+
|
91
|
+
def self.test_directories=(*dirs)
|
92
|
+
dirs = dirs.flatten
|
93
|
+
dirs.each do |dir|
|
94
|
+
unless Dir.exist?(dir)
|
95
|
+
raise Errno::ENOENT, %Q(invalid directory given: "#{dir}" ) +
|
96
|
+
%Q{("#{File.join(Dir.pwd, dir)}")}
|
97
|
+
end
|
98
|
+
end
|
99
|
+
@test_directories = dirs
|
100
|
+
end
|
101
|
+
def self.test_directories; @test_directories.dup; end
|
102
|
+
|
103
|
+
# FIXME: make this configurable, the test directory could be something else
|
104
|
+
self.test_directories = []
|
105
|
+
if Dir.exist?('test')
|
106
|
+
self.test_directories = self.test_directories + ['test']
|
107
|
+
end
|
108
|
+
if Dir.exist?('spec')
|
109
|
+
self.test_directories = self.test_directories + ['spec']
|
110
|
+
end
|
111
|
+
|
112
|
+
def self.register_coverage_collection!
|
113
|
+
# stdlib Coverage can't run at the same time as CovetCoverage or
|
114
|
+
# bad things will happen
|
115
|
+
if defined?(Coverage) && !Coverage.respond_to?(:peek_result)
|
116
|
+
# There's no way to tell if coverage is enabled or not, and
|
117
|
+
# if we try stopping the coverage and it's not enabled, it raises
|
118
|
+
# a RuntimeError.
|
119
|
+
Coverage.stop rescue nil
|
120
|
+
end
|
121
|
+
CovetCoverage.start # needs to be called before any application code gets required
|
122
|
+
Covet::TestRunners.const_get(
|
123
|
+
@test_runner.to_s.capitalize
|
124
|
+
).hook_into_test_methods!
|
125
|
+
end
|
126
|
+
|
127
|
+
# @return String
|
128
|
+
def self.cmdline_for_run_list(run_list)
|
129
|
+
Covet::TestRunners.const_get(
|
130
|
+
@test_runner.to_s.capitalize
|
131
|
+
).cmdline_for_run_list(run_list)
|
132
|
+
end
|
133
|
+
|
134
|
+
# Returns coverage information for before `block` ran, and after `block` ran
|
135
|
+
# for the codebase in its current state.
|
136
|
+
# @return Array
|
137
|
+
def self.coverage_before_and_after # yields
|
138
|
+
before = CovetCoverage.peek_result
|
139
|
+
yield if block_given?
|
140
|
+
after = CovetCoverage.peek_result
|
141
|
+
before = normalize_coverage_info(before)
|
142
|
+
if Covet::BASE_COVERAGE.any?
|
143
|
+
before = diff_coverages(Covet::BASE_COVERAGE, before)
|
144
|
+
end
|
145
|
+
after = normalize_coverage_info(after)
|
146
|
+
after = diff_coverages(before, after)
|
147
|
+
[before, after]
|
148
|
+
end
|
149
|
+
|
150
|
+
def self.normalize_coverage_info(coverage_info)
|
151
|
+
filtered = CollectionFilter.filter(coverage_info)
|
152
|
+
CollectionCompressor.compress(filtered)
|
153
|
+
end
|
154
|
+
|
155
|
+
# Generates a mapping of filenames to the lines and test methods that
|
156
|
+
# caused the changes.
|
157
|
+
# @return Hash, example:
|
158
|
+
# { "/home/me/workspace/myproj/myproj.rb" => { 1 => ['test_method_that_caused_changed']} }
|
159
|
+
def self.generate_run_list_for_method(before, after, options = {})
|
160
|
+
cov_map = Hash.new { |h, file| h[file] = Hash.new { |i, line| i[line] = [] } }
|
161
|
+
after.each do |file, lines_hash|
|
162
|
+
file_map = cov_map[file]
|
163
|
+
|
164
|
+
lines_hash.each do |lineno, exec_times|
|
165
|
+
# add the test name to the map. Multiple tests can execute the same
|
166
|
+
# line, so we need to use an array.
|
167
|
+
file_map[lineno] << (options[:method_name] || '???').to_s
|
168
|
+
end
|
169
|
+
end
|
170
|
+
cov_map
|
171
|
+
end
|
172
|
+
|
173
|
+
# @return Hash
|
174
|
+
def self.diff_coverages(before, after)
|
175
|
+
ret = after.each_with_object({}) do |(file_name, after_line_cov), res|
|
176
|
+
before_line_cov = before[file_name] || {}
|
177
|
+
next if before_line_cov == after_line_cov
|
178
|
+
|
179
|
+
cov = {}
|
180
|
+
|
181
|
+
after_line_cov.each do |lineno, exec_times|
|
182
|
+
# no change
|
183
|
+
if (before_exec_times = before_line_cov[lineno]) == exec_times
|
184
|
+
next
|
185
|
+
end
|
186
|
+
|
187
|
+
# execution of previous line number
|
188
|
+
if before_exec_times && exec_times
|
189
|
+
cov[lineno] = exec_times - before_exec_times
|
190
|
+
elsif exec_times
|
191
|
+
cov[lineno] = exec_times
|
192
|
+
else
|
193
|
+
raise "shouldn't get here"
|
194
|
+
end
|
195
|
+
end
|
196
|
+
|
197
|
+
# add the "diffed" coverage to the hash
|
198
|
+
res[file_name] = cov
|
199
|
+
end
|
200
|
+
ret
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
if ENV['COVET_COLLECT'] == '1'
|
205
|
+
Covet.register_coverage_collection!
|
206
|
+
end
|