covet 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: f7fd3d912a0b3a182a3b8004c0d4bd4b7a75c313
4
+ data.tar.gz: 090be05aac594fac1273ef2ea6c90a732367d021
5
+ SHA512:
6
+ metadata.gz: 90adde7bc36fbfe9550e16212bdd52da65161adb3de5c003227b9a5782990762ce9b6c552148748652d6a559f48d7f190b6084a14e107c24d8dcc8c10b75a726
7
+ data.tar.gz: a3ddfd5fd2997c04542bb35e3e7c2ce34ba29f6bf1c6853d0502b84eb2dcc7e482b3b67ef6ab71c19c75a965fedd2ff924a30d4ff8820fc6030c0d3c2cce1f25
data/Gemfile ADDED
@@ -0,0 +1,12 @@
1
+ source 'https://rubygems.org'
2
+ gemspec
3
+
4
+ group :test, :development do
5
+ gem 'rspec'
6
+ gem 'minitest'
7
+ #if RUBY_VERSION.to_i < 2
8
+ #gem 'debugger'
9
+ #else
10
+ #gem 'byebug'
11
+ #end
12
+ end
data/README.md ADDED
@@ -0,0 +1,82 @@
1
+ Covet
2
+ =====
3
+
4
+ What Is It?
5
+ -----------
6
+
7
+ It's a regression test selection tool for ruby, an implementation
8
+ of some ideas by Mr. Tenderlove expressed
9
+ [here](http://tenderlovemaking.com/2015/02/13/predicting-test-failues.html).
10
+
11
+ How Does It Work?
12
+ -----------------
13
+
14
+ Reading the article will give you a good idea, but here's a short summary:
15
+
16
+ 1) Gather coverage information during test runs, before and after
17
+ each test method in order to know which test methods ran which
18
+ files and lines of the tested application code.
19
+
20
+ 2) Change the application code.
21
+
22
+ 3) Covet shows you which tests to run based on the coverage information
23
+ gathered in step 1, and the fact that `git` knows that you changed
24
+ certain lines of the application code.
25
+
26
+ Usage
27
+ -----
28
+
29
+ Coverage Collection:
30
+
31
+ Run your test suite with coverage collection on. To enable this,
32
+ add `require 'covet'` before any tests run (in a test helper file or similar),
33
+ and run your suite with: `covet -c $CMD`, where $CMD is the command to run your
34
+ test suite. Example:
35
+
36
+ $ covet -c "rake test"
37
+
38
+ Covet should output a message before any other message:
39
+
40
+ Collecting coverage information for each test method...
41
+
42
+ By default, `covet` hooks into `minitest` and collects coverage before
43
+ and after each method. If you're using `rspec`, make sure to pass the `-t`
44
+ option:
45
+
46
+ $ covet -t rspec -c "rake test"
47
+
48
+ After this, you should have 2 new files: `run_log.json`, and
49
+ `run_log_index.json`.
50
+
51
+ Now, by default the `covet` command will print out which test
52
+ files should be run based off the changes in your git repo since
53
+ the last commit.
54
+
55
+ For example:
56
+
57
+ $ covet
58
+
59
+ You need to run:
60
+ - /home/luke/Desktop/code/rails/activesupport/test/test\_case\_test.rb
61
+
62
+ To execute the run list, simply:
63
+
64
+ $ covet -e
65
+
66
+ Testing Gems
67
+ ------------
68
+
69
+ By default, `covet` removes all standard library and gem files from the `run_log`, because
70
+ it assumes you're testing your own library code. In order to test a gem, you need to add the
71
+ `--whitelist-gems` option. For example:
72
+
73
+ $ covet -c "rake test" --whitelist-gems "activesupport,rails"
74
+
75
+ Caveats/Bugs
76
+ ------------
77
+
78
+ 1) It's not tested thoroughly enough.
79
+
80
+ 2) Don't rely on this library to be correct yet (ie: don't forgo full test
81
+ suite runs before committing to a repository). It's still early days.
82
+ Please contribute code, docs, or ideas, though!
data/Rakefile ADDED
@@ -0,0 +1,37 @@
1
+ require 'rake/testtask'
2
+ require 'rake/extensiontask'
3
+ require 'rspec/core/rake_task'
4
+ require 'rake/clean'
5
+
6
+ task :default => [:tests_and_specs]
7
+
8
+ Rake::TestTask.new(:test) do |t|
9
+ t.test_files = FileList['test/**/*_test.rb'].to_a
10
+ t.verbose = true
11
+ if t.respond_to?(:description=)
12
+ t.description = 'Run all minitest tests'
13
+ end
14
+ end
15
+
16
+ desc 'Run all rspec specs'
17
+ RSpec::Core::RakeTask.new(:spec) do |t|
18
+ t.pattern = Dir.glob('spec/**/*_spec.rb')
19
+ t.verbose = true
20
+ end
21
+
22
+ desc 'Run minitest and rspec tests (default)'
23
+ task :tests_and_specs => [:test, :spec]
24
+
25
+ desc 'compile internal coverage C extension'
26
+ Rake::ExtensionTask.new('covet_coverage')
27
+
28
+ desc 'recompile internal coverage C extension'
29
+ task :recompile => [:clobber, :compile, :tests_and_specs]
30
+
31
+ # for rake:clobber
32
+ CLOBBER.include(
33
+ 'run_log.json',
34
+ 'run_log_index.json',
35
+ 'ext/covet_coverage/*.{so,o}',
36
+ 'ext/covet_coverage/Makefile',
37
+ )
data/bin/covet ADDED
@@ -0,0 +1,3 @@
1
+ #!/usr/bin/env ruby
2
+ require_relative '../lib/covet'
3
+ Covet::CLI.new(ARGV.dup).run
@@ -0,0 +1,153 @@
1
+ /************************************************
2
+
3
+ coverage.c -
4
+
5
+ $Author: $
6
+
7
+ Copyright (c) 2008 Yusuke Endoh
8
+
9
+ ************************************************/
10
+
11
+ #include "ruby.h"
12
+ #include "extconf.h"
13
+
14
+ extern VALUE rb_get_coverages(void);
15
+ extern void rb_set_coverages(VALUE);
16
+ extern void rb_reset_coverages(void);
17
+
18
+ #ifdef HAVE_RB_OBJ_HIDE
19
+ extern VALUE rb_obj_hide(VALUE);
20
+ static void hide_obj(VALUE val)
21
+ {
22
+ rb_obj_hide(val);
23
+ }
24
+ #else
25
+ static void hide_obj(VALUE val)
26
+ {
27
+ RBASIC(val)->klass = 0;
28
+ }
29
+ #endif
30
+
31
+ static VALUE rb_coverages = Qundef;
32
+
33
+
34
+ /*
35
+ * call-seq:
36
+ * CovetCoverage.start => nil
37
+ * * Enables coverage measurement.
38
+ */
39
+ static VALUE
40
+ rb_coverage_start(void)
41
+ {
42
+ if (!RTEST(rb_get_coverages())) {
43
+ if (rb_coverages == Qundef) {
44
+ rb_coverages = rb_hash_new();
45
+ hide_obj(rb_coverages);
46
+ }
47
+ rb_set_coverages(rb_coverages);
48
+ }
49
+ return Qnil;
50
+ }
51
+
52
+ static int
53
+ coverage_clear_result_i(st_data_t key, st_data_t val, st_data_t h)
54
+ {
55
+ VALUE coverage = (VALUE)val;
56
+ rb_ary_clear(coverage);
57
+ return ST_CONTINUE;
58
+ }
59
+
60
+ static int
61
+ coverage_peek_result_i(st_data_t key, st_data_t val, st_data_t h)
62
+ {
63
+ VALUE path = (VALUE)key;
64
+ VALUE coverage = (VALUE)val;
65
+ VALUE coverages = (VALUE)h;
66
+ coverage = rb_ary_dup(coverage);
67
+ rb_ary_freeze(coverage);
68
+ rb_hash_aset(coverages, path, coverage);
69
+ return ST_CONTINUE;
70
+ }
71
+
72
+ /*
73
+ * call-seq:
74
+ * CovetCoverage.peek_result => hash
75
+ *
76
+ * Returns a hash that contains filename as key and coverage array as value.
77
+ */
78
+ static VALUE
79
+ rb_coverage_peek_result(void)
80
+ {
81
+ VALUE coverages = rb_get_coverages();
82
+ VALUE ncoverages = rb_hash_new();
83
+ if (!RTEST(coverages)) {
84
+ rb_raise(rb_eRuntimeError, "coverage measurement is not enabled (covet)");
85
+ }
86
+ st_foreach(RHASH_TBL(coverages), coverage_peek_result_i, ncoverages);
87
+ rb_hash_freeze(ncoverages);
88
+ return ncoverages;
89
+ }
90
+
91
+ static int
92
+ coverage_result_i(st_data_t key, st_data_t val, st_data_t h)
93
+ {
94
+ VALUE path = (VALUE)key;
95
+ VALUE coverage = (VALUE)val;
96
+ VALUE coverages = (VALUE)h;
97
+ coverage = rb_ary_dup(coverage);
98
+ rb_ary_clear((VALUE)val);
99
+ rb_ary_freeze(coverage);
100
+ rb_hash_aset(coverages, path, coverage);
101
+ return ST_CONTINUE;
102
+ }
103
+
104
+ /*
105
+ * call-seq:
106
+ * CovetCoverage.result => hash
107
+ *
108
+ * Returns a hash that contains filename as key and coverage array as value
109
+ * and disables coverage measurement.
110
+ */
111
+ static VALUE
112
+ rb_coverage_result(void)
113
+ {
114
+ VALUE coverages = rb_get_coverages();
115
+ VALUE ncoverages = rb_hash_new();
116
+ if (!RTEST(coverages)) {
117
+ rb_raise(rb_eRuntimeError, "coverage measurement is not enabled");
118
+ }
119
+ st_foreach(RHASH_TBL(coverages), coverage_result_i, ncoverages);
120
+ rb_hash_freeze(ncoverages);
121
+ rb_reset_coverages();
122
+ return ncoverages;
123
+ }
124
+
125
+ static VALUE
126
+ rb_coverage_stop(void) {
127
+ if (!RTEST(rb_get_coverages())) {
128
+ rb_raise(rb_eRuntimeError, "coverage measurement is not enabled (covet)");
129
+ }
130
+ rb_reset_coverages();
131
+ return Qnil;
132
+ }
133
+
134
+ static VALUE
135
+ rb_coverage_enabled_p(void) {
136
+ if (RTEST(rb_get_coverages())) {
137
+ return Qtrue;
138
+ } else {
139
+ return Qfalse;
140
+ }
141
+ }
142
+
143
+ void
144
+ Init_covet_coverage(void)
145
+ {
146
+ VALUE rb_mCoverage = rb_define_module("CovetCoverage");
147
+ rb_define_module_function(rb_mCoverage, "start", rb_coverage_start, 0);
148
+ rb_define_module_function(rb_mCoverage, "stop", rb_coverage_stop, 0);
149
+ rb_define_module_function(rb_mCoverage, "result", rb_coverage_result, 0);
150
+ rb_define_module_function(rb_mCoverage, "peek_result", rb_coverage_peek_result, 0);
151
+ rb_define_module_function(rb_mCoverage, "enabled?", rb_coverage_enabled_p, 0);
152
+ rb_gc_register_address(&rb_coverages);
153
+ }
@@ -0,0 +1,7 @@
1
+ require 'mkmf'
2
+
3
+ have_func('rb_obj_hide')
4
+
5
+ extension_name = 'covet_coverage'
6
+ create_header
7
+ create_makefile(extension_name)
data/lib/covet.rb ADDED
@@ -0,0 +1,206 @@
1
+ require_relative 'covet/version'
2
+ if defined?(Coverage) && Coverage.respond_to?(:peek_result)
3
+ CovetCoverage = Coverage
4
+ else
5
+ # TODO: error out if non-mri ruby
6
+ begin
7
+ require_relative 'covet_coverage.so'
8
+ rescue Exception # re-raised
9
+ $stderr.puts "Error loading 'covet' C extension.\n" \
10
+ "Please report this bug along with a backtrace. Thanks :)"
11
+ raise
12
+ end
13
+ end
14
+ require_relative 'covet/collection_filter'
15
+ require_relative 'covet/collection_compressor'
16
+ require_relative 'covet/line_changes_vcs'
17
+ require_relative 'covet/log_collection'
18
+ require_relative 'covet/cli'
19
+
20
+ # just for testing purposes
21
+ if ENV['COVET_DEBUG']
22
+ if RUBY_VERSION < '2.0'
23
+ gem 'debugger'
24
+ require 'debugger'
25
+ else
26
+ gem 'byebug'
27
+ require 'byebug'
28
+ end
29
+ else
30
+ if !defined?(debugger)
31
+ def debugger; end
32
+ end
33
+ end
34
+
35
+ module Covet
36
+ BASE_COVERAGE = {}
37
+
38
+ # @return Hash (frozen)
39
+ def self.options
40
+ CLI.options || Options::DEFAULTS
41
+ end
42
+
43
+ def self.log_collection
44
+ @log_collection
45
+ end
46
+
47
+ # TODO: filename should depend on covet options and there should
48
+ # be multiple run logs in a .covet_run_logs directory or something
49
+ @log_collection = LogCollection.new(
50
+ :filename => File.join(Dir.pwd, 'run_log.json'),
51
+ :bufsize => 50,
52
+ )
53
+
54
+ def self.vcs=(vcs)
55
+ @vcs = vcs.intern
56
+ if @vcs != :git
57
+ raise NotImplementedError, "Can only use git as the VCS for now."
58
+ end
59
+ end
60
+ def self.vcs; @vcs; end
61
+
62
+ self.vcs = :git # default
63
+
64
+ def self.test_runner=(runner)
65
+ @test_runner = runner.intern
66
+ require_relative "covet/test_runners/#{runner}"
67
+ rescue LoadError
68
+ raise ArgumentError, "invalid test runner given: '#{runner}'. " \
69
+ "Expected 'rspec' or 'minitest'"
70
+ end
71
+ def self.test_runner; @test_runner; end
72
+
73
+ if (runner = ENV['COVET_TEST_RUNNER'])
74
+ self.test_runner = runner
75
+ else
76
+ self.test_runner = :minitest # default
77
+ end
78
+
79
+ VALID_TEST_ORDERS = [:random_seeded, :random, :ordered].freeze
80
+ def self.test_order=(order)
81
+ unless VALID_TEST_ORDERS.include?(order.intern)
82
+ raise ArgumentError, "Invalid test order given. Expected one of " \
83
+ "#{VALID_TEST_ORDERS.map(&:inspect).join(", ")} - #{order.intern.inspect} given"
84
+ end
85
+ @test_order = order
86
+ end
87
+ def self.test_order; @test_order; end
88
+
89
+ self.test_order = :random_seeded # default
90
+
91
+ def self.test_directories=(*dirs)
92
+ dirs = dirs.flatten
93
+ dirs.each do |dir|
94
+ unless Dir.exist?(dir)
95
+ raise Errno::ENOENT, %Q(invalid directory given: "#{dir}" ) +
96
+ %Q{("#{File.join(Dir.pwd, dir)}")}
97
+ end
98
+ end
99
+ @test_directories = dirs
100
+ end
101
+ def self.test_directories; @test_directories.dup; end
102
+
103
+ # FIXME: make this configurable, the test directory could be something else
104
+ self.test_directories = []
105
+ if Dir.exist?('test')
106
+ self.test_directories = self.test_directories + ['test']
107
+ end
108
+ if Dir.exist?('spec')
109
+ self.test_directories = self.test_directories + ['spec']
110
+ end
111
+
112
+ def self.register_coverage_collection!
113
+ # stdlib Coverage can't run at the same time as CovetCoverage or
114
+ # bad things will happen
115
+ if defined?(Coverage) && !Coverage.respond_to?(:peek_result)
116
+ # There's no way to tell if coverage is enabled or not, and
117
+ # if we try stopping the coverage and it's not enabled, it raises
118
+ # a RuntimeError.
119
+ Coverage.stop rescue nil
120
+ end
121
+ CovetCoverage.start # needs to be called before any application code gets required
122
+ Covet::TestRunners.const_get(
123
+ @test_runner.to_s.capitalize
124
+ ).hook_into_test_methods!
125
+ end
126
+
127
+ # @return String
128
+ def self.cmdline_for_run_list(run_list)
129
+ Covet::TestRunners.const_get(
130
+ @test_runner.to_s.capitalize
131
+ ).cmdline_for_run_list(run_list)
132
+ end
133
+
134
+ # Returns coverage information for before `block` ran, and after `block` ran
135
+ # for the codebase in its current state.
136
+ # @return Array
137
+ def self.coverage_before_and_after # yields
138
+ before = CovetCoverage.peek_result
139
+ yield if block_given?
140
+ after = CovetCoverage.peek_result
141
+ before = normalize_coverage_info(before)
142
+ if Covet::BASE_COVERAGE.any?
143
+ before = diff_coverages(Covet::BASE_COVERAGE, before)
144
+ end
145
+ after = normalize_coverage_info(after)
146
+ after = diff_coverages(before, after)
147
+ [before, after]
148
+ end
149
+
150
+ def self.normalize_coverage_info(coverage_info)
151
+ filtered = CollectionFilter.filter(coverage_info)
152
+ CollectionCompressor.compress(filtered)
153
+ end
154
+
155
+ # Generates a mapping of filenames to the lines and test methods that
156
+ # caused the changes.
157
+ # @return Hash, example:
158
+ # { "/home/me/workspace/myproj/myproj.rb" => { 1 => ['test_method_that_caused_changed']} }
159
+ def self.generate_run_list_for_method(before, after, options = {})
160
+ cov_map = Hash.new { |h, file| h[file] = Hash.new { |i, line| i[line] = [] } }
161
+ after.each do |file, lines_hash|
162
+ file_map = cov_map[file]
163
+
164
+ lines_hash.each do |lineno, exec_times|
165
+ # add the test name to the map. Multiple tests can execute the same
166
+ # line, so we need to use an array.
167
+ file_map[lineno] << (options[:method_name] || '???').to_s
168
+ end
169
+ end
170
+ cov_map
171
+ end
172
+
173
+ # @return Hash
174
+ def self.diff_coverages(before, after)
175
+ ret = after.each_with_object({}) do |(file_name, after_line_cov), res|
176
+ before_line_cov = before[file_name] || {}
177
+ next if before_line_cov == after_line_cov
178
+
179
+ cov = {}
180
+
181
+ after_line_cov.each do |lineno, exec_times|
182
+ # no change
183
+ if (before_exec_times = before_line_cov[lineno]) == exec_times
184
+ next
185
+ end
186
+
187
+ # execution of previous line number
188
+ if before_exec_times && exec_times
189
+ cov[lineno] = exec_times - before_exec_times
190
+ elsif exec_times
191
+ cov[lineno] = exec_times
192
+ else
193
+ raise "shouldn't get here"
194
+ end
195
+ end
196
+
197
+ # add the "diffed" coverage to the hash
198
+ res[file_name] = cov
199
+ end
200
+ ret
201
+ end
202
+ end
203
+
204
+ if ENV['COVET_COLLECT'] == '1'
205
+ Covet.register_coverage_collection!
206
+ end