rspec-bisect 0.0.1 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +5 -1
- data/bin/rspec-bisect +97 -5
- data/features/accepts_seeds.feature +23 -0
- data/features/failing_tests/multiple_tests_order_dependencies.feature +27 -0
- data/features/failing_tests/prints_command_to_reproduce.feature +21 -0
- data/features/failing_tests/single_test_order_dependencies.feature +53 -0
- data/features/failing_tests/which_tests_have_order_dependencies.feature +46 -0
- data/features/support/env.rb +1 -1
- data/lib/rspec/bisect/version.rb +1 -1
- data/rspec-bisect.gemspec +5 -4
- metadata +36 -12
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fa219fdf62e2b1443b569705aa35d614fff5f054
|
4
|
+
data.tar.gz: de6a63d2fee72bbd3ac882b52c1d66878b5ca4d3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c9ea2dcdf6e665205f547c10f5cea43c50f376e2c031f673a0d27061599db2a7a197d6527fb89ed2dc7e72d17fd1c109216ec5b7ea87d2952ba31a5d2f9db575
|
7
|
+
data.tar.gz: 44261365ac61fae0389adcebd4a93ffc73b143cd5ac42c763b1ac0b4723dfd2a666f5afdf7ac47b0746ec0ea7195b873080b6b73b363090ffd49ee66c6bb4513
|
data/README.md
CHANGED
@@ -20,7 +20,11 @@ Or install it yourself as:
|
|
20
20
|
|
21
21
|
## Usage
|
22
22
|
|
23
|
-
|
23
|
+
So you think you have order dependencies? This tool will tell you exactly which tests are causing that dependency. It won't try to figure out if one exists or tell you why it's happening.
|
24
|
+
|
25
|
+
Have your rspec config set so that ```rspec --seed 1234``` runs your test suite with the specified seed.
|
26
|
+
|
27
|
+
Run ```rspec-bisect --seed 1234``` and rspec-bisect will tell you which tests are from an order dependency and the minimum tests required to reproduce it.
|
24
28
|
|
25
29
|
## Contributing
|
26
30
|
|
data/bin/rspec-bisect
CHANGED
@@ -1,15 +1,107 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
3
|
require 'json'
|
4
|
+
require 'colorize'
|
5
|
+
require 'optparse'
|
4
6
|
|
5
|
-
|
7
|
+
options = {}
|
8
|
+
OptionParser.new do |opts|
|
9
|
+
opts.banner = 'Usage: rspec-bisect [options]'
|
10
|
+
|
11
|
+
opts.on('-s', '--seed N', Integer, 'Seed that causes order dependencies') do |s|
|
12
|
+
options[:seed] = s
|
13
|
+
end
|
14
|
+
|
15
|
+
opts.on('--help', 'Show this message') do
|
16
|
+
puts opts
|
17
|
+
exit
|
18
|
+
end
|
19
|
+
end.parse!
|
20
|
+
|
21
|
+
if options[:seed].nil?
|
22
|
+
puts 'Running tests with no seed'
|
23
|
+
else
|
24
|
+
puts "Running tests with seed #{options[:seed]}"
|
25
|
+
end
|
26
|
+
|
27
|
+
result = `rspec --format json #{options[:seed].nil? ? '' : "--seed #{options[:seed]}"}`
|
6
28
|
parsed = JSON.parse result
|
7
29
|
|
8
|
-
|
30
|
+
examples = parsed['examples']
|
31
|
+
failure_count = parsed['summary']['failure_count']
|
9
32
|
if failure_count > 0
|
10
|
-
puts "#{failure_count} failing test#{failure_count > 1 ? 's' : ''}."
|
33
|
+
puts "#{failure_count} failing test#{failure_count > 1 ? 's' : ''}.".red
|
34
|
+
else
|
35
|
+
puts 'No failing tests.'.green
|
36
|
+
end
|
37
|
+
|
38
|
+
failing_examples = examples.select { |e| e['status'] == 'failed' }
|
39
|
+
|
40
|
+
def examples_as_rspec_params(examples)
|
41
|
+
examples.map { |e| "#{e['file_path']}:#{e['line_number']}" }.join ' '
|
42
|
+
end
|
43
|
+
|
44
|
+
def run_examples_command(examples)
|
45
|
+
"rspec #{examples_as_rspec_params(examples)}"
|
46
|
+
end
|
47
|
+
|
48
|
+
def run_examples(examples)
|
49
|
+
`#{run_examples_command(examples)}`
|
50
|
+
end
|
51
|
+
|
52
|
+
def last_command_passed?
|
53
|
+
$?.exitstatus == 0
|
54
|
+
end
|
55
|
+
|
56
|
+
def last_command_failed?
|
57
|
+
not last_command_passed?
|
58
|
+
end
|
59
|
+
|
60
|
+
order_dependent_examples = failing_examples.select do |example|
|
61
|
+
run_examples([example])
|
62
|
+
last_command_passed?
|
63
|
+
end
|
64
|
+
|
65
|
+
if order_dependent_examples.size > 0
|
66
|
+
puts "Order dependenc#{ order_dependent_examples.size > 1 ? 'ies' : 'y'} detected:"
|
67
|
+
order_dependent_examples.each do |example|
|
68
|
+
puts "\t#{example['full_description']}".red
|
69
|
+
end
|
11
70
|
else
|
12
|
-
puts
|
71
|
+
puts 'No order dependencies.'.green
|
72
|
+
end
|
73
|
+
|
74
|
+
order_dependent_examples.each do |example|
|
75
|
+
puts
|
76
|
+
puts "Culprits for #{example['full_description']}:"
|
77
|
+
|
78
|
+
culprits = examples.take_while do |e|
|
79
|
+
example['file_path'] != e['file_path'] ||
|
80
|
+
example['line_number'] != e['line_number']
|
81
|
+
end
|
82
|
+
|
83
|
+
culprit_count_theory = 1
|
84
|
+
while culprits.size > culprit_count_theory
|
85
|
+
found_useless_group = false
|
86
|
+
culprits.each_slice(culprits.size / (culprit_count_theory + 1)) do |excluded_culprits|
|
87
|
+
culprit_group = culprits - excluded_culprits
|
88
|
+
|
89
|
+
run_examples culprit_group + [example]
|
90
|
+
if last_command_failed?
|
91
|
+
culprits = culprit_group
|
92
|
+
|
93
|
+
found_useless_group = true
|
94
|
+
break
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
culprit_count_theory += 1 unless found_useless_group
|
99
|
+
end
|
100
|
+
|
101
|
+
culprits.each { |candidate| puts candidate['full_description'].green }
|
102
|
+
|
103
|
+
puts example['full_description'].red
|
104
|
+
|
105
|
+
puts (run_examples_command culprits + [example]).red
|
13
106
|
end
|
14
107
|
|
15
|
-
puts "No order dependencies."
|
@@ -0,0 +1,23 @@
|
|
1
|
+
Feature: accepts seeds for first rspec run
|
2
|
+
|
3
|
+
@wip
|
4
|
+
Scenario: suite requires seeds
|
5
|
+
Given a file named "spec/failing_spec.rb" with:
|
6
|
+
"""ruby
|
7
|
+
RSpec.describe 'a test that' do
|
8
|
+
cause_failure = false
|
9
|
+
it 'fails' do
|
10
|
+
expect(cause_failure).to be false
|
11
|
+
end
|
12
|
+
|
13
|
+
it 'passes' do
|
14
|
+
cause_failure = true
|
15
|
+
end
|
16
|
+
end
|
17
|
+
"""
|
18
|
+
When I run `rspec-bisect --seed 2`
|
19
|
+
Then the output should contain:
|
20
|
+
"""
|
21
|
+
a test that passes
|
22
|
+
a test that fails
|
23
|
+
"""
|
@@ -0,0 +1,27 @@
|
|
1
|
+
Feature: prints order dependencies for multiple test dependencies
|
2
|
+
|
3
|
+
Scenario: two tests causes another test to fail
|
4
|
+
Given a file named "spec/failing_spec.rb" with:
|
5
|
+
"""ruby
|
6
|
+
RSpec.describe 'a test that' do
|
7
|
+
cause_failure = 0
|
8
|
+
it 'passes' do
|
9
|
+
cause_failure += 1
|
10
|
+
end
|
11
|
+
|
12
|
+
it 'also passes' do
|
13
|
+
cause_failure += 1
|
14
|
+
end
|
15
|
+
|
16
|
+
it 'fails' do
|
17
|
+
expect(cause_failure).to be < 2
|
18
|
+
end
|
19
|
+
end
|
20
|
+
"""
|
21
|
+
When I run `rspec-bisect`
|
22
|
+
Then the output should contain:
|
23
|
+
"""
|
24
|
+
a test that passes
|
25
|
+
a test that also passes
|
26
|
+
a test that fails
|
27
|
+
"""
|
@@ -0,0 +1,21 @@
|
|
1
|
+
Feature: prints order dependencies for multiple test dependencies
|
2
|
+
|
3
|
+
Scenario: one test causes another test to fail
|
4
|
+
Given a file named "spec/failing_spec.rb" with:
|
5
|
+
"""ruby
|
6
|
+
RSpec.describe 'a test that' do
|
7
|
+
cause_failure = false
|
8
|
+
it 'passes' do
|
9
|
+
cause_failure = true
|
10
|
+
end
|
11
|
+
|
12
|
+
it 'fails' do
|
13
|
+
expect(cause_failure).to be false
|
14
|
+
end
|
15
|
+
end
|
16
|
+
"""
|
17
|
+
When I run `rspec-bisect`
|
18
|
+
Then the output should contain:
|
19
|
+
"""
|
20
|
+
rspec ./spec/failing_spec.rb:3 ./spec/failing_spec.rb:7
|
21
|
+
"""
|
@@ -0,0 +1,53 @@
|
|
1
|
+
Feature: prints order dependencies
|
2
|
+
|
3
|
+
Scenario: one test causes another test to fail
|
4
|
+
Given a file named "spec/failing_spec.rb" with:
|
5
|
+
"""ruby
|
6
|
+
RSpec.describe 'a test that' do
|
7
|
+
cause_failure = false
|
8
|
+
it 'passes' do
|
9
|
+
cause_failure = true
|
10
|
+
end
|
11
|
+
|
12
|
+
it 'fails' do
|
13
|
+
expect(cause_failure).to be false
|
14
|
+
end
|
15
|
+
end
|
16
|
+
"""
|
17
|
+
When I run `rspec-bisect`
|
18
|
+
Then the output should contain:
|
19
|
+
"""
|
20
|
+
a test that passes
|
21
|
+
a test that fails
|
22
|
+
"""
|
23
|
+
|
24
|
+
Scenario: one test causes two other tests to fail
|
25
|
+
Given a file named "spec/failing_spec.rb" with:
|
26
|
+
"""ruby
|
27
|
+
RSpec.describe 'a test that' do
|
28
|
+
cause_failure = false
|
29
|
+
it 'passes' do
|
30
|
+
cause_failure = true
|
31
|
+
end
|
32
|
+
|
33
|
+
it 'fails' do
|
34
|
+
expect(cause_failure).to be false
|
35
|
+
end
|
36
|
+
|
37
|
+
it 'also fails' do
|
38
|
+
expect(cause_failure).to be false
|
39
|
+
end
|
40
|
+
end
|
41
|
+
"""
|
42
|
+
When I run `rspec-bisect`
|
43
|
+
Then the output should contain:
|
44
|
+
"""
|
45
|
+
a test that passes
|
46
|
+
a test that fails
|
47
|
+
"""
|
48
|
+
|
49
|
+
And the output should contain:
|
50
|
+
"""
|
51
|
+
a test that passes
|
52
|
+
a test that also fails
|
53
|
+
"""
|
@@ -0,0 +1,46 @@
|
|
1
|
+
Feature: which tests have order dependencies
|
2
|
+
|
3
|
+
Scenario: one test causes another test to fail
|
4
|
+
Given a file named "spec/failing_spec.rb" with:
|
5
|
+
"""ruby
|
6
|
+
RSpec.describe 'a test that' do
|
7
|
+
cause_failure = false
|
8
|
+
it 'passes' do
|
9
|
+
cause_failure = true
|
10
|
+
end
|
11
|
+
|
12
|
+
it 'fails' do
|
13
|
+
expect(cause_failure).to be false
|
14
|
+
end
|
15
|
+
end
|
16
|
+
"""
|
17
|
+
When I run `rspec-bisect`
|
18
|
+
Then the output should contain "1 failing test."
|
19
|
+
And the output should not contain "No order dependencies."
|
20
|
+
And the output should contain "Order dependency detected:"
|
21
|
+
And the output should contain "a test that fails"
|
22
|
+
|
23
|
+
Scenario: one test causes two other tests to fail
|
24
|
+
Given a file named "spec/failing_spec.rb" with:
|
25
|
+
"""ruby
|
26
|
+
RSpec.describe 'a test that' do
|
27
|
+
cause_failure = false
|
28
|
+
it 'passes' do
|
29
|
+
cause_failure = true
|
30
|
+
end
|
31
|
+
|
32
|
+
it 'fails' do
|
33
|
+
expect(cause_failure).to be false
|
34
|
+
end
|
35
|
+
|
36
|
+
it 'also fails' do
|
37
|
+
expect(cause_failure).to be false
|
38
|
+
end
|
39
|
+
end
|
40
|
+
"""
|
41
|
+
When I run `rspec-bisect`
|
42
|
+
Then the output should contain "2 failing tests."
|
43
|
+
And the output should not contain "No order dependencies."
|
44
|
+
And the output should contain "Order dependencies detected:"
|
45
|
+
And the output should contain "a test that fails"
|
46
|
+
And the output should contain "a test that also fails"
|
data/features/support/env.rb
CHANGED
data/lib/rspec/bisect/version.rb
CHANGED
data/rspec-bisect.gemspec
CHANGED
@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
|
|
10
10
|
spec.email = ["shelby@shelbyd.com"]
|
11
11
|
spec.summary = %q{Detect order dependencies in rspec test suites.}
|
12
12
|
spec.description = %q{Detect order dependencies in rspec test suites. Also print the minimal set of tests to reproduce the failure(s).}
|
13
|
-
spec.homepage = ""
|
13
|
+
spec.homepage = "https://github.com/shelbyd/rspec-bisect"
|
14
14
|
spec.license = "MIT"
|
15
15
|
|
16
16
|
spec.files = `git ls-files -z`.split("\x0")
|
@@ -20,7 +20,8 @@ Gem::Specification.new do |spec|
|
|
20
20
|
|
21
21
|
spec.add_development_dependency "bundler", "~> 1.6"
|
22
22
|
spec.add_development_dependency "rake", "~> 10.0"
|
23
|
-
spec.add_development_dependency "cucumber"
|
24
|
-
spec.add_development_dependency "aruba"
|
25
|
-
spec.
|
23
|
+
spec.add_development_dependency "cucumber", "1.3.17"
|
24
|
+
spec.add_development_dependency "aruba", "0.6.1"
|
25
|
+
spec.add_dependency "rspec", "~> 3"
|
26
|
+
spec.add_dependency "colorize", "0.7.3"
|
26
27
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rspec-bisect
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shelby Doolittle
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-10-
|
11
|
+
date: 2014-10-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -42,30 +42,30 @@ dependencies:
|
|
42
42
|
name: cucumber
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- -
|
45
|
+
- - '='
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
47
|
+
version: 1.3.17
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- -
|
52
|
+
- - '='
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version:
|
54
|
+
version: 1.3.17
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: aruba
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- -
|
59
|
+
- - '='
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version:
|
61
|
+
version: 0.6.1
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- -
|
66
|
+
- - '='
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version:
|
68
|
+
version: 0.6.1
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: rspec
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -73,13 +73,27 @@ dependencies:
|
|
73
73
|
- - "~>"
|
74
74
|
- !ruby/object:Gem::Version
|
75
75
|
version: '3'
|
76
|
-
type: :
|
76
|
+
type: :runtime
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
80
|
- - "~>"
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '3'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: colorize
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - '='
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: 0.7.3
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - '='
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: 0.7.3
|
83
97
|
description: Detect order dependencies in rspec test suites. Also print the minimal
|
84
98
|
set of tests to reproduce the failure(s).
|
85
99
|
email:
|
@@ -95,14 +109,19 @@ files:
|
|
95
109
|
- README.md
|
96
110
|
- Rakefile
|
97
111
|
- bin/rspec-bisect
|
112
|
+
- features/accepts_seeds.feature
|
113
|
+
- features/failing_tests/multiple_tests_order_dependencies.feature
|
98
114
|
- features/failing_tests/no_order_dependencies.feature
|
115
|
+
- features/failing_tests/prints_command_to_reproduce.feature
|
116
|
+
- features/failing_tests/single_test_order_dependencies.feature
|
117
|
+
- features/failing_tests/which_tests_have_order_dependencies.feature
|
99
118
|
- features/no_tests.feature
|
100
119
|
- features/step_definitions/rspec-bisect_steps.rb
|
101
120
|
- features/support/env.rb
|
102
121
|
- lib/rspec/bisect.rb
|
103
122
|
- lib/rspec/bisect/version.rb
|
104
123
|
- rspec-bisect.gemspec
|
105
|
-
homepage:
|
124
|
+
homepage: https://github.com/shelbyd/rspec-bisect
|
106
125
|
licenses:
|
107
126
|
- MIT
|
108
127
|
metadata: {}
|
@@ -127,7 +146,12 @@ signing_key:
|
|
127
146
|
specification_version: 4
|
128
147
|
summary: Detect order dependencies in rspec test suites.
|
129
148
|
test_files:
|
149
|
+
- features/accepts_seeds.feature
|
150
|
+
- features/failing_tests/multiple_tests_order_dependencies.feature
|
130
151
|
- features/failing_tests/no_order_dependencies.feature
|
152
|
+
- features/failing_tests/prints_command_to_reproduce.feature
|
153
|
+
- features/failing_tests/single_test_order_dependencies.feature
|
154
|
+
- features/failing_tests/which_tests_have_order_dependencies.feature
|
131
155
|
- features/no_tests.feature
|
132
156
|
- features/step_definitions/rspec-bisect_steps.rb
|
133
157
|
- features/support/env.rb
|