flakey_spec_catcher 0.9.4 → 0.10.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/README.md +2 -0
- data/flakey_spec_catcher.gemspec +6 -4
- data/lib/flakey_spec_catcher/change_capsule.rb +1 -1
- data/lib/flakey_spec_catcher/cli_override.rb +15 -1
- data/lib/flakey_spec_catcher/git_controller.rb +7 -40
- data/lib/flakey_spec_catcher/runner.rb +29 -5
- data/lib/flakey_spec_catcher/user_config.rb +6 -2
- data/lib/flakey_spec_catcher/version.rb +1 -1
- metadata +43 -15
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9894969be42fdb7dfafa961942ce16b1313ca3cffdc746d9613ebe1398d39069
|
4
|
+
data.tar.gz: 75238a79f25ddb8cec6fd60064e9f06d5f5651b8bb3c27e423f317f653f65774
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d3698a10fd2c5d4ae75f3fe0250d7578a82906eb6ad478c759756911501b8f53125a63f1ced325e51c6115a9f76c7ac2264ae28a42c8b65273376186e8e7aa34
|
7
|
+
data.tar.gz: 440e5ef180775556f3292548330cf9572db13d82394921fbbca6053b37ed4c088aa5c37b9dcb06ab2fb4049879c7fe2a093a60cf05da649abb950bf01d0af6cf
|
data/.gitignore
CHANGED
data/README.md
CHANGED
@@ -160,8 +160,10 @@ FSC_USAGE_PATTERNS = '{ spec/ui => bundle exec rspec }, { spec/api => parallel_r
|
|
160
160
|
--node-index Specify the index this node represents in a split run
|
161
161
|
-d, --dry-run Performs all setup but doesn't run any tests
|
162
162
|
--dry-run-quiet Prints list of tests to be run
|
163
|
+
--verbose Send all output from running tests to stdout
|
163
164
|
-v, --version Prints current flakey_spec_catcher_version
|
164
165
|
-h, --help Displays available flakey_spec_catcher cli overrides
|
166
|
+
--rspec-options '[OPTIONS]' Execute default usage with rspec options
|
165
167
|
```
|
166
168
|
|
167
169
|
Examples:
|
data/flakey_spec_catcher.gemspec
CHANGED
@@ -28,11 +28,13 @@ Gem::Specification.new do |gem|
|
|
28
28
|
gem.require_paths = ['lib']
|
29
29
|
|
30
30
|
gem.metadata['allowed_push_host'] = 'https://rubygems.org'
|
31
|
-
gem.required_ruby_version = '>= 2.
|
31
|
+
gem.required_ruby_version = '>= 2.6'
|
32
32
|
|
33
|
-
gem.add_dependency 'rspec', '~> 3.
|
33
|
+
gem.add_dependency 'rspec', '~> 3.10'
|
34
|
+
gem.add_development_dependency 'byebug', '~> 11.1'
|
34
35
|
gem.add_development_dependency 'climate_control', '~> 0.2'
|
35
|
-
gem.add_development_dependency 'rake', '~>
|
36
|
-
gem.add_development_dependency '
|
36
|
+
gem.add_development_dependency 'rake', '~> 13.0'
|
37
|
+
gem.add_development_dependency 'rubocop', '~> 0.93.1'
|
38
|
+
gem.add_development_dependency 'simplecov', '~> 0.19'
|
37
39
|
end
|
38
40
|
# rubocop:enable Layout/ExtraSpacing, Layout/SpaceAroundOperators
|
@@ -8,13 +8,15 @@ module FlakeySpecCatcher
|
|
8
8
|
# Captures command line arguments for manual re-runs
|
9
9
|
class CliOverride
|
10
10
|
attr_reader :rerun_patterns, :rerun_usage, :repeat_factor, :enable_runs, :excluded_tags, :use_parent, :dry_run
|
11
|
-
attr_reader :output_file, :split_nodes, :split_index
|
11
|
+
attr_reader :output_file, :split_nodes, :split_index, :verbose, :test_options, :break_on_first_failure
|
12
12
|
|
13
13
|
def initialize
|
14
14
|
@dry_run = false
|
15
15
|
@enable_runs = true
|
16
16
|
@excluded_tags = []
|
17
17
|
@use_parent = false
|
18
|
+
@verbose = false
|
19
|
+
@test_options = []
|
18
20
|
parse_command_line_args
|
19
21
|
validate_arguments
|
20
22
|
end
|
@@ -54,6 +56,10 @@ module FlakeySpecCatcher
|
|
54
56
|
@repeat_factor = remove_formatter_quotes(repeat).to_i
|
55
57
|
end
|
56
58
|
|
59
|
+
opts.on('--break-on-first-failure', 'Break on first failure') do |break_on_first_failure|
|
60
|
+
@break_on_first_failure = break_on_first_failure
|
61
|
+
end
|
62
|
+
|
57
63
|
opts.on('-e', '--excluded-tags=EXCLUDED_TAGS',
|
58
64
|
'Specify tags to exclude in a comma separated list') do |tags|
|
59
65
|
@excluded_tags = parse_tags(tags)
|
@@ -78,10 +84,18 @@ module FlakeySpecCatcher
|
|
78
84
|
@dry_run = true
|
79
85
|
end
|
80
86
|
|
87
|
+
opts.on('--verbose', 'Send all output from running tests to stdout') do
|
88
|
+
@verbose = true
|
89
|
+
end
|
90
|
+
|
81
91
|
opts.on('-h', '--help', 'Displays available flakey_spec_catcher cli overrides') do
|
82
92
|
puts opts
|
83
93
|
@enable_runs = false
|
84
94
|
end
|
95
|
+
|
96
|
+
opts.on("--rspec-options '[OPTIONS]'", 'execute default usage with rspec options') do |arg|
|
97
|
+
@test_options = arg.split(/[ ](?=(?:[^"]*"[^"]*")*[^"]*$)(?=(?:[^']*'[^']*')*[^']*$)/)
|
98
|
+
end
|
85
99
|
end.parse!
|
86
100
|
end
|
87
101
|
|
@@ -56,45 +56,12 @@ module FlakeySpecCatcher
|
|
56
56
|
return nil if @remote.nil?
|
57
57
|
|
58
58
|
working_branch = `git branch | grep '*'`.delete('*').gsub(/\s+/, '')
|
59
|
-
|
60
|
-
|
61
|
-
# Assume master if no matches
|
62
|
-
|
63
|
-
# Example output
|
64
|
-
# * remote origin
|
65
|
-
# Fetch URL: gerrit:repo
|
66
|
-
# Push URL: gerrit:repo
|
67
|
-
# HEAD branch: master
|
68
|
-
# Remote branches:
|
69
|
-
# dev/reports tracked
|
70
|
-
# edge tracked
|
71
|
-
# master tracked
|
72
|
-
# Local branch configured for 'git pull':
|
73
|
-
# master merges with remote master
|
74
|
-
# Local refs configured for 'git push':
|
75
|
-
# dev/reports pushes to dev/reports (up to date)
|
76
|
-
# master pushes to master (up to date)
|
77
|
-
|
78
|
-
remote_branches = `git remote show #{@remote}`.split("\n")
|
79
|
-
|
80
|
-
# Separate 'dev/reports pushes to dev/reports (up to date)' into
|
81
|
-
# ['dev/reports', 'dev/reports'] or [<LOCAL BRANCH>, <REMOTE BRANCH>]
|
82
|
-
remote_pairs = remote_branches.map { |r| r.scan(/(\S*)\s+pushes to\s+(\S*)\s+/).flatten }
|
83
|
-
|
84
|
-
# check if the working branch (currently checked out branch) corresponds to a remote_pair
|
85
|
-
# if so, use that remote pair for comparison, else use master
|
86
|
-
match = remote_pairs.find do |pair|
|
87
|
-
# working branch (pair[0]) pushes to remote (pair[1])
|
88
|
-
pair[0] == working_branch
|
89
|
-
end
|
59
|
+
branch_remote = `git config branch.#{working_branch}.remote`.strip
|
60
|
+
return 'master' unless @remote == branch_remote
|
90
61
|
|
91
|
-
remote_branch =
|
92
|
-
|
93
|
-
|
94
|
-
# match is formatted as [working_branch, remote]
|
95
|
-
match[1]
|
96
|
-
end
|
97
|
-
remote_branch
|
62
|
+
remote_branch = `git config branch.#{branch}.merge`.strip.sub(%r{^refs/heads/}, '')
|
63
|
+
remote_branch = nil if remote_branch.empty?
|
64
|
+
remote_branch || 'master'
|
98
65
|
end
|
99
66
|
|
100
67
|
def initialize_git_comparison(test_mode)
|
@@ -108,7 +75,7 @@ module FlakeySpecCatcher
|
|
108
75
|
@git_comparison = "#{@base_commit_sha}..#{@working_commit_sha}"
|
109
76
|
end
|
110
77
|
|
111
|
-
# rubocop:disable Metrics/
|
78
|
+
# rubocop:disable Metrics/CyclomaticComplexity
|
112
79
|
def parse_changes
|
113
80
|
# For each file, get the change block
|
114
81
|
diff_files.each do |filename|
|
@@ -128,7 +95,7 @@ module FlakeySpecCatcher
|
|
128
95
|
end
|
129
96
|
end
|
130
97
|
end
|
131
|
-
# rubocop:enable Metrics/
|
98
|
+
# rubocop:enable Metrics/CyclomaticComplexity
|
132
99
|
|
133
100
|
def identify_change_contexts
|
134
101
|
@capsule_manager.change_capsules.each(&:fill_contexts)
|
@@ -24,6 +24,7 @@ module FlakeySpecCatcher
|
|
24
24
|
@rerun_manager = rerun_manager
|
25
25
|
@rspec_result_manager = result_manager
|
26
26
|
@test_run_count = 0
|
27
|
+
@temp_output_file = @user_config.output_file + 'temp' unless @user_config.output_file == File::NULL
|
27
28
|
end
|
28
29
|
|
29
30
|
# Debug Methods
|
@@ -35,10 +36,11 @@ module FlakeySpecCatcher
|
|
35
36
|
puts " Current Sha: #{@git_controller.working_commit_sha}"
|
36
37
|
puts " Base Sha: #{@git_controller.base_commit_sha}"
|
37
38
|
puts " Repeat factor: #{@user_config.repeat_factor}"
|
39
|
+
puts " Break on first failure: #{@user_config.break_on_first_failure}" if @user_config.break_on_first_failure
|
38
40
|
puts " Node Total: #{@user_config.split_nodes}" if @user_config.split_nodes
|
39
41
|
puts " Node Index: #{@user_config.split_index}" if @user_config.split_index
|
40
42
|
puts " Changed Specs Detected: #{@git_controller.changed_examples}"
|
41
|
-
return if @user_config.output_file ==
|
43
|
+
return if @user_config.output_file == File::NULL
|
42
44
|
|
43
45
|
puts " Verbose Output Path: #{@user_config.output_file}"
|
44
46
|
end
|
@@ -74,10 +76,12 @@ module FlakeySpecCatcher
|
|
74
76
|
@user_config.repeat_factor.times do
|
75
77
|
iteration_status = handle_capsule_rerun(capsule)
|
76
78
|
status = [status, iteration_status].max
|
79
|
+
break if @user_config.break_on_first_failure && !status.zero?
|
77
80
|
end
|
78
81
|
end
|
79
82
|
|
80
83
|
display_results(status)
|
84
|
+
copy_to_user_output unless @user_config.output_file == File::NULL
|
81
85
|
|
82
86
|
# Always return 0 if silent_mode is enabled
|
83
87
|
@user_config.silent_mode ? 0 : status
|
@@ -93,16 +97,23 @@ module FlakeySpecCatcher
|
|
93
97
|
def invoke_rspec_runner(test)
|
94
98
|
configure_listener
|
95
99
|
# Pass in CLI options to suppress normal output, and only run the specified test
|
96
|
-
rspec_args = ['--out', @user_config.output_file, test]
|
97
|
-
|
100
|
+
rspec_args = ['--format', 'documentation', '--out', @user_config.output_file, test]
|
101
|
+
# Rspec output sent to stdout if verbose option is true
|
102
|
+
rspec_args << '-fd' if @user_config.verbose
|
103
|
+
return_status = RSpec::Core::Runner.run(@user_config.test_options.concat(rspec_args))
|
98
104
|
RSpec.clear_examples
|
105
|
+
copy_output_to_temp_file unless @user_config.output_file == File::NULL
|
99
106
|
return_status
|
100
107
|
end
|
101
108
|
|
102
109
|
def invoke_custom_rspec_runner(usage, testcase)
|
103
|
-
|
110
|
+
if @user_config.verbose
|
111
|
+
$stdout << custom_usage_output = `#{usage} #{testcase.join(' ')}`
|
112
|
+
else
|
113
|
+
custom_usage_output = `#{usage} #{testcase.join(' ')}`
|
114
|
+
end
|
104
115
|
|
105
|
-
File.open(@user_config.output_file, 'a') { |f| f.puts custom_usage_output } if @user_config.output_file !=
|
116
|
+
File.open(@user_config.output_file, 'a') { |f| f.puts custom_usage_output } if @user_config.output_file != File::NULL
|
106
117
|
|
107
118
|
$?.exitstatus # rubocop:disable Style/SpecialGlobalVars
|
108
119
|
end
|
@@ -134,5 +145,18 @@ module FlakeySpecCatcher
|
|
134
145
|
:example_failed, :example_passed
|
135
146
|
end
|
136
147
|
end
|
148
|
+
|
149
|
+
def copy_output_to_temp_file
|
150
|
+
# copy contents of output file, it will get overwritten in RSpec::Core::Runner
|
151
|
+
File.open(@temp_output_file, 'a') do |f|
|
152
|
+
f.puts IO.readlines(@user_config.output_file)
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
def copy_to_user_output
|
157
|
+
# copy all appended output to original output file, delete the temp output file
|
158
|
+
IO.copy_stream(@temp_output_file, @user_config.output_file) if File.exist?(@temp_output_file)
|
159
|
+
File.delete(@temp_output_file) if File.exist?(@temp_output_file)
|
160
|
+
end
|
137
161
|
end
|
138
162
|
end
|
@@ -11,7 +11,8 @@ module FlakeySpecCatcher
|
|
11
11
|
attr_reader :rerun_file_only, :rspec_usage_patterns, :excluded_tags
|
12
12
|
attr_reader :manual_rerun_patterns, :manual_rerun_usage
|
13
13
|
attr_reader :enable_runs, :output_file, :use_parent, :dry_run
|
14
|
-
attr_reader :split_nodes, :split_index
|
14
|
+
attr_reader :split_nodes, :split_index, :verbose, :test_options
|
15
|
+
attr_reader :break_on_first_failure
|
15
16
|
|
16
17
|
USER_CONFIG_ENV_VARS = %w[FSC_REPEAT_FACTOR FSC_IGNORE_FILES FSC_IGNORE_BRANCHES
|
17
18
|
FSC_SILENT_MODE FSC_RERUN_FILE_ONLY FSC_USAGE_PATTERNS
|
@@ -44,6 +45,7 @@ module FlakeySpecCatcher
|
|
44
45
|
@manual_rerun_usage = @cli_override.rerun_usage
|
45
46
|
@use_parent = @cli_override.use_parent
|
46
47
|
@repeat_factor = @cli_override.repeat_factor if @cli_override.repeat_factor.to_i.positive?
|
48
|
+
@break_on_first_failure = @cli_override.break_on_first_failure
|
47
49
|
@enable_runs = @cli_override.enable_runs
|
48
50
|
@dry_run = @cli_override.dry_run
|
49
51
|
@split_nodes = @cli_override.split_nodes unless @cli_override.split_nodes.nil?
|
@@ -54,6 +56,8 @@ module FlakeySpecCatcher
|
|
54
56
|
@cli_override.excluded_tags
|
55
57
|
end
|
56
58
|
@output_file = set_output_file
|
59
|
+
@verbose = @cli_override.verbose
|
60
|
+
@test_options = @cli_override.test_options
|
57
61
|
end
|
58
62
|
# rubocop:enable Metrics/AbcSize
|
59
63
|
|
@@ -68,7 +72,7 @@ module FlakeySpecCatcher
|
|
68
72
|
if !@cli_override.output_file.nil?
|
69
73
|
@cli_override.output_file
|
70
74
|
elsif @output_file.nil? || @output_file.strip.empty?
|
71
|
-
|
75
|
+
File::NULL
|
72
76
|
else
|
73
77
|
@output_file
|
74
78
|
end
|
metadata
CHANGED
@@ -1,16 +1,16 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flakey_spec_catcher
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.10.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Brian Watson
|
8
8
|
- Mikey Hargiss
|
9
9
|
- Ben Nelson
|
10
|
-
autorequire:
|
10
|
+
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date:
|
13
|
+
date: 2021-07-28 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: rspec
|
@@ -18,14 +18,28 @@ dependencies:
|
|
18
18
|
requirements:
|
19
19
|
- - "~>"
|
20
20
|
- !ruby/object:Gem::Version
|
21
|
-
version: '3.
|
21
|
+
version: '3.10'
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
24
|
version_requirements: !ruby/object:Gem::Requirement
|
25
25
|
requirements:
|
26
26
|
- - "~>"
|
27
27
|
- !ruby/object:Gem::Version
|
28
|
-
version: '3.
|
28
|
+
version: '3.10'
|
29
|
+
- !ruby/object:Gem::Dependency
|
30
|
+
name: byebug
|
31
|
+
requirement: !ruby/object:Gem::Requirement
|
32
|
+
requirements:
|
33
|
+
- - "~>"
|
34
|
+
- !ruby/object:Gem::Version
|
35
|
+
version: '11.1'
|
36
|
+
type: :development
|
37
|
+
prerelease: false
|
38
|
+
version_requirements: !ruby/object:Gem::Requirement
|
39
|
+
requirements:
|
40
|
+
- - "~>"
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
version: '11.1'
|
29
43
|
- !ruby/object:Gem::Dependency
|
30
44
|
name: climate_control
|
31
45
|
requirement: !ruby/object:Gem::Requirement
|
@@ -46,29 +60,43 @@ dependencies:
|
|
46
60
|
requirements:
|
47
61
|
- - "~>"
|
48
62
|
- !ruby/object:Gem::Version
|
49
|
-
version: '
|
63
|
+
version: '13.0'
|
64
|
+
type: :development
|
65
|
+
prerelease: false
|
66
|
+
version_requirements: !ruby/object:Gem::Requirement
|
67
|
+
requirements:
|
68
|
+
- - "~>"
|
69
|
+
- !ruby/object:Gem::Version
|
70
|
+
version: '13.0'
|
71
|
+
- !ruby/object:Gem::Dependency
|
72
|
+
name: rubocop
|
73
|
+
requirement: !ruby/object:Gem::Requirement
|
74
|
+
requirements:
|
75
|
+
- - "~>"
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: 0.93.1
|
50
78
|
type: :development
|
51
79
|
prerelease: false
|
52
80
|
version_requirements: !ruby/object:Gem::Requirement
|
53
81
|
requirements:
|
54
82
|
- - "~>"
|
55
83
|
- !ruby/object:Gem::Version
|
56
|
-
version:
|
84
|
+
version: 0.93.1
|
57
85
|
- !ruby/object:Gem::Dependency
|
58
86
|
name: simplecov
|
59
87
|
requirement: !ruby/object:Gem::Requirement
|
60
88
|
requirements:
|
61
89
|
- - "~>"
|
62
90
|
- !ruby/object:Gem::Version
|
63
|
-
version: '0.
|
91
|
+
version: '0.19'
|
64
92
|
type: :development
|
65
93
|
prerelease: false
|
66
94
|
version_requirements: !ruby/object:Gem::Requirement
|
67
95
|
requirements:
|
68
96
|
- - "~>"
|
69
97
|
- !ruby/object:Gem::Version
|
70
|
-
version: '0.
|
71
|
-
description:
|
98
|
+
version: '0.19'
|
99
|
+
description:
|
72
100
|
email:
|
73
101
|
- bwatson@instructure.com
|
74
102
|
- mhargiss@instructure.com
|
@@ -102,12 +130,12 @@ files:
|
|
102
130
|
- lib/flakey_spec_catcher/version.rb
|
103
131
|
- lib/helpers/colorize.rb
|
104
132
|
- lib/helpers/indent_string.rb
|
105
|
-
homepage:
|
133
|
+
homepage:
|
106
134
|
licenses:
|
107
135
|
- MIT
|
108
136
|
metadata:
|
109
137
|
allowed_push_host: https://rubygems.org
|
110
|
-
post_install_message:
|
138
|
+
post_install_message:
|
111
139
|
rdoc_options: []
|
112
140
|
require_paths:
|
113
141
|
- lib
|
@@ -115,15 +143,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
115
143
|
requirements:
|
116
144
|
- - ">="
|
117
145
|
- !ruby/object:Gem::Version
|
118
|
-
version: '2.
|
146
|
+
version: '2.6'
|
119
147
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
120
148
|
requirements:
|
121
149
|
- - ">="
|
122
150
|
- !ruby/object:Gem::Version
|
123
151
|
version: '0'
|
124
152
|
requirements: []
|
125
|
-
rubygems_version: 3.0.
|
126
|
-
signing_key:
|
153
|
+
rubygems_version: 3.0.1
|
154
|
+
signing_key:
|
127
155
|
specification_version: 4
|
128
156
|
summary: Run new or changed specs many times to prevent unreliable specs
|
129
157
|
test_files: []
|