flakey_spec_catcher 0.6.1 → 0.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/flakey_spec_catcher/capsule_manager.rb +26 -3
- data/lib/flakey_spec_catcher/change_context.rb +59 -3
- data/lib/flakey_spec_catcher/cli_override.rb +30 -1
- data/lib/flakey_spec_catcher/rerun_manager.rb +19 -1
- data/lib/flakey_spec_catcher/runner.rb +2 -1
- data/lib/flakey_spec_catcher/user_config.rb +42 -5
- data/lib/flakey_spec_catcher/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 993c31c2baefd8e58b525db89b1ff79ea1d6220c2d515151fda4ce2c2bcd6df7
|
4
|
+
data.tar.gz: 4760b425917fc1284e46eaaa73e8cf48f4f1f6224fc423beb18f749df6756d0f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 59be5545a0be53ffc435a299458d665893c35b7df44d9d3c8a8522792cd3ad68069b777d0ac9bd4bc8f10850ad38f679bcaac5fa198023836fe472b633daf401
|
7
|
+
data.tar.gz: aff2fe42b718049783361bf81d554182928963cc0e9bb0a4a27d9d0ceef7cc72dfbc155e24ad6a1a0c5effb5621ac35b3ae2c69f7c3ed8b16d87107eafecacc4
|
@@ -27,12 +27,14 @@ module FlakeySpecCatcher
|
|
27
27
|
@change_capsules.map(&:file_name).uniq
|
28
28
|
end
|
29
29
|
|
30
|
+
def sorted_change_contexts
|
31
|
+
@change_capsules.map(&:change_contexts).flatten.sort_by { |c| c.line_number.to_i }
|
32
|
+
end
|
33
|
+
|
30
34
|
def condense_reruns
|
31
35
|
# Don't re-run if the parent context of a change will be run
|
32
36
|
reruns = []
|
33
|
-
all_contexts =
|
34
|
-
.sort_by { |c| c.line_number.to_i }
|
35
|
-
|
37
|
+
all_contexts = sorted_change_contexts
|
36
38
|
all_contexts.each do |context|
|
37
39
|
next if reruns.include?(context.file_name) || context.ancestor_present_in_reruns(reruns)
|
38
40
|
|
@@ -40,5 +42,26 @@ module FlakeySpecCatcher
|
|
40
42
|
end
|
41
43
|
reruns
|
42
44
|
end
|
45
|
+
|
46
|
+
def tag_values_equal?(user_excluded_tag_value, rspec_tag_value)
|
47
|
+
return true if user_excluded_tag_value.nil? && rspec_tag_value.nil?
|
48
|
+
|
49
|
+
return false if user_excluded_tag_value.nil? || rspec_tag_value.nil?
|
50
|
+
|
51
|
+
(user_excluded_tag_value.tr('\'\"', '') == rspec_tag_value.tr('\'\" ', ''))
|
52
|
+
end
|
53
|
+
|
54
|
+
def find_reruns_by_tags(user_excluded_tags)
|
55
|
+
dropped_tests = []
|
56
|
+
sorted_change_contexts.each do |context|
|
57
|
+
context.tags.keys.each do |tag|
|
58
|
+
next unless user_excluded_tags.key?(tag) &&
|
59
|
+
tag_values_equal?(user_excluded_tags[tag], context.tags[tag])
|
60
|
+
|
61
|
+
dropped_tests << context.rerun_info
|
62
|
+
end
|
63
|
+
end
|
64
|
+
dropped_tests
|
65
|
+
end
|
43
66
|
end
|
44
67
|
end
|
@@ -15,15 +15,16 @@ module FlakeySpecCatcher
|
|
15
15
|
# the 'describe' block only.
|
16
16
|
class ChangeContext
|
17
17
|
attr_reader :description, :line_number
|
18
|
-
attr_reader :ancestor_contexts, :file_name
|
18
|
+
attr_reader :ancestor_contexts, :file_name, :tags
|
19
19
|
|
20
|
-
def initialize(description:, line_number:, file_name:, ancestor_contexts: [])
|
20
|
+
def initialize(description:, line_number:, file_name:, ancestor_contexts: [], tags: {})
|
21
21
|
@description = description
|
22
22
|
@line_number = line_number
|
23
|
-
|
24
23
|
@file_name = file_name
|
25
24
|
@ancestor_contexts = ancestor_contexts
|
25
|
+
@tags = tags
|
26
26
|
update_descriptions
|
27
|
+
identify_tags_and_values
|
27
28
|
end
|
28
29
|
|
29
30
|
def update_descriptions
|
@@ -33,6 +34,36 @@ module FlakeySpecCatcher
|
|
33
34
|
@line_number = nil
|
34
35
|
end
|
35
36
|
|
37
|
+
def example_group?
|
38
|
+
# If the description starts with 'it' or 'scenario' then the test
|
39
|
+
# can provide assertions/expectations and is not an example group
|
40
|
+
if @description =~ /^\s*(it|scenario)\s+/
|
41
|
+
false
|
42
|
+
else
|
43
|
+
true
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
# Get tag strings which optionally may continue tags with their values
|
48
|
+
# and return a hash
|
49
|
+
def identify_tags_and_values
|
50
|
+
tags_with_values = {}
|
51
|
+
tag_strings = identify_tag_strings
|
52
|
+
return if tag_strings.nil? || tag_strings.empty?
|
53
|
+
|
54
|
+
# Since tags can have a value represented as strings or symbols
|
55
|
+
# we'll only remove the hash rockets and not colons
|
56
|
+
# Example: ":tag => 'special'" => tags_with_values[:tag] = 'special'
|
57
|
+
# Example: ":tag => :special" => tags_with_values[:tag] = :special
|
58
|
+
|
59
|
+
tag_strings.each do |str|
|
60
|
+
tag_and_value = str.sub(/=>/, ' ').split(' ')
|
61
|
+
tags_with_values[tag_and_value[0]] = tag_and_value[1]
|
62
|
+
end
|
63
|
+
|
64
|
+
@tags = tags_with_values
|
65
|
+
end
|
66
|
+
|
36
67
|
def ==(other)
|
37
68
|
@description == other.description &&
|
38
69
|
@line_number == other.line_number &&
|
@@ -54,5 +85,30 @@ module FlakeySpecCatcher
|
|
54
85
|
end
|
55
86
|
false
|
56
87
|
end
|
88
|
+
|
89
|
+
private
|
90
|
+
|
91
|
+
# Since RSpec doesn't allow for tags to be excluded for runs when explicitly
|
92
|
+
# specifying an example group via `rspec test:line_number`, we have to
|
93
|
+
# do the filtering ourselves which requires us to also identify tags
|
94
|
+
# Assume here that tags are comma delimited following the initial description
|
95
|
+
def identify_tag_strings
|
96
|
+
return if @description.empty?
|
97
|
+
|
98
|
+
tags = @description.split(',')
|
99
|
+
return if tags.count == 1
|
100
|
+
|
101
|
+
# Remove the description
|
102
|
+
tags.shift
|
103
|
+
# Remove the trailing ' do' from the last tag
|
104
|
+
tags.last.sub!(/ do$/, '')
|
105
|
+
|
106
|
+
# Remove trailing and leading whitespace
|
107
|
+
tags.map!(&:strip)
|
108
|
+
|
109
|
+
# Grab tags which are represented as symbols
|
110
|
+
tags.select! { |tag| tag.start_with? ':' }
|
111
|
+
tags
|
112
|
+
end
|
57
113
|
end
|
58
114
|
end
|
@@ -7,10 +7,11 @@ module FlakeySpecCatcher
|
|
7
7
|
#
|
8
8
|
# Captures command line arguments for manual re-runs
|
9
9
|
class CliOverride
|
10
|
-
attr_reader :rerun_pattern, :rerun_usage, :repeat_factor, :enable_runs
|
10
|
+
attr_reader :rerun_pattern, :rerun_usage, :repeat_factor, :enable_runs, :excluded_tags
|
11
11
|
|
12
12
|
def initialize
|
13
13
|
@enable_runs = true
|
14
|
+
@excluded_tags = []
|
14
15
|
parse_command_line_args
|
15
16
|
end
|
16
17
|
|
@@ -40,6 +41,11 @@ module FlakeySpecCatcher
|
|
40
41
|
@repeat_factor = parsed_repeat_factor if parsed_repeat_factor.positive?
|
41
42
|
end
|
42
43
|
|
44
|
+
opts.on('-e', '--excluded-tags=EXCLUDED_TAGS',
|
45
|
+
'Specify tags to exclude in a comma separated list') do |tags|
|
46
|
+
@excluded_tags = parse_tags(tags)
|
47
|
+
end
|
48
|
+
|
43
49
|
opts.on('-v', '--version', 'Prints current flakey_spec_catcher_version') do
|
44
50
|
puts "flakey_spec_catcher Version: #{FlakeySpecCatcher::VERSION}"
|
45
51
|
@enable_runs = false
|
@@ -56,5 +62,28 @@ module FlakeySpecCatcher
|
|
56
62
|
def remove_formatter_quotes(env_var)
|
57
63
|
env_var.tr('\'\"', '')
|
58
64
|
end
|
65
|
+
|
66
|
+
def parse_tags(tag_string)
|
67
|
+
tags = tag_string.tr('\'\" ', '').split(',').uniq
|
68
|
+
tags.map do |tag|
|
69
|
+
if tag[0] == ':'
|
70
|
+
tag
|
71
|
+
else
|
72
|
+
tag.prepend(':') unless tag[0] == ':'
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
# Since tags can have a value represented as strings or symbols
|
77
|
+
# we'll only remove the hash rockets and not colons
|
78
|
+
# Example: ":tag => 'special'" => tags_with_values[:tag] = 'special'
|
79
|
+
# Example: ":tag => :special" => tags_with_values[:tag] = :special
|
80
|
+
tags_with_values = {}
|
81
|
+
tags.each do |str|
|
82
|
+
tag_and_value = str.sub(/=>/, ' ').split(' ')
|
83
|
+
tags_with_values[tag_and_value[0]] = tag_and_value[1]
|
84
|
+
end
|
85
|
+
|
86
|
+
tags_with_values
|
87
|
+
end
|
59
88
|
end
|
60
89
|
end
|
@@ -32,12 +32,30 @@ module FlakeySpecCatcher
|
|
32
32
|
def tests_for_rerun
|
33
33
|
tests = if @user_config.rerun_file_only
|
34
34
|
@git_controller.capsule_manager.changed_files
|
35
|
+
# If no tags are specified, condense re-runs else expand them
|
36
|
+
elsif @user_config.excluded_tags.empty?
|
37
|
+
condense_reruns - identify_tag_excluded_reruns
|
35
38
|
else
|
36
|
-
|
39
|
+
all_non_example_groups - identify_tag_excluded_reruns
|
37
40
|
end
|
38
41
|
filter_reruns_by_ignore_files(tests)
|
39
42
|
end
|
40
43
|
|
44
|
+
def condense_reruns
|
45
|
+
@git_controller.capsule_manager.condense_reruns
|
46
|
+
end
|
47
|
+
|
48
|
+
# Get all change contexts that can invoke expectations/assertions
|
49
|
+
def all_non_example_groups
|
50
|
+
@git_controller.capsule_manager.sorted_change_contexts.reject(&:example_group?)
|
51
|
+
.map(&:rerun_info)
|
52
|
+
end
|
53
|
+
|
54
|
+
# Identifies all contexts that have a tag present
|
55
|
+
def identify_tag_excluded_reruns
|
56
|
+
@git_controller.capsule_manager.find_reruns_by_tags(@user_config.excluded_tags)
|
57
|
+
end
|
58
|
+
|
41
59
|
def pair_reruns_with_usages
|
42
60
|
reruns = tests_for_rerun
|
43
61
|
configured_usage_patterns = @user_config.rspec_usage_patterns
|
@@ -73,7 +73,8 @@ module FlakeySpecCatcher
|
|
73
73
|
def invoke_rspec_runner(test)
|
74
74
|
configure_listener
|
75
75
|
# Pass in CLI options to suppress normal output, and only run the specified test
|
76
|
-
|
76
|
+
rspec_args = ['--out', '/dev/null', test]
|
77
|
+
return_status = RSpec::Core::Runner.run(rspec_args)
|
77
78
|
RSpec.clear_examples
|
78
79
|
return_status
|
79
80
|
end
|
@@ -5,9 +5,11 @@ module FlakeySpecCatcher
|
|
5
5
|
# UserConfig class
|
6
6
|
#
|
7
7
|
# Captures user-defined settings to configure RSpec re-run settings.
|
8
|
+
|
9
|
+
# rubocop:disable Metrics/ClassLength
|
8
10
|
class UserConfig
|
9
11
|
attr_reader :repeat_factor, :ignore_files, :ignore_branches, :silent_mode
|
10
|
-
attr_reader :rerun_file_only, :rspec_usage_patterns
|
12
|
+
attr_reader :rerun_file_only, :rspec_usage_patterns, :excluded_tags
|
11
13
|
attr_reader :manual_rerun_pattern, :manual_rerun_usage
|
12
14
|
attr_reader :enable_runs
|
13
15
|
|
@@ -15,17 +17,24 @@ module FlakeySpecCatcher
|
|
15
17
|
FSC_SILENT_MODE FSC_RERUN_FILE_ONLY FSC_USAGE_PATTERNS].freeze
|
16
18
|
|
17
19
|
def initialize(cli_override: CliOverride.new)
|
20
|
+
apply_env_var_settings
|
21
|
+
@cli_override = cli_override
|
22
|
+
override_settings
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
# rubocop:disable Metrics/AbcSize
|
28
|
+
def apply_env_var_settings
|
18
29
|
@repeat_factor = initialize_repeat_factor(ENV['FSC_REPEAT_FACTOR'])
|
19
30
|
@ignore_files = env_var_string_to_array(ENV['FSC_IGNORE_FILES'])
|
20
31
|
@ignore_branches = env_var_string_to_array(ENV['FSC_IGNORE_BRANCHES'])
|
21
32
|
@silent_mode = env_var_string_to_bool(ENV['FSC_SILENT_MODE'])
|
22
33
|
@rerun_file_only = env_var_string_to_bool(ENV['FSC_RERUN_FILE_ONLY'])
|
23
34
|
@rspec_usage_patterns = env_var_string_to_pairs(ENV['FSC_USAGE_PATTERNS'])
|
24
|
-
@
|
25
|
-
override_settings
|
35
|
+
@excluded_tags = env_var_string_to_tags(ENV['FSC_EXCLUDED_TAGS'])
|
26
36
|
end
|
27
|
-
|
28
|
-
private
|
37
|
+
# rubocop:enable Metrics/AbcSize
|
29
38
|
|
30
39
|
def override_settings
|
31
40
|
apply_cli_override
|
@@ -39,6 +48,11 @@ module FlakeySpecCatcher
|
|
39
48
|
@manual_rerun_usage = @cli_override.rerun_usage
|
40
49
|
@repeat_factor = @cli_override.repeat_factor if @cli_override.repeat_factor.to_i.positive?
|
41
50
|
@enable_runs = @cli_override.enable_runs
|
51
|
+
@excluded_tags = if @cli_override.excluded_tags.empty?
|
52
|
+
@excluded_tags
|
53
|
+
else
|
54
|
+
@cli_override.excluded_tags
|
55
|
+
end
|
42
56
|
end
|
43
57
|
|
44
58
|
def remove_formatter_quotes(env_var)
|
@@ -75,6 +89,28 @@ module FlakeySpecCatcher
|
|
75
89
|
end
|
76
90
|
end
|
77
91
|
|
92
|
+
def env_var_string_to_tags(tag_string)
|
93
|
+
if tag_string.nil? || tag_string.empty?
|
94
|
+
{}
|
95
|
+
else
|
96
|
+
tag_strings = tag_string.tr('\'\" ', '').split(',').uniq
|
97
|
+
tags_with_values = {}
|
98
|
+
return if tag_strings.nil? || tag_strings.empty?
|
99
|
+
|
100
|
+
# Since tags can have a value represented as strings or symbols
|
101
|
+
# we'll only remove the hash rockets and not colons
|
102
|
+
# Example: ":tag => 'special'" => tags_with_values[:tag] = 'special'
|
103
|
+
# Example: ":tag => :special" => tags_with_values[:tag] = :special
|
104
|
+
|
105
|
+
tag_strings.each do |str|
|
106
|
+
tag_and_value = str.sub(/=>/, ' ').split(' ')
|
107
|
+
tags_with_values[tag_and_value[0]] = tag_and_value[1]
|
108
|
+
end
|
109
|
+
|
110
|
+
tags_with_values
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
78
114
|
def format_regex_scan_results(matches)
|
79
115
|
# Scanning the commit message will result in an array of matches
|
80
116
|
# based on the specified regex. If the pattern uses groups like ours does
|
@@ -118,4 +154,5 @@ module FlakeySpecCatcher
|
|
118
154
|
end
|
119
155
|
# rubocop:enable Metrics/CyclomaticComplexity
|
120
156
|
end
|
157
|
+
# rubocop:enable Metrics/ClassLength
|
121
158
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flakey_spec_catcher
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.7.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Brian Watson
|
@@ -122,7 +122,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
122
122
|
- !ruby/object:Gem::Version
|
123
123
|
version: '0'
|
124
124
|
requirements: []
|
125
|
-
|
125
|
+
rubyforge_project:
|
126
|
+
rubygems_version: 2.7.7
|
126
127
|
signing_key:
|
127
128
|
specification_version: 4
|
128
129
|
summary: Run new or changed specs many times to prevent unreliable specs
|