flakey_spec_catcher 0.11.2 → 0.12.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/flakey_spec_catcher.gemspec +1 -0
- data/lib/flakey_spec_catcher/cli_override.rb +5 -1
- data/lib/flakey_spec_catcher/rspec_result.rb +28 -9
- data/lib/flakey_spec_catcher/rspec_result_manager.rb +12 -1
- data/lib/flakey_spec_catcher/runner.rb +25 -6
- data/lib/flakey_spec_catcher/timecop_manager.rb +72 -0
- data/lib/flakey_spec_catcher/user_config.rb +13 -15
- data/lib/flakey_spec_catcher/version.rb +1 -1
- metadata +17 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 306e789630ec567368871033c6334208c08a77e69bf901d76d6a2f0f1135470d
|
4
|
+
data.tar.gz: c80514cf76ad55fceda815f217e070f67659feff6bdd6ab923356cd42c722267
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 87c7f748c9577946ab735549729b5695978853225e0b10b01c205ffcbaab28db4140dada4e498eed2d169bcc26e2b940ac169002eda5bc92718159c6201996d2
|
7
|
+
data.tar.gz: 1f10d9cd60b5dd635c8e8759a54df20d54e467882c3c5e95ef2dad6ba3530ee99719371162032aa6775d11e3827db02cb2e1139cbf376763faa18006d98a8bd0
|
data/flakey_spec_catcher.gemspec
CHANGED
@@ -31,6 +31,7 @@ Gem::Specification.new do |gem|
|
|
31
31
|
gem.required_ruby_version = '>= 2.6'
|
32
32
|
|
33
33
|
gem.add_dependency 'rspec', '~> 3.10'
|
34
|
+
gem.add_dependency 'timecop', '~> 0.9'
|
34
35
|
gem.add_development_dependency 'byebug', '~> 11.1'
|
35
36
|
gem.add_development_dependency 'climate_control', '~> 0.2'
|
36
37
|
gem.add_development_dependency 'rake', '~> 13.0'
|
@@ -9,7 +9,7 @@ module FlakeySpecCatcher
|
|
9
9
|
class CliOverride
|
10
10
|
attr_reader :rerun_patterns, :rerun_usage, :repeat_factor, :enable_runs, :excluded_tags, :use_parent, :dry_run
|
11
11
|
attr_reader :output_file, :split_nodes, :split_index, :verbose, :test_options, :break_on_first_failure
|
12
|
-
attr_reader :list_child_specs
|
12
|
+
attr_reader :list_child_specs, :random_timing
|
13
13
|
|
14
14
|
def initialize
|
15
15
|
@dry_run = false
|
@@ -65,6 +65,10 @@ module FlakeySpecCatcher
|
|
65
65
|
@list_child_specs = list_child_specs
|
66
66
|
end
|
67
67
|
|
68
|
+
opts.on('--random-timing', 'Run Specs at random times') do |random_timing|
|
69
|
+
@random_timing = random_timing
|
70
|
+
end
|
71
|
+
|
68
72
|
opts.on('-e', '--excluded-tags=EXCLUDED_TAGS',
|
69
73
|
'Specify tags to exclude in a comma separated list') do |tags|
|
70
74
|
@excluded_tags = parse_tags(tags)
|
@@ -13,13 +13,15 @@ module FlakeySpecCatcher
|
|
13
13
|
# This class will then organize and output the results accordingly.
|
14
14
|
class RspecResult
|
15
15
|
attr_accessor :description, :location, :total_times_run, :total_failures
|
16
|
+
attr_reader :spec_start_times, :failures
|
16
17
|
|
17
|
-
def initialize(description, location, exception_message = nil)
|
18
|
+
def initialize(description, location, spec_start_times, exception_message = nil)
|
18
19
|
@description = description
|
19
20
|
@location = location
|
20
21
|
@total_times_run = 1
|
21
22
|
@total_failures = exception_message ? 1 : 0
|
22
23
|
@failures = []
|
24
|
+
@spec_start_times = spec_start_times
|
23
25
|
add_failure(exception_message) if exception_message
|
24
26
|
end
|
25
27
|
|
@@ -34,30 +36,47 @@ module FlakeySpecCatcher
|
|
34
36
|
|
35
37
|
def add_failure(exception_message)
|
36
38
|
failure = @failures.find { |f| f.exception_message == exception_message }
|
37
|
-
|
39
|
+
if failure
|
40
|
+
failure.add_failure(current_spec_start_time)
|
41
|
+
else
|
42
|
+
@failures.push(RSpecFailure.new(exception_message, current_spec_start_time))
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def current_spec_start_time
|
47
|
+
@spec_start_times[@total_times_run - 1]
|
38
48
|
end
|
39
49
|
|
40
50
|
def print_results
|
41
51
|
puts "\n#{@description.yellow} (#{location})
|
42
52
|
\nFAILED #{total_failures} / #{total_times_run} times"
|
43
53
|
|
44
|
-
@failures.each
|
45
|
-
puts "#{f.count.to_s.indent(2)} times with exception message:"
|
46
|
-
puts f.exception_message.indent(4).red.to_s
|
47
|
-
end
|
54
|
+
@failures.each { |failure| puts failure.failure_summary }
|
48
55
|
end
|
49
56
|
|
50
57
|
# Simple class to contain failed example data
|
51
58
|
class RSpecFailure
|
52
|
-
attr_reader :exception_message, :count
|
59
|
+
attr_reader :exception_message, :count, :spec_start_times
|
53
60
|
|
54
|
-
def initialize(exception_message)
|
61
|
+
def initialize(exception_message, spec_start_time = nil)
|
55
62
|
@exception_message = exception_message
|
56
63
|
@count = 1
|
64
|
+
@spec_start_times = [spec_start_time].compact
|
57
65
|
end
|
58
66
|
|
59
|
-
def add_failure
|
67
|
+
def add_failure(spec_start_time = nil)
|
60
68
|
@count += 1
|
69
|
+
@spec_start_times.push(spec_start_time) unless spec_start_time.nil?
|
70
|
+
end
|
71
|
+
|
72
|
+
def failure_summary
|
73
|
+
summary = "#{count.to_s.indent(2)} times with exception message:\n"
|
74
|
+
summary += exception_message.indent(4).red.to_s
|
75
|
+
return summary if spec_start_times.empty?
|
76
|
+
|
77
|
+
summary += "\n\nFailed at the following times:\n".indent(2)
|
78
|
+
summary += spec_start_times.map { |time| time.indent(4).yellow.to_s }.join("\n").to_s
|
79
|
+
summary
|
61
80
|
end
|
62
81
|
end
|
63
82
|
end
|
@@ -12,14 +12,25 @@ module FlakeySpecCatcher
|
|
12
12
|
# distinct example. It also provides helpers for adding new results,
|
13
13
|
# displaying aggregate results, and checking the state of the collection.
|
14
14
|
class RspecResultManager
|
15
|
+
attr_reader :results
|
16
|
+
|
15
17
|
def initialize(rspec_result_class)
|
16
18
|
@result_class = rspec_result_class
|
17
19
|
@results = []
|
20
|
+
@spec_start_times = []
|
21
|
+
end
|
22
|
+
|
23
|
+
def track_spec_start_times(spec_times)
|
24
|
+
@spec_start_times = spec_times
|
18
25
|
end
|
19
26
|
|
20
27
|
def add_result(desc, location, message = nil)
|
21
28
|
result = @results.find { |r| r.location == location }
|
22
|
-
|
29
|
+
if result
|
30
|
+
result.add_run(message)
|
31
|
+
else
|
32
|
+
@results.push(@result_class.new(desc, location, @spec_start_times, message))
|
33
|
+
end
|
23
34
|
end
|
24
35
|
|
25
36
|
def print_results
|
@@ -7,10 +7,11 @@ require_relative './user_config'
|
|
7
7
|
require_relative './rerun_manager'
|
8
8
|
require_relative './rspec_result_manager'
|
9
9
|
require_relative './event_listener.rb'
|
10
|
+
require_relative './timecop_manager.rb'
|
10
11
|
|
11
12
|
module FlakeySpecCatcher
|
12
13
|
class Runner
|
13
|
-
attr_reader :user_config, :rerun_manager, :git_controller, :test_run_count
|
14
|
+
attr_reader :user_config, :rerun_manager, :git_controller, :test_run_count, :random_dates
|
14
15
|
|
15
16
|
def initialize(test_mode: false,
|
16
17
|
user_config: FlakeySpecCatcher::UserConfig.new,
|
@@ -18,17 +19,17 @@ module FlakeySpecCatcher
|
|
18
19
|
result_manager: FlakeySpecCatcher::RspecResultManager.new(FlakeySpecCatcher::RspecResult),
|
19
20
|
rerun_manager: FlakeySpecCatcher::RerunManager.new(git_controller: git_controller,
|
20
21
|
user_config: user_config))
|
21
|
-
|
22
22
|
@git_controller = git_controller
|
23
23
|
@user_config = user_config
|
24
24
|
@rerun_manager = rerun_manager
|
25
25
|
@rspec_result_manager = result_manager
|
26
26
|
@test_run_count = 0
|
27
27
|
@temp_output_file = @user_config.output_file + 'temp' unless @user_config.output_file == File::NULL
|
28
|
+
enable_random_timing
|
28
29
|
end
|
29
30
|
|
30
31
|
# Debug Methods
|
31
|
-
# rubocop:disable Metrics/AbcSize
|
32
|
+
# rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity
|
32
33
|
def show_settings
|
33
34
|
puts 'Flakey Spec Catcher Settings:'
|
34
35
|
puts " Current Branch: #{@git_controller.branch}" unless @user_config.use_parent
|
@@ -39,12 +40,13 @@ module FlakeySpecCatcher
|
|
39
40
|
puts " Break on first failure: #{@user_config.break_on_first_failure}" if @user_config.break_on_first_failure
|
40
41
|
puts " Node Total: #{@user_config.split_nodes}" if @user_config.split_nodes
|
41
42
|
puts " Node Index: #{@user_config.split_index}" if @user_config.split_index
|
43
|
+
puts ' Random Timing: Enabled' if @user_config.random_timing
|
42
44
|
puts " Changed Specs Detected: #{@git_controller.changed_examples}"
|
43
45
|
return if @user_config.output_file == File::NULL
|
44
46
|
|
45
47
|
puts " Verbose Output Path: #{@user_config.output_file}"
|
46
48
|
end
|
47
|
-
# rubocop:enable Metrics/AbcSize
|
49
|
+
# rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity
|
48
50
|
|
49
51
|
def rerun_preview
|
50
52
|
puts "\n********************************************"
|
@@ -73,8 +75,8 @@ module FlakeySpecCatcher
|
|
73
75
|
|
74
76
|
status = 0
|
75
77
|
@rerun_manager.rerun_capsules.sort.each do |capsule|
|
76
|
-
@user_config.repeat_factor.times do
|
77
|
-
iteration_status =
|
78
|
+
@user_config.repeat_factor.times do |iteration|
|
79
|
+
iteration_status = timecop_wrapper(capsule, iteration)
|
78
80
|
status = [status, iteration_status].max
|
79
81
|
break if @user_config.break_on_first_failure && !status.zero?
|
80
82
|
end
|
@@ -127,6 +129,23 @@ module FlakeySpecCatcher
|
|
127
129
|
end
|
128
130
|
end
|
129
131
|
|
132
|
+
def enable_random_timing
|
133
|
+
@random_dates = if @user_config.random_timing
|
134
|
+
FlakeySpecCatcher::TimecopManager.generate_dates(@user_config.repeat_factor)
|
135
|
+
else
|
136
|
+
[]
|
137
|
+
end
|
138
|
+
@rspec_result_manager.track_spec_start_times(@random_dates)
|
139
|
+
end
|
140
|
+
|
141
|
+
def timecop_wrapper(capsule, iteration)
|
142
|
+
if @user_config.random_timing
|
143
|
+
FlakeySpecCatcher::TimecopManager.randomly_travel_in_time(@random_dates[iteration]) { handle_capsule_rerun(capsule) }
|
144
|
+
else
|
145
|
+
handle_capsule_rerun(capsule)
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
130
149
|
def print_flakey_specs_detected_message
|
131
150
|
puts "\n**********************************************".magenta
|
132
151
|
puts ' Flakiness Detected!'.magenta
|
@@ -0,0 +1,72 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'date'
|
4
|
+
require 'time'
|
5
|
+
require 'timecop'
|
6
|
+
|
7
|
+
module FlakeySpecCatcher
|
8
|
+
class TimecopManager
|
9
|
+
def self.randomly_travel_in_time(date)
|
10
|
+
status = 0
|
11
|
+
Timecop.travel(date) do
|
12
|
+
status = yield
|
13
|
+
end
|
14
|
+
status
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.current_month
|
18
|
+
Time.now.month
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.current_year
|
22
|
+
Time.now.year
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.random_month
|
26
|
+
(current_month..12).to_a.sample
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.random_seconds
|
30
|
+
[0, 59].sample
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.random_minutes
|
34
|
+
[0, (1..58).to_a.sample, 59].sample
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.random_day(year, month)
|
38
|
+
last_day = last_day_of_month(year, month).day
|
39
|
+
[1, (1..last_day).to_a.sample, last_day].sample
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.last_day_of_month(year, month)
|
43
|
+
day = Date.new(year, month, -1).day
|
44
|
+
Time.local(year, month, day, 23, 59, 59)
|
45
|
+
end
|
46
|
+
|
47
|
+
def self.last_day_of_year(year)
|
48
|
+
Time.local(year, 12, 31, 11, 59, 59)
|
49
|
+
end
|
50
|
+
|
51
|
+
def self.random_hour
|
52
|
+
[0, (1..22).to_a.sample, 23].sample
|
53
|
+
end
|
54
|
+
|
55
|
+
def self.random_date
|
56
|
+
year = current_year
|
57
|
+
month = random_month
|
58
|
+
day = random_day(year, month)
|
59
|
+
Time.local(year, month, day, random_hour, random_minutes, random_seconds)
|
60
|
+
end
|
61
|
+
|
62
|
+
def self.prioritized_dates
|
63
|
+
[last_day_of_month(current_year, current_month), last_day_of_year(current_year)]
|
64
|
+
end
|
65
|
+
|
66
|
+
def self.generate_dates(count)
|
67
|
+
dates = prioritized_dates
|
68
|
+
(count - prioritized_dates.count).times { dates << random_date }
|
69
|
+
dates.map(&:iso8601).slice(0, count)
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
@@ -3,20 +3,18 @@
|
|
3
3
|
require_relative './cli_override'
|
4
4
|
module FlakeySpecCatcher
|
5
5
|
# UserConfig class
|
6
|
-
#
|
7
6
|
# Captures user-defined settings to configure RSpec re-run settings.
|
8
7
|
|
9
8
|
class UserConfig
|
10
|
-
attr_reader :repeat_factor, :ignore_files, :ignore_branches, :silent_mode
|
11
|
-
attr_reader :
|
12
|
-
attr_reader :
|
13
|
-
attr_reader :
|
14
|
-
attr_reader :split_nodes, :split_index, :verbose, :test_options
|
15
|
-
attr_reader :break_on_first_failure, :list_child_specs
|
9
|
+
attr_reader :repeat_factor, :ignore_files, :ignore_branches, :silent_mode, :rerun_file_only, :rspec_usage_patterns
|
10
|
+
attr_reader :excluded_tags, :manual_rerun_patterns, :manual_rerun_usage, :enable_runs, :output_file
|
11
|
+
attr_reader :use_parent, :dry_run, :split_nodes, :split_index, :verbose, :test_options, :break_on_first_failure
|
12
|
+
attr_reader :list_child_specs, :random_timing
|
16
13
|
|
17
14
|
USER_CONFIG_ENV_VARS = %w[FSC_REPEAT_FACTOR FSC_IGNORE_FILES FSC_IGNORE_BRANCHES
|
18
15
|
FSC_SILENT_MODE FSC_RERUN_FILE_ONLY FSC_USAGE_PATTERNS
|
19
|
-
FSC_EXCLUDED_TAGS FSC_OUTPUT_FILE FSC_LIST_CHILD_SPECS
|
16
|
+
FSC_EXCLUDED_TAGS FSC_OUTPUT_FILE FSC_LIST_CHILD_SPECS
|
17
|
+
FSC_RANDOM_TIMING].freeze
|
20
18
|
|
21
19
|
def initialize(cli_override: CliOverride.new)
|
22
20
|
apply_env_var_settings
|
@@ -26,7 +24,7 @@ module FlakeySpecCatcher
|
|
26
24
|
|
27
25
|
private
|
28
26
|
|
29
|
-
# rubocop:disable Metrics/AbcSize
|
27
|
+
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
30
28
|
def apply_env_var_settings
|
31
29
|
@repeat_factor = initialize_repeat_factor(ENV['FSC_REPEAT_FACTOR'])
|
32
30
|
@ignore_files = env_var_string_to_array(ENV['FSC_IGNORE_FILES'])
|
@@ -34,6 +32,7 @@ module FlakeySpecCatcher
|
|
34
32
|
@silent_mode = env_var_string_to_bool(ENV['FSC_SILENT_MODE'])
|
35
33
|
@rerun_file_only = env_var_string_to_bool(ENV['FSC_RERUN_FILE_ONLY'])
|
36
34
|
@list_child_specs = env_var_string_to_bool(ENV['FSC_LIST_CHILD_SPECS'])
|
35
|
+
@random_timing = env_var_string_to_bool(ENV['FSC_RANDOM_TIMING'])
|
37
36
|
@rspec_usage_patterns = env_var_string_to_pairs(ENV['FSC_USAGE_PATTERNS'])
|
38
37
|
@excluded_tags = env_var_string_to_tags(ENV['FSC_EXCLUDED_TAGS'])
|
39
38
|
@output_file = ENV['FSC_OUTPUT_FILE']
|
@@ -48,6 +47,7 @@ module FlakeySpecCatcher
|
|
48
47
|
@repeat_factor = @cli_override.repeat_factor if @cli_override.repeat_factor.to_i.positive?
|
49
48
|
@break_on_first_failure = @cli_override.break_on_first_failure
|
50
49
|
@list_child_specs = @cli_override.list_child_specs unless @cli_override.list_child_specs.nil?
|
50
|
+
@random_timing = @cli_override.random_timing unless @cli_override.random_timing.nil?
|
51
51
|
@enable_runs = @cli_override.enable_runs
|
52
52
|
@dry_run = @cli_override.dry_run
|
53
53
|
@split_nodes = @cli_override.split_nodes unless @cli_override.split_nodes.nil?
|
@@ -61,7 +61,7 @@ module FlakeySpecCatcher
|
|
61
61
|
@verbose = @cli_override.verbose
|
62
62
|
@test_options = @cli_override.test_options
|
63
63
|
end
|
64
|
-
# rubocop:enable Metrics/AbcSize
|
64
|
+
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
|
65
65
|
|
66
66
|
def override_settings
|
67
67
|
apply_cli_override
|
@@ -109,11 +109,7 @@ module FlakeySpecCatcher
|
|
109
109
|
end
|
110
110
|
|
111
111
|
def env_var_string_to_bool(env_var)
|
112
|
-
|
113
|
-
true
|
114
|
-
else
|
115
|
-
false
|
116
|
-
end
|
112
|
+
env_var.to_s.casecmp('true').zero?
|
117
113
|
end
|
118
114
|
|
119
115
|
def env_var_string_to_pairs(env_var)
|
@@ -187,6 +183,8 @@ module FlakeySpecCatcher
|
|
187
183
|
@rerun_file_only = env_var_string_to_bool(env_value)
|
188
184
|
when 'FSC_LIST_CHILD_SPECS'
|
189
185
|
@list_child_specs = env_var_string_to_bool(env_value)
|
186
|
+
when 'FSC_RANDOM_TIMING'
|
187
|
+
@random_timing = env_var_string_to_bool(env_value)
|
190
188
|
when 'FSC_USAGE_PATTERNS'
|
191
189
|
@rspec_usage_patterns = env_var_string_to_pairs(env_value)
|
192
190
|
when 'FSC_EXCLUDED_TAGS'
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flakey_spec_catcher
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.12.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Brian Watson
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2021-
|
13
|
+
date: 2021-11-08 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: rspec
|
@@ -26,6 +26,20 @@ dependencies:
|
|
26
26
|
- - "~>"
|
27
27
|
- !ruby/object:Gem::Version
|
28
28
|
version: '3.10'
|
29
|
+
- !ruby/object:Gem::Dependency
|
30
|
+
name: timecop
|
31
|
+
requirement: !ruby/object:Gem::Requirement
|
32
|
+
requirements:
|
33
|
+
- - "~>"
|
34
|
+
- !ruby/object:Gem::Version
|
35
|
+
version: '0.9'
|
36
|
+
type: :runtime
|
37
|
+
prerelease: false
|
38
|
+
version_requirements: !ruby/object:Gem::Requirement
|
39
|
+
requirements:
|
40
|
+
- - "~>"
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
version: '0.9'
|
29
43
|
- !ruby/object:Gem::Dependency
|
30
44
|
name: byebug
|
31
45
|
requirement: !ruby/object:Gem::Requirement
|
@@ -126,6 +140,7 @@ files:
|
|
126
140
|
- lib/flakey_spec_catcher/rspec_result.rb
|
127
141
|
- lib/flakey_spec_catcher/rspec_result_manager.rb
|
128
142
|
- lib/flakey_spec_catcher/runner.rb
|
143
|
+
- lib/flakey_spec_catcher/timecop_manager.rb
|
129
144
|
- lib/flakey_spec_catcher/user_config.rb
|
130
145
|
- lib/flakey_spec_catcher/version.rb
|
131
146
|
- lib/helpers/colorize.rb
|