flakey_spec_catcher 0.11.2 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/flakey_spec_catcher.gemspec +1 -0
- data/lib/flakey_spec_catcher/cli_override.rb +5 -1
- data/lib/flakey_spec_catcher/rspec_result.rb +28 -9
- data/lib/flakey_spec_catcher/rspec_result_manager.rb +12 -1
- data/lib/flakey_spec_catcher/runner.rb +25 -6
- data/lib/flakey_spec_catcher/timecop_manager.rb +72 -0
- data/lib/flakey_spec_catcher/user_config.rb +13 -15
- data/lib/flakey_spec_catcher/version.rb +1 -1
- metadata +17 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 306e789630ec567368871033c6334208c08a77e69bf901d76d6a2f0f1135470d
|
4
|
+
data.tar.gz: c80514cf76ad55fceda815f217e070f67659feff6bdd6ab923356cd42c722267
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 87c7f748c9577946ab735549729b5695978853225e0b10b01c205ffcbaab28db4140dada4e498eed2d169bcc26e2b940ac169002eda5bc92718159c6201996d2
|
7
|
+
data.tar.gz: 1f10d9cd60b5dd635c8e8759a54df20d54e467882c3c5e95ef2dad6ba3530ee99719371162032aa6775d11e3827db02cb2e1139cbf376763faa18006d98a8bd0
|
data/flakey_spec_catcher.gemspec
CHANGED
@@ -31,6 +31,7 @@ Gem::Specification.new do |gem|
|
|
31
31
|
gem.required_ruby_version = '>= 2.6'
|
32
32
|
|
33
33
|
gem.add_dependency 'rspec', '~> 3.10'
|
34
|
+
gem.add_dependency 'timecop', '~> 0.9'
|
34
35
|
gem.add_development_dependency 'byebug', '~> 11.1'
|
35
36
|
gem.add_development_dependency 'climate_control', '~> 0.2'
|
36
37
|
gem.add_development_dependency 'rake', '~> 13.0'
|
@@ -9,7 +9,7 @@ module FlakeySpecCatcher
|
|
9
9
|
class CliOverride
|
10
10
|
attr_reader :rerun_patterns, :rerun_usage, :repeat_factor, :enable_runs, :excluded_tags, :use_parent, :dry_run
|
11
11
|
attr_reader :output_file, :split_nodes, :split_index, :verbose, :test_options, :break_on_first_failure
|
12
|
-
attr_reader :list_child_specs
|
12
|
+
attr_reader :list_child_specs, :random_timing
|
13
13
|
|
14
14
|
def initialize
|
15
15
|
@dry_run = false
|
@@ -65,6 +65,10 @@ module FlakeySpecCatcher
|
|
65
65
|
@list_child_specs = list_child_specs
|
66
66
|
end
|
67
67
|
|
68
|
+
opts.on('--random-timing', 'Run Specs at random times') do |random_timing|
|
69
|
+
@random_timing = random_timing
|
70
|
+
end
|
71
|
+
|
68
72
|
opts.on('-e', '--excluded-tags=EXCLUDED_TAGS',
|
69
73
|
'Specify tags to exclude in a comma separated list') do |tags|
|
70
74
|
@excluded_tags = parse_tags(tags)
|
@@ -13,13 +13,15 @@ module FlakeySpecCatcher
|
|
13
13
|
# This class will then organize and output the results accordingly.
|
14
14
|
class RspecResult
|
15
15
|
attr_accessor :description, :location, :total_times_run, :total_failures
|
16
|
+
attr_reader :spec_start_times, :failures
|
16
17
|
|
17
|
-
def initialize(description, location, exception_message = nil)
|
18
|
+
def initialize(description, location, spec_start_times, exception_message = nil)
|
18
19
|
@description = description
|
19
20
|
@location = location
|
20
21
|
@total_times_run = 1
|
21
22
|
@total_failures = exception_message ? 1 : 0
|
22
23
|
@failures = []
|
24
|
+
@spec_start_times = spec_start_times
|
23
25
|
add_failure(exception_message) if exception_message
|
24
26
|
end
|
25
27
|
|
@@ -34,30 +36,47 @@ module FlakeySpecCatcher
|
|
34
36
|
|
35
37
|
def add_failure(exception_message)
|
36
38
|
failure = @failures.find { |f| f.exception_message == exception_message }
|
37
|
-
|
39
|
+
if failure
|
40
|
+
failure.add_failure(current_spec_start_time)
|
41
|
+
else
|
42
|
+
@failures.push(RSpecFailure.new(exception_message, current_spec_start_time))
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def current_spec_start_time
|
47
|
+
@spec_start_times[@total_times_run - 1]
|
38
48
|
end
|
39
49
|
|
40
50
|
def print_results
|
41
51
|
puts "\n#{@description.yellow} (#{location})
|
42
52
|
\nFAILED #{total_failures} / #{total_times_run} times"
|
43
53
|
|
44
|
-
@failures.each
|
45
|
-
puts "#{f.count.to_s.indent(2)} times with exception message:"
|
46
|
-
puts f.exception_message.indent(4).red.to_s
|
47
|
-
end
|
54
|
+
@failures.each { |failure| puts failure.failure_summary }
|
48
55
|
end
|
49
56
|
|
50
57
|
# Simple class to contain failed example data
|
51
58
|
class RSpecFailure
|
52
|
-
attr_reader :exception_message, :count
|
59
|
+
attr_reader :exception_message, :count, :spec_start_times
|
53
60
|
|
54
|
-
def initialize(exception_message)
|
61
|
+
def initialize(exception_message, spec_start_time = nil)
|
55
62
|
@exception_message = exception_message
|
56
63
|
@count = 1
|
64
|
+
@spec_start_times = [spec_start_time].compact
|
57
65
|
end
|
58
66
|
|
59
|
-
def add_failure
|
67
|
+
def add_failure(spec_start_time = nil)
|
60
68
|
@count += 1
|
69
|
+
@spec_start_times.push(spec_start_time) unless spec_start_time.nil?
|
70
|
+
end
|
71
|
+
|
72
|
+
def failure_summary
|
73
|
+
summary = "#{count.to_s.indent(2)} times with exception message:\n"
|
74
|
+
summary += exception_message.indent(4).red.to_s
|
75
|
+
return summary if spec_start_times.empty?
|
76
|
+
|
77
|
+
summary += "\n\nFailed at the following times:\n".indent(2)
|
78
|
+
summary += spec_start_times.map { |time| time.indent(4).yellow.to_s }.join("\n").to_s
|
79
|
+
summary
|
61
80
|
end
|
62
81
|
end
|
63
82
|
end
|
@@ -12,14 +12,25 @@ module FlakeySpecCatcher
|
|
12
12
|
# distinct example. It also provides helpers for adding new results,
|
13
13
|
# displaying aggregate results, and checking the state of the collection.
|
14
14
|
class RspecResultManager
|
15
|
+
attr_reader :results
|
16
|
+
|
15
17
|
def initialize(rspec_result_class)
|
16
18
|
@result_class = rspec_result_class
|
17
19
|
@results = []
|
20
|
+
@spec_start_times = []
|
21
|
+
end
|
22
|
+
|
23
|
+
def track_spec_start_times(spec_times)
|
24
|
+
@spec_start_times = spec_times
|
18
25
|
end
|
19
26
|
|
20
27
|
def add_result(desc, location, message = nil)
|
21
28
|
result = @results.find { |r| r.location == location }
|
22
|
-
|
29
|
+
if result
|
30
|
+
result.add_run(message)
|
31
|
+
else
|
32
|
+
@results.push(@result_class.new(desc, location, @spec_start_times, message))
|
33
|
+
end
|
23
34
|
end
|
24
35
|
|
25
36
|
def print_results
|
@@ -7,10 +7,11 @@ require_relative './user_config'
|
|
7
7
|
require_relative './rerun_manager'
|
8
8
|
require_relative './rspec_result_manager'
|
9
9
|
require_relative './event_listener.rb'
|
10
|
+
require_relative './timecop_manager.rb'
|
10
11
|
|
11
12
|
module FlakeySpecCatcher
|
12
13
|
class Runner
|
13
|
-
attr_reader :user_config, :rerun_manager, :git_controller, :test_run_count
|
14
|
+
attr_reader :user_config, :rerun_manager, :git_controller, :test_run_count, :random_dates
|
14
15
|
|
15
16
|
def initialize(test_mode: false,
|
16
17
|
user_config: FlakeySpecCatcher::UserConfig.new,
|
@@ -18,17 +19,17 @@ module FlakeySpecCatcher
|
|
18
19
|
result_manager: FlakeySpecCatcher::RspecResultManager.new(FlakeySpecCatcher::RspecResult),
|
19
20
|
rerun_manager: FlakeySpecCatcher::RerunManager.new(git_controller: git_controller,
|
20
21
|
user_config: user_config))
|
21
|
-
|
22
22
|
@git_controller = git_controller
|
23
23
|
@user_config = user_config
|
24
24
|
@rerun_manager = rerun_manager
|
25
25
|
@rspec_result_manager = result_manager
|
26
26
|
@test_run_count = 0
|
27
27
|
@temp_output_file = @user_config.output_file + 'temp' unless @user_config.output_file == File::NULL
|
28
|
+
enable_random_timing
|
28
29
|
end
|
29
30
|
|
30
31
|
# Debug Methods
|
31
|
-
# rubocop:disable Metrics/AbcSize
|
32
|
+
# rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity
|
32
33
|
def show_settings
|
33
34
|
puts 'Flakey Spec Catcher Settings:'
|
34
35
|
puts " Current Branch: #{@git_controller.branch}" unless @user_config.use_parent
|
@@ -39,12 +40,13 @@ module FlakeySpecCatcher
|
|
39
40
|
puts " Break on first failure: #{@user_config.break_on_first_failure}" if @user_config.break_on_first_failure
|
40
41
|
puts " Node Total: #{@user_config.split_nodes}" if @user_config.split_nodes
|
41
42
|
puts " Node Index: #{@user_config.split_index}" if @user_config.split_index
|
43
|
+
puts ' Random Timing: Enabled' if @user_config.random_timing
|
42
44
|
puts " Changed Specs Detected: #{@git_controller.changed_examples}"
|
43
45
|
return if @user_config.output_file == File::NULL
|
44
46
|
|
45
47
|
puts " Verbose Output Path: #{@user_config.output_file}"
|
46
48
|
end
|
47
|
-
# rubocop:enable Metrics/AbcSize
|
49
|
+
# rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity
|
48
50
|
|
49
51
|
def rerun_preview
|
50
52
|
puts "\n********************************************"
|
@@ -73,8 +75,8 @@ module FlakeySpecCatcher
|
|
73
75
|
|
74
76
|
status = 0
|
75
77
|
@rerun_manager.rerun_capsules.sort.each do |capsule|
|
76
|
-
@user_config.repeat_factor.times do
|
77
|
-
iteration_status =
|
78
|
+
@user_config.repeat_factor.times do |iteration|
|
79
|
+
iteration_status = timecop_wrapper(capsule, iteration)
|
78
80
|
status = [status, iteration_status].max
|
79
81
|
break if @user_config.break_on_first_failure && !status.zero?
|
80
82
|
end
|
@@ -127,6 +129,23 @@ module FlakeySpecCatcher
|
|
127
129
|
end
|
128
130
|
end
|
129
131
|
|
132
|
+
def enable_random_timing
|
133
|
+
@random_dates = if @user_config.random_timing
|
134
|
+
FlakeySpecCatcher::TimecopManager.generate_dates(@user_config.repeat_factor)
|
135
|
+
else
|
136
|
+
[]
|
137
|
+
end
|
138
|
+
@rspec_result_manager.track_spec_start_times(@random_dates)
|
139
|
+
end
|
140
|
+
|
141
|
+
def timecop_wrapper(capsule, iteration)
|
142
|
+
if @user_config.random_timing
|
143
|
+
FlakeySpecCatcher::TimecopManager.randomly_travel_in_time(@random_dates[iteration]) { handle_capsule_rerun(capsule) }
|
144
|
+
else
|
145
|
+
handle_capsule_rerun(capsule)
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
130
149
|
def print_flakey_specs_detected_message
|
131
150
|
puts "\n**********************************************".magenta
|
132
151
|
puts ' Flakiness Detected!'.magenta
|
@@ -0,0 +1,72 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'date'
|
4
|
+
require 'time'
|
5
|
+
require 'timecop'
|
6
|
+
|
7
|
+
module FlakeySpecCatcher
|
8
|
+
class TimecopManager
|
9
|
+
def self.randomly_travel_in_time(date)
|
10
|
+
status = 0
|
11
|
+
Timecop.travel(date) do
|
12
|
+
status = yield
|
13
|
+
end
|
14
|
+
status
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.current_month
|
18
|
+
Time.now.month
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.current_year
|
22
|
+
Time.now.year
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.random_month
|
26
|
+
(current_month..12).to_a.sample
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.random_seconds
|
30
|
+
[0, 59].sample
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.random_minutes
|
34
|
+
[0, (1..58).to_a.sample, 59].sample
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.random_day(year, month)
|
38
|
+
last_day = last_day_of_month(year, month).day
|
39
|
+
[1, (1..last_day).to_a.sample, last_day].sample
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.last_day_of_month(year, month)
|
43
|
+
day = Date.new(year, month, -1).day
|
44
|
+
Time.local(year, month, day, 23, 59, 59)
|
45
|
+
end
|
46
|
+
|
47
|
+
def self.last_day_of_year(year)
|
48
|
+
Time.local(year, 12, 31, 11, 59, 59)
|
49
|
+
end
|
50
|
+
|
51
|
+
def self.random_hour
|
52
|
+
[0, (1..22).to_a.sample, 23].sample
|
53
|
+
end
|
54
|
+
|
55
|
+
def self.random_date
|
56
|
+
year = current_year
|
57
|
+
month = random_month
|
58
|
+
day = random_day(year, month)
|
59
|
+
Time.local(year, month, day, random_hour, random_minutes, random_seconds)
|
60
|
+
end
|
61
|
+
|
62
|
+
def self.prioritized_dates
|
63
|
+
[last_day_of_month(current_year, current_month), last_day_of_year(current_year)]
|
64
|
+
end
|
65
|
+
|
66
|
+
def self.generate_dates(count)
|
67
|
+
dates = prioritized_dates
|
68
|
+
(count - prioritized_dates.count).times { dates << random_date }
|
69
|
+
dates.map(&:iso8601).slice(0, count)
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
@@ -3,20 +3,18 @@
|
|
3
3
|
require_relative './cli_override'
|
4
4
|
module FlakeySpecCatcher
|
5
5
|
# UserConfig class
|
6
|
-
#
|
7
6
|
# Captures user-defined settings to configure RSpec re-run settings.
|
8
7
|
|
9
8
|
class UserConfig
|
10
|
-
attr_reader :repeat_factor, :ignore_files, :ignore_branches, :silent_mode
|
11
|
-
attr_reader :
|
12
|
-
attr_reader :
|
13
|
-
attr_reader :
|
14
|
-
attr_reader :split_nodes, :split_index, :verbose, :test_options
|
15
|
-
attr_reader :break_on_first_failure, :list_child_specs
|
9
|
+
attr_reader :repeat_factor, :ignore_files, :ignore_branches, :silent_mode, :rerun_file_only, :rspec_usage_patterns
|
10
|
+
attr_reader :excluded_tags, :manual_rerun_patterns, :manual_rerun_usage, :enable_runs, :output_file
|
11
|
+
attr_reader :use_parent, :dry_run, :split_nodes, :split_index, :verbose, :test_options, :break_on_first_failure
|
12
|
+
attr_reader :list_child_specs, :random_timing
|
16
13
|
|
17
14
|
USER_CONFIG_ENV_VARS = %w[FSC_REPEAT_FACTOR FSC_IGNORE_FILES FSC_IGNORE_BRANCHES
|
18
15
|
FSC_SILENT_MODE FSC_RERUN_FILE_ONLY FSC_USAGE_PATTERNS
|
19
|
-
FSC_EXCLUDED_TAGS FSC_OUTPUT_FILE FSC_LIST_CHILD_SPECS
|
16
|
+
FSC_EXCLUDED_TAGS FSC_OUTPUT_FILE FSC_LIST_CHILD_SPECS
|
17
|
+
FSC_RANDOM_TIMING].freeze
|
20
18
|
|
21
19
|
def initialize(cli_override: CliOverride.new)
|
22
20
|
apply_env_var_settings
|
@@ -26,7 +24,7 @@ module FlakeySpecCatcher
|
|
26
24
|
|
27
25
|
private
|
28
26
|
|
29
|
-
# rubocop:disable Metrics/AbcSize
|
27
|
+
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
30
28
|
def apply_env_var_settings
|
31
29
|
@repeat_factor = initialize_repeat_factor(ENV['FSC_REPEAT_FACTOR'])
|
32
30
|
@ignore_files = env_var_string_to_array(ENV['FSC_IGNORE_FILES'])
|
@@ -34,6 +32,7 @@ module FlakeySpecCatcher
|
|
34
32
|
@silent_mode = env_var_string_to_bool(ENV['FSC_SILENT_MODE'])
|
35
33
|
@rerun_file_only = env_var_string_to_bool(ENV['FSC_RERUN_FILE_ONLY'])
|
36
34
|
@list_child_specs = env_var_string_to_bool(ENV['FSC_LIST_CHILD_SPECS'])
|
35
|
+
@random_timing = env_var_string_to_bool(ENV['FSC_RANDOM_TIMING'])
|
37
36
|
@rspec_usage_patterns = env_var_string_to_pairs(ENV['FSC_USAGE_PATTERNS'])
|
38
37
|
@excluded_tags = env_var_string_to_tags(ENV['FSC_EXCLUDED_TAGS'])
|
39
38
|
@output_file = ENV['FSC_OUTPUT_FILE']
|
@@ -48,6 +47,7 @@ module FlakeySpecCatcher
|
|
48
47
|
@repeat_factor = @cli_override.repeat_factor if @cli_override.repeat_factor.to_i.positive?
|
49
48
|
@break_on_first_failure = @cli_override.break_on_first_failure
|
50
49
|
@list_child_specs = @cli_override.list_child_specs unless @cli_override.list_child_specs.nil?
|
50
|
+
@random_timing = @cli_override.random_timing unless @cli_override.random_timing.nil?
|
51
51
|
@enable_runs = @cli_override.enable_runs
|
52
52
|
@dry_run = @cli_override.dry_run
|
53
53
|
@split_nodes = @cli_override.split_nodes unless @cli_override.split_nodes.nil?
|
@@ -61,7 +61,7 @@ module FlakeySpecCatcher
|
|
61
61
|
@verbose = @cli_override.verbose
|
62
62
|
@test_options = @cli_override.test_options
|
63
63
|
end
|
64
|
-
# rubocop:enable Metrics/AbcSize
|
64
|
+
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
|
65
65
|
|
66
66
|
def override_settings
|
67
67
|
apply_cli_override
|
@@ -109,11 +109,7 @@ module FlakeySpecCatcher
|
|
109
109
|
end
|
110
110
|
|
111
111
|
def env_var_string_to_bool(env_var)
|
112
|
-
|
113
|
-
true
|
114
|
-
else
|
115
|
-
false
|
116
|
-
end
|
112
|
+
env_var.to_s.casecmp('true').zero?
|
117
113
|
end
|
118
114
|
|
119
115
|
def env_var_string_to_pairs(env_var)
|
@@ -187,6 +183,8 @@ module FlakeySpecCatcher
|
|
187
183
|
@rerun_file_only = env_var_string_to_bool(env_value)
|
188
184
|
when 'FSC_LIST_CHILD_SPECS'
|
189
185
|
@list_child_specs = env_var_string_to_bool(env_value)
|
186
|
+
when 'FSC_RANDOM_TIMING'
|
187
|
+
@random_timing = env_var_string_to_bool(env_value)
|
190
188
|
when 'FSC_USAGE_PATTERNS'
|
191
189
|
@rspec_usage_patterns = env_var_string_to_pairs(env_value)
|
192
190
|
when 'FSC_EXCLUDED_TAGS'
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flakey_spec_catcher
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.12.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Brian Watson
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2021-
|
13
|
+
date: 2021-11-08 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: rspec
|
@@ -26,6 +26,20 @@ dependencies:
|
|
26
26
|
- - "~>"
|
27
27
|
- !ruby/object:Gem::Version
|
28
28
|
version: '3.10'
|
29
|
+
- !ruby/object:Gem::Dependency
|
30
|
+
name: timecop
|
31
|
+
requirement: !ruby/object:Gem::Requirement
|
32
|
+
requirements:
|
33
|
+
- - "~>"
|
34
|
+
- !ruby/object:Gem::Version
|
35
|
+
version: '0.9'
|
36
|
+
type: :runtime
|
37
|
+
prerelease: false
|
38
|
+
version_requirements: !ruby/object:Gem::Requirement
|
39
|
+
requirements:
|
40
|
+
- - "~>"
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
version: '0.9'
|
29
43
|
- !ruby/object:Gem::Dependency
|
30
44
|
name: byebug
|
31
45
|
requirement: !ruby/object:Gem::Requirement
|
@@ -126,6 +140,7 @@ files:
|
|
126
140
|
- lib/flakey_spec_catcher/rspec_result.rb
|
127
141
|
- lib/flakey_spec_catcher/rspec_result_manager.rb
|
128
142
|
- lib/flakey_spec_catcher/runner.rb
|
143
|
+
- lib/flakey_spec_catcher/timecop_manager.rb
|
129
144
|
- lib/flakey_spec_catcher/user_config.rb
|
130
145
|
- lib/flakey_spec_catcher/version.rb
|
131
146
|
- lib/helpers/colorize.rb
|