flakey_spec_catcher 0.11.2 → 0.12.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: adf31fd04e49611a2c10622b19350ab444224589815ac5f54948ec6aad53fc3a
4
- data.tar.gz: 23526cc9694ac22402e6b47fbe956197af5be8b5648abba785e822884e1cd747
3
+ metadata.gz: 306e789630ec567368871033c6334208c08a77e69bf901d76d6a2f0f1135470d
4
+ data.tar.gz: c80514cf76ad55fceda815f217e070f67659feff6bdd6ab923356cd42c722267
5
5
  SHA512:
6
- metadata.gz: 9e15abca29cc1a0186ceadcb83f6238c3a847b4f59747e8c8bc0ef23387ed2a8b5cf3777675d32690bfb529b8e7aba6e6e8ba9970d3d9628628392cfa346506b
7
- data.tar.gz: ce915b99014c9eee3210e55663dec289b20dcb39936a28ffd299fc7c0bb003561ef3d67e4a620bf28057e8ae4fe60aca1ed17f2e66436aa6d0f0fdae928e2629
6
+ metadata.gz: 87c7f748c9577946ab735549729b5695978853225e0b10b01c205ffcbaab28db4140dada4e498eed2d169bcc26e2b940ac169002eda5bc92718159c6201996d2
7
+ data.tar.gz: 1f10d9cd60b5dd635c8e8759a54df20d54e467882c3c5e95ef2dad6ba3530ee99719371162032aa6775d11e3827db02cb2e1139cbf376763faa18006d98a8bd0
@@ -31,6 +31,7 @@ Gem::Specification.new do |gem|
31
31
  gem.required_ruby_version = '>= 2.6'
32
32
 
33
33
  gem.add_dependency 'rspec', '~> 3.10'
34
+ gem.add_dependency 'timecop', '~> 0.9'
34
35
  gem.add_development_dependency 'byebug', '~> 11.1'
35
36
  gem.add_development_dependency 'climate_control', '~> 0.2'
36
37
  gem.add_development_dependency 'rake', '~> 13.0'
@@ -9,7 +9,7 @@ module FlakeySpecCatcher
9
9
  class CliOverride
10
10
  attr_reader :rerun_patterns, :rerun_usage, :repeat_factor, :enable_runs, :excluded_tags, :use_parent, :dry_run
11
11
  attr_reader :output_file, :split_nodes, :split_index, :verbose, :test_options, :break_on_first_failure
12
- attr_reader :list_child_specs
12
+ attr_reader :list_child_specs, :random_timing
13
13
 
14
14
  def initialize
15
15
  @dry_run = false
@@ -65,6 +65,10 @@ module FlakeySpecCatcher
65
65
  @list_child_specs = list_child_specs
66
66
  end
67
67
 
68
+ opts.on('--random-timing', 'Run Specs at random times') do |random_timing|
69
+ @random_timing = random_timing
70
+ end
71
+
68
72
  opts.on('-e', '--excluded-tags=EXCLUDED_TAGS',
69
73
  'Specify tags to exclude in a comma separated list') do |tags|
70
74
  @excluded_tags = parse_tags(tags)
@@ -13,13 +13,15 @@ module FlakeySpecCatcher
13
13
  # This class will then organize and output the results accordingly.
14
14
  class RspecResult
15
15
  attr_accessor :description, :location, :total_times_run, :total_failures
16
+ attr_reader :spec_start_times, :failures
16
17
 
17
- def initialize(description, location, exception_message = nil)
18
+ def initialize(description, location, spec_start_times, exception_message = nil)
18
19
  @description = description
19
20
  @location = location
20
21
  @total_times_run = 1
21
22
  @total_failures = exception_message ? 1 : 0
22
23
  @failures = []
24
+ @spec_start_times = spec_start_times
23
25
  add_failure(exception_message) if exception_message
24
26
  end
25
27
 
@@ -34,30 +36,47 @@ module FlakeySpecCatcher
34
36
 
35
37
  def add_failure(exception_message)
36
38
  failure = @failures.find { |f| f.exception_message == exception_message }
37
- failure ? failure.add_failure : @failures.push(RSpecFailure.new(exception_message))
39
+ if failure
40
+ failure.add_failure(current_spec_start_time)
41
+ else
42
+ @failures.push(RSpecFailure.new(exception_message, current_spec_start_time))
43
+ end
44
+ end
45
+
46
+ def current_spec_start_time
47
+ @spec_start_times[@total_times_run - 1]
38
48
  end
39
49
 
40
50
  def print_results
41
51
  puts "\n#{@description.yellow} (#{location})
42
52
  \nFAILED #{total_failures} / #{total_times_run} times"
43
53
 
44
- @failures.each do |f|
45
- puts "#{f.count.to_s.indent(2)} times with exception message:"
46
- puts f.exception_message.indent(4).red.to_s
47
- end
54
+ @failures.each { |failure| puts failure.failure_summary }
48
55
  end
49
56
 
50
57
  # Simple class to contain failed example data
51
58
  class RSpecFailure
52
- attr_reader :exception_message, :count
59
+ attr_reader :exception_message, :count, :spec_start_times
53
60
 
54
- def initialize(exception_message)
61
+ def initialize(exception_message, spec_start_time = nil)
55
62
  @exception_message = exception_message
56
63
  @count = 1
64
+ @spec_start_times = [spec_start_time].compact
57
65
  end
58
66
 
59
- def add_failure
67
+ def add_failure(spec_start_time = nil)
60
68
  @count += 1
69
+ @spec_start_times.push(spec_start_time) unless spec_start_time.nil?
70
+ end
71
+
72
+ def failure_summary
73
+ summary = "#{count.to_s.indent(2)} times with exception message:\n"
74
+ summary += exception_message.indent(4).red.to_s
75
+ return summary if spec_start_times.empty?
76
+
77
+ summary += "\n\nFailed at the following times:\n".indent(2)
78
+ summary += spec_start_times.map { |time| time.indent(4).yellow.to_s }.join("\n").to_s
79
+ summary
61
80
  end
62
81
  end
63
82
  end
@@ -12,14 +12,25 @@ module FlakeySpecCatcher
12
12
  # distinct example. It also provides helpers for adding new results,
13
13
  # displaying aggregate results, and checking the state of the collection.
14
14
  class RspecResultManager
15
+ attr_reader :results
16
+
15
17
  def initialize(rspec_result_class)
16
18
  @result_class = rspec_result_class
17
19
  @results = []
20
+ @spec_start_times = []
21
+ end
22
+
23
+ def track_spec_start_times(spec_times)
24
+ @spec_start_times = spec_times
18
25
  end
19
26
 
20
27
  def add_result(desc, location, message = nil)
21
28
  result = @results.find { |r| r.location == location }
22
- result ? result.add_run(message) : @results.push(@result_class.new(desc, location, message))
29
+ if result
30
+ result.add_run(message)
31
+ else
32
+ @results.push(@result_class.new(desc, location, @spec_start_times, message))
33
+ end
23
34
  end
24
35
 
25
36
  def print_results
@@ -7,10 +7,11 @@ require_relative './user_config'
7
7
  require_relative './rerun_manager'
8
8
  require_relative './rspec_result_manager'
9
9
  require_relative './event_listener.rb'
10
+ require_relative './timecop_manager.rb'
10
11
 
11
12
  module FlakeySpecCatcher
12
13
  class Runner
13
- attr_reader :user_config, :rerun_manager, :git_controller, :test_run_count
14
+ attr_reader :user_config, :rerun_manager, :git_controller, :test_run_count, :random_dates
14
15
 
15
16
  def initialize(test_mode: false,
16
17
  user_config: FlakeySpecCatcher::UserConfig.new,
@@ -18,17 +19,17 @@ module FlakeySpecCatcher
18
19
  result_manager: FlakeySpecCatcher::RspecResultManager.new(FlakeySpecCatcher::RspecResult),
19
20
  rerun_manager: FlakeySpecCatcher::RerunManager.new(git_controller: git_controller,
20
21
  user_config: user_config))
21
-
22
22
  @git_controller = git_controller
23
23
  @user_config = user_config
24
24
  @rerun_manager = rerun_manager
25
25
  @rspec_result_manager = result_manager
26
26
  @test_run_count = 0
27
27
  @temp_output_file = @user_config.output_file + 'temp' unless @user_config.output_file == File::NULL
28
+ enable_random_timing
28
29
  end
29
30
 
30
31
  # Debug Methods
31
- # rubocop:disable Metrics/AbcSize
32
+ # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity
32
33
  def show_settings
33
34
  puts 'Flakey Spec Catcher Settings:'
34
35
  puts " Current Branch: #{@git_controller.branch}" unless @user_config.use_parent
@@ -39,12 +40,13 @@ module FlakeySpecCatcher
39
40
  puts " Break on first failure: #{@user_config.break_on_first_failure}" if @user_config.break_on_first_failure
40
41
  puts " Node Total: #{@user_config.split_nodes}" if @user_config.split_nodes
41
42
  puts " Node Index: #{@user_config.split_index}" if @user_config.split_index
43
+ puts ' Random Timing: Enabled' if @user_config.random_timing
42
44
  puts " Changed Specs Detected: #{@git_controller.changed_examples}"
43
45
  return if @user_config.output_file == File::NULL
44
46
 
45
47
  puts " Verbose Output Path: #{@user_config.output_file}"
46
48
  end
47
- # rubocop:enable Metrics/AbcSize
49
+ # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity
48
50
 
49
51
  def rerun_preview
50
52
  puts "\n********************************************"
@@ -73,8 +75,8 @@ module FlakeySpecCatcher
73
75
 
74
76
  status = 0
75
77
  @rerun_manager.rerun_capsules.sort.each do |capsule|
76
- @user_config.repeat_factor.times do
77
- iteration_status = handle_capsule_rerun(capsule)
78
+ @user_config.repeat_factor.times do |iteration|
79
+ iteration_status = timecop_wrapper(capsule, iteration)
78
80
  status = [status, iteration_status].max
79
81
  break if @user_config.break_on_first_failure && !status.zero?
80
82
  end
@@ -127,6 +129,23 @@ module FlakeySpecCatcher
127
129
  end
128
130
  end
129
131
 
132
+ def enable_random_timing
133
+ @random_dates = if @user_config.random_timing
134
+ FlakeySpecCatcher::TimecopManager.generate_dates(@user_config.repeat_factor)
135
+ else
136
+ []
137
+ end
138
+ @rspec_result_manager.track_spec_start_times(@random_dates)
139
+ end
140
+
141
+ def timecop_wrapper(capsule, iteration)
142
+ if @user_config.random_timing
143
+ FlakeySpecCatcher::TimecopManager.randomly_travel_in_time(@random_dates[iteration]) { handle_capsule_rerun(capsule) }
144
+ else
145
+ handle_capsule_rerun(capsule)
146
+ end
147
+ end
148
+
130
149
  def print_flakey_specs_detected_message
131
150
  puts "\n**********************************************".magenta
132
151
  puts ' Flakiness Detected!'.magenta
@@ -0,0 +1,72 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'date'
4
+ require 'time'
5
+ require 'timecop'
6
+
7
+ module FlakeySpecCatcher
8
+ class TimecopManager
9
+ def self.randomly_travel_in_time(date)
10
+ status = 0
11
+ Timecop.travel(date) do
12
+ status = yield
13
+ end
14
+ status
15
+ end
16
+
17
+ def self.current_month
18
+ Time.now.month
19
+ end
20
+
21
+ def self.current_year
22
+ Time.now.year
23
+ end
24
+
25
+ def self.random_month
26
+ (current_month..12).to_a.sample
27
+ end
28
+
29
+ def self.random_seconds
30
+ [0, 59].sample
31
+ end
32
+
33
+ def self.random_minutes
34
+ [0, (1..58).to_a.sample, 59].sample
35
+ end
36
+
37
+ def self.random_day(year, month)
38
+ last_day = last_day_of_month(year, month).day
39
+ [1, (1..last_day).to_a.sample, last_day].sample
40
+ end
41
+
42
+ def self.last_day_of_month(year, month)
43
+ day = Date.new(year, month, -1).day
44
+ Time.local(year, month, day, 23, 59, 59)
45
+ end
46
+
47
+ def self.last_day_of_year(year)
48
+ Time.local(year, 12, 31, 11, 59, 59)
49
+ end
50
+
51
+ def self.random_hour
52
+ [0, (1..22).to_a.sample, 23].sample
53
+ end
54
+
55
+ def self.random_date
56
+ year = current_year
57
+ month = random_month
58
+ day = random_day(year, month)
59
+ Time.local(year, month, day, random_hour, random_minutes, random_seconds)
60
+ end
61
+
62
+ def self.prioritized_dates
63
+ [last_day_of_month(current_year, current_month), last_day_of_year(current_year)]
64
+ end
65
+
66
+ def self.generate_dates(count)
67
+ dates = prioritized_dates
68
+ (count - prioritized_dates.count).times { dates << random_date }
69
+ dates.map(&:iso8601).slice(0, count)
70
+ end
71
+ end
72
+ end
@@ -3,20 +3,18 @@
3
3
  require_relative './cli_override'
4
4
  module FlakeySpecCatcher
5
5
  # UserConfig class
6
- #
7
6
  # Captures user-defined settings to configure RSpec re-run settings.
8
7
 
9
8
  class UserConfig
10
- attr_reader :repeat_factor, :ignore_files, :ignore_branches, :silent_mode
11
- attr_reader :rerun_file_only, :rspec_usage_patterns, :excluded_tags
12
- attr_reader :manual_rerun_patterns, :manual_rerun_usage
13
- attr_reader :enable_runs, :output_file, :use_parent, :dry_run
14
- attr_reader :split_nodes, :split_index, :verbose, :test_options
15
- attr_reader :break_on_first_failure, :list_child_specs
9
+ attr_reader :repeat_factor, :ignore_files, :ignore_branches, :silent_mode, :rerun_file_only, :rspec_usage_patterns
10
+ attr_reader :excluded_tags, :manual_rerun_patterns, :manual_rerun_usage, :enable_runs, :output_file
11
+ attr_reader :use_parent, :dry_run, :split_nodes, :split_index, :verbose, :test_options, :break_on_first_failure
12
+ attr_reader :list_child_specs, :random_timing
16
13
 
17
14
  USER_CONFIG_ENV_VARS = %w[FSC_REPEAT_FACTOR FSC_IGNORE_FILES FSC_IGNORE_BRANCHES
18
15
  FSC_SILENT_MODE FSC_RERUN_FILE_ONLY FSC_USAGE_PATTERNS
19
- FSC_EXCLUDED_TAGS FSC_OUTPUT_FILE FSC_LIST_CHILD_SPECS].freeze
16
+ FSC_EXCLUDED_TAGS FSC_OUTPUT_FILE FSC_LIST_CHILD_SPECS
17
+ FSC_RANDOM_TIMING].freeze
20
18
 
21
19
  def initialize(cli_override: CliOverride.new)
22
20
  apply_env_var_settings
@@ -26,7 +24,7 @@ module FlakeySpecCatcher
26
24
 
27
25
  private
28
26
 
29
- # rubocop:disable Metrics/AbcSize
27
+ # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
30
28
  def apply_env_var_settings
31
29
  @repeat_factor = initialize_repeat_factor(ENV['FSC_REPEAT_FACTOR'])
32
30
  @ignore_files = env_var_string_to_array(ENV['FSC_IGNORE_FILES'])
@@ -34,6 +32,7 @@ module FlakeySpecCatcher
34
32
  @silent_mode = env_var_string_to_bool(ENV['FSC_SILENT_MODE'])
35
33
  @rerun_file_only = env_var_string_to_bool(ENV['FSC_RERUN_FILE_ONLY'])
36
34
  @list_child_specs = env_var_string_to_bool(ENV['FSC_LIST_CHILD_SPECS'])
35
+ @random_timing = env_var_string_to_bool(ENV['FSC_RANDOM_TIMING'])
37
36
  @rspec_usage_patterns = env_var_string_to_pairs(ENV['FSC_USAGE_PATTERNS'])
38
37
  @excluded_tags = env_var_string_to_tags(ENV['FSC_EXCLUDED_TAGS'])
39
38
  @output_file = ENV['FSC_OUTPUT_FILE']
@@ -48,6 +47,7 @@ module FlakeySpecCatcher
48
47
  @repeat_factor = @cli_override.repeat_factor if @cli_override.repeat_factor.to_i.positive?
49
48
  @break_on_first_failure = @cli_override.break_on_first_failure
50
49
  @list_child_specs = @cli_override.list_child_specs unless @cli_override.list_child_specs.nil?
50
+ @random_timing = @cli_override.random_timing unless @cli_override.random_timing.nil?
51
51
  @enable_runs = @cli_override.enable_runs
52
52
  @dry_run = @cli_override.dry_run
53
53
  @split_nodes = @cli_override.split_nodes unless @cli_override.split_nodes.nil?
@@ -61,7 +61,7 @@ module FlakeySpecCatcher
61
61
  @verbose = @cli_override.verbose
62
62
  @test_options = @cli_override.test_options
63
63
  end
64
- # rubocop:enable Metrics/AbcSize
64
+ # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
65
65
 
66
66
  def override_settings
67
67
  apply_cli_override
@@ -109,11 +109,7 @@ module FlakeySpecCatcher
109
109
  end
110
110
 
111
111
  def env_var_string_to_bool(env_var)
112
- if env_var.to_s.casecmp('true').zero?
113
- true
114
- else
115
- false
116
- end
112
+ env_var.to_s.casecmp('true').zero?
117
113
  end
118
114
 
119
115
  def env_var_string_to_pairs(env_var)
@@ -187,6 +183,8 @@ module FlakeySpecCatcher
187
183
  @rerun_file_only = env_var_string_to_bool(env_value)
188
184
  when 'FSC_LIST_CHILD_SPECS'
189
185
  @list_child_specs = env_var_string_to_bool(env_value)
186
+ when 'FSC_RANDOM_TIMING'
187
+ @random_timing = env_var_string_to_bool(env_value)
190
188
  when 'FSC_USAGE_PATTERNS'
191
189
  @rspec_usage_patterns = env_var_string_to_pairs(env_value)
192
190
  when 'FSC_EXCLUDED_TAGS'
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module FlakeySpecCatcher
4
- VERSION = '0.11.2'
4
+ VERSION = '0.12.0'
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: flakey_spec_catcher
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.11.2
4
+ version: 0.12.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Brian Watson
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2021-10-15 00:00:00.000000000 Z
13
+ date: 2021-11-08 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: rspec
@@ -26,6 +26,20 @@ dependencies:
26
26
  - - "~>"
27
27
  - !ruby/object:Gem::Version
28
28
  version: '3.10'
29
+ - !ruby/object:Gem::Dependency
30
+ name: timecop
31
+ requirement: !ruby/object:Gem::Requirement
32
+ requirements:
33
+ - - "~>"
34
+ - !ruby/object:Gem::Version
35
+ version: '0.9'
36
+ type: :runtime
37
+ prerelease: false
38
+ version_requirements: !ruby/object:Gem::Requirement
39
+ requirements:
40
+ - - "~>"
41
+ - !ruby/object:Gem::Version
42
+ version: '0.9'
29
43
  - !ruby/object:Gem::Dependency
30
44
  name: byebug
31
45
  requirement: !ruby/object:Gem::Requirement
@@ -126,6 +140,7 @@ files:
126
140
  - lib/flakey_spec_catcher/rspec_result.rb
127
141
  - lib/flakey_spec_catcher/rspec_result_manager.rb
128
142
  - lib/flakey_spec_catcher/runner.rb
143
+ - lib/flakey_spec_catcher/timecop_manager.rb
129
144
  - lib/flakey_spec_catcher/user_config.rb
130
145
  - lib/flakey_spec_catcher/version.rb
131
146
  - lib/helpers/colorize.rb