flakey_spec_catcher 0.11.2 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: adf31fd04e49611a2c10622b19350ab444224589815ac5f54948ec6aad53fc3a
4
- data.tar.gz: 23526cc9694ac22402e6b47fbe956197af5be8b5648abba785e822884e1cd747
3
+ metadata.gz: 306e789630ec567368871033c6334208c08a77e69bf901d76d6a2f0f1135470d
4
+ data.tar.gz: c80514cf76ad55fceda815f217e070f67659feff6bdd6ab923356cd42c722267
5
5
  SHA512:
6
- metadata.gz: 9e15abca29cc1a0186ceadcb83f6238c3a847b4f59747e8c8bc0ef23387ed2a8b5cf3777675d32690bfb529b8e7aba6e6e8ba9970d3d9628628392cfa346506b
7
- data.tar.gz: ce915b99014c9eee3210e55663dec289b20dcb39936a28ffd299fc7c0bb003561ef3d67e4a620bf28057e8ae4fe60aca1ed17f2e66436aa6d0f0fdae928e2629
6
+ metadata.gz: 87c7f748c9577946ab735549729b5695978853225e0b10b01c205ffcbaab28db4140dada4e498eed2d169bcc26e2b940ac169002eda5bc92718159c6201996d2
7
+ data.tar.gz: 1f10d9cd60b5dd635c8e8759a54df20d54e467882c3c5e95ef2dad6ba3530ee99719371162032aa6775d11e3827db02cb2e1139cbf376763faa18006d98a8bd0
@@ -31,6 +31,7 @@ Gem::Specification.new do |gem|
31
31
  gem.required_ruby_version = '>= 2.6'
32
32
 
33
33
  gem.add_dependency 'rspec', '~> 3.10'
34
+ gem.add_dependency 'timecop', '~> 0.9'
34
35
  gem.add_development_dependency 'byebug', '~> 11.1'
35
36
  gem.add_development_dependency 'climate_control', '~> 0.2'
36
37
  gem.add_development_dependency 'rake', '~> 13.0'
@@ -9,7 +9,7 @@ module FlakeySpecCatcher
9
9
  class CliOverride
10
10
  attr_reader :rerun_patterns, :rerun_usage, :repeat_factor, :enable_runs, :excluded_tags, :use_parent, :dry_run
11
11
  attr_reader :output_file, :split_nodes, :split_index, :verbose, :test_options, :break_on_first_failure
12
- attr_reader :list_child_specs
12
+ attr_reader :list_child_specs, :random_timing
13
13
 
14
14
  def initialize
15
15
  @dry_run = false
@@ -65,6 +65,10 @@ module FlakeySpecCatcher
65
65
  @list_child_specs = list_child_specs
66
66
  end
67
67
 
68
+ opts.on('--random-timing', 'Run Specs at random times') do |random_timing|
69
+ @random_timing = random_timing
70
+ end
71
+
68
72
  opts.on('-e', '--excluded-tags=EXCLUDED_TAGS',
69
73
  'Specify tags to exclude in a comma separated list') do |tags|
70
74
  @excluded_tags = parse_tags(tags)
@@ -13,13 +13,15 @@ module FlakeySpecCatcher
13
13
  # This class will then organize and output the results accordingly.
14
14
  class RspecResult
15
15
  attr_accessor :description, :location, :total_times_run, :total_failures
16
+ attr_reader :spec_start_times, :failures
16
17
 
17
- def initialize(description, location, exception_message = nil)
18
+ def initialize(description, location, spec_start_times, exception_message = nil)
18
19
  @description = description
19
20
  @location = location
20
21
  @total_times_run = 1
21
22
  @total_failures = exception_message ? 1 : 0
22
23
  @failures = []
24
+ @spec_start_times = spec_start_times
23
25
  add_failure(exception_message) if exception_message
24
26
  end
25
27
 
@@ -34,30 +36,47 @@ module FlakeySpecCatcher
34
36
 
35
37
  def add_failure(exception_message)
36
38
  failure = @failures.find { |f| f.exception_message == exception_message }
37
- failure ? failure.add_failure : @failures.push(RSpecFailure.new(exception_message))
39
+ if failure
40
+ failure.add_failure(current_spec_start_time)
41
+ else
42
+ @failures.push(RSpecFailure.new(exception_message, current_spec_start_time))
43
+ end
44
+ end
45
+
46
+ def current_spec_start_time
47
+ @spec_start_times[@total_times_run - 1]
38
48
  end
39
49
 
40
50
  def print_results
41
51
  puts "\n#{@description.yellow} (#{location})
42
52
  \nFAILED #{total_failures} / #{total_times_run} times"
43
53
 
44
- @failures.each do |f|
45
- puts "#{f.count.to_s.indent(2)} times with exception message:"
46
- puts f.exception_message.indent(4).red.to_s
47
- end
54
+ @failures.each { |failure| puts failure.failure_summary }
48
55
  end
49
56
 
50
57
  # Simple class to contain failed example data
51
58
  class RSpecFailure
52
- attr_reader :exception_message, :count
59
+ attr_reader :exception_message, :count, :spec_start_times
53
60
 
54
- def initialize(exception_message)
61
+ def initialize(exception_message, spec_start_time = nil)
55
62
  @exception_message = exception_message
56
63
  @count = 1
64
+ @spec_start_times = [spec_start_time].compact
57
65
  end
58
66
 
59
- def add_failure
67
+ def add_failure(spec_start_time = nil)
60
68
  @count += 1
69
+ @spec_start_times.push(spec_start_time) unless spec_start_time.nil?
70
+ end
71
+
72
+ def failure_summary
73
+ summary = "#{count.to_s.indent(2)} times with exception message:\n"
74
+ summary += exception_message.indent(4).red.to_s
75
+ return summary if spec_start_times.empty?
76
+
77
+ summary += "\n\nFailed at the following times:\n".indent(2)
78
+ summary += spec_start_times.map { |time| time.indent(4).yellow.to_s }.join("\n").to_s
79
+ summary
61
80
  end
62
81
  end
63
82
  end
@@ -12,14 +12,25 @@ module FlakeySpecCatcher
12
12
  # distinct example. It also provides helpers for adding new results,
13
13
  # displaying aggregate results, and checking the state of the collection.
14
14
  class RspecResultManager
15
+ attr_reader :results
16
+
15
17
  def initialize(rspec_result_class)
16
18
  @result_class = rspec_result_class
17
19
  @results = []
20
+ @spec_start_times = []
21
+ end
22
+
23
+ def track_spec_start_times(spec_times)
24
+ @spec_start_times = spec_times
18
25
  end
19
26
 
20
27
  def add_result(desc, location, message = nil)
21
28
  result = @results.find { |r| r.location == location }
22
- result ? result.add_run(message) : @results.push(@result_class.new(desc, location, message))
29
+ if result
30
+ result.add_run(message)
31
+ else
32
+ @results.push(@result_class.new(desc, location, @spec_start_times, message))
33
+ end
23
34
  end
24
35
 
25
36
  def print_results
@@ -7,10 +7,11 @@ require_relative './user_config'
7
7
  require_relative './rerun_manager'
8
8
  require_relative './rspec_result_manager'
9
9
  require_relative './event_listener.rb'
10
+ require_relative './timecop_manager.rb'
10
11
 
11
12
  module FlakeySpecCatcher
12
13
  class Runner
13
- attr_reader :user_config, :rerun_manager, :git_controller, :test_run_count
14
+ attr_reader :user_config, :rerun_manager, :git_controller, :test_run_count, :random_dates
14
15
 
15
16
  def initialize(test_mode: false,
16
17
  user_config: FlakeySpecCatcher::UserConfig.new,
@@ -18,17 +19,17 @@ module FlakeySpecCatcher
18
19
  result_manager: FlakeySpecCatcher::RspecResultManager.new(FlakeySpecCatcher::RspecResult),
19
20
  rerun_manager: FlakeySpecCatcher::RerunManager.new(git_controller: git_controller,
20
21
  user_config: user_config))
21
-
22
22
  @git_controller = git_controller
23
23
  @user_config = user_config
24
24
  @rerun_manager = rerun_manager
25
25
  @rspec_result_manager = result_manager
26
26
  @test_run_count = 0
27
27
  @temp_output_file = @user_config.output_file + 'temp' unless @user_config.output_file == File::NULL
28
+ enable_random_timing
28
29
  end
29
30
 
30
31
  # Debug Methods
31
- # rubocop:disable Metrics/AbcSize
32
+ # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity
32
33
  def show_settings
33
34
  puts 'Flakey Spec Catcher Settings:'
34
35
  puts " Current Branch: #{@git_controller.branch}" unless @user_config.use_parent
@@ -39,12 +40,13 @@ module FlakeySpecCatcher
39
40
  puts " Break on first failure: #{@user_config.break_on_first_failure}" if @user_config.break_on_first_failure
40
41
  puts " Node Total: #{@user_config.split_nodes}" if @user_config.split_nodes
41
42
  puts " Node Index: #{@user_config.split_index}" if @user_config.split_index
43
+ puts ' Random Timing: Enabled' if @user_config.random_timing
42
44
  puts " Changed Specs Detected: #{@git_controller.changed_examples}"
43
45
  return if @user_config.output_file == File::NULL
44
46
 
45
47
  puts " Verbose Output Path: #{@user_config.output_file}"
46
48
  end
47
- # rubocop:enable Metrics/AbcSize
49
+ # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity
48
50
 
49
51
  def rerun_preview
50
52
  puts "\n********************************************"
@@ -73,8 +75,8 @@ module FlakeySpecCatcher
73
75
 
74
76
  status = 0
75
77
  @rerun_manager.rerun_capsules.sort.each do |capsule|
76
- @user_config.repeat_factor.times do
77
- iteration_status = handle_capsule_rerun(capsule)
78
+ @user_config.repeat_factor.times do |iteration|
79
+ iteration_status = timecop_wrapper(capsule, iteration)
78
80
  status = [status, iteration_status].max
79
81
  break if @user_config.break_on_first_failure && !status.zero?
80
82
  end
@@ -127,6 +129,23 @@ module FlakeySpecCatcher
127
129
  end
128
130
  end
129
131
 
132
+ def enable_random_timing
133
+ @random_dates = if @user_config.random_timing
134
+ FlakeySpecCatcher::TimecopManager.generate_dates(@user_config.repeat_factor)
135
+ else
136
+ []
137
+ end
138
+ @rspec_result_manager.track_spec_start_times(@random_dates)
139
+ end
140
+
141
+ def timecop_wrapper(capsule, iteration)
142
+ if @user_config.random_timing
143
+ FlakeySpecCatcher::TimecopManager.randomly_travel_in_time(@random_dates[iteration]) { handle_capsule_rerun(capsule) }
144
+ else
145
+ handle_capsule_rerun(capsule)
146
+ end
147
+ end
148
+
130
149
  def print_flakey_specs_detected_message
131
150
  puts "\n**********************************************".magenta
132
151
  puts ' Flakiness Detected!'.magenta
@@ -0,0 +1,72 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'date'
4
+ require 'time'
5
+ require 'timecop'
6
+
7
+ module FlakeySpecCatcher
8
+ class TimecopManager
9
+ def self.randomly_travel_in_time(date)
10
+ status = 0
11
+ Timecop.travel(date) do
12
+ status = yield
13
+ end
14
+ status
15
+ end
16
+
17
+ def self.current_month
18
+ Time.now.month
19
+ end
20
+
21
+ def self.current_year
22
+ Time.now.year
23
+ end
24
+
25
+ def self.random_month
26
+ (current_month..12).to_a.sample
27
+ end
28
+
29
+ def self.random_seconds
30
+ [0, 59].sample
31
+ end
32
+
33
+ def self.random_minutes
34
+ [0, (1..58).to_a.sample, 59].sample
35
+ end
36
+
37
+ def self.random_day(year, month)
38
+ last_day = last_day_of_month(year, month).day
39
+ [1, (1..last_day).to_a.sample, last_day].sample
40
+ end
41
+
42
+ def self.last_day_of_month(year, month)
43
+ day = Date.new(year, month, -1).day
44
+ Time.local(year, month, day, 23, 59, 59)
45
+ end
46
+
47
+ def self.last_day_of_year(year)
48
+ Time.local(year, 12, 31, 11, 59, 59)
49
+ end
50
+
51
+ def self.random_hour
52
+ [0, (1..22).to_a.sample, 23].sample
53
+ end
54
+
55
+ def self.random_date
56
+ year = current_year
57
+ month = random_month
58
+ day = random_day(year, month)
59
+ Time.local(year, month, day, random_hour, random_minutes, random_seconds)
60
+ end
61
+
62
+ def self.prioritized_dates
63
+ [last_day_of_month(current_year, current_month), last_day_of_year(current_year)]
64
+ end
65
+
66
+ def self.generate_dates(count)
67
+ dates = prioritized_dates
68
+ (count - prioritized_dates.count).times { dates << random_date }
69
+ dates.map(&:iso8601).slice(0, count)
70
+ end
71
+ end
72
+ end
@@ -3,20 +3,18 @@
3
3
  require_relative './cli_override'
4
4
  module FlakeySpecCatcher
5
5
  # UserConfig class
6
- #
7
6
  # Captures user-defined settings to configure RSpec re-run settings.
8
7
 
9
8
  class UserConfig
10
- attr_reader :repeat_factor, :ignore_files, :ignore_branches, :silent_mode
11
- attr_reader :rerun_file_only, :rspec_usage_patterns, :excluded_tags
12
- attr_reader :manual_rerun_patterns, :manual_rerun_usage
13
- attr_reader :enable_runs, :output_file, :use_parent, :dry_run
14
- attr_reader :split_nodes, :split_index, :verbose, :test_options
15
- attr_reader :break_on_first_failure, :list_child_specs
9
+ attr_reader :repeat_factor, :ignore_files, :ignore_branches, :silent_mode, :rerun_file_only, :rspec_usage_patterns
10
+ attr_reader :excluded_tags, :manual_rerun_patterns, :manual_rerun_usage, :enable_runs, :output_file
11
+ attr_reader :use_parent, :dry_run, :split_nodes, :split_index, :verbose, :test_options, :break_on_first_failure
12
+ attr_reader :list_child_specs, :random_timing
16
13
 
17
14
  USER_CONFIG_ENV_VARS = %w[FSC_REPEAT_FACTOR FSC_IGNORE_FILES FSC_IGNORE_BRANCHES
18
15
  FSC_SILENT_MODE FSC_RERUN_FILE_ONLY FSC_USAGE_PATTERNS
19
- FSC_EXCLUDED_TAGS FSC_OUTPUT_FILE FSC_LIST_CHILD_SPECS].freeze
16
+ FSC_EXCLUDED_TAGS FSC_OUTPUT_FILE FSC_LIST_CHILD_SPECS
17
+ FSC_RANDOM_TIMING].freeze
20
18
 
21
19
  def initialize(cli_override: CliOverride.new)
22
20
  apply_env_var_settings
@@ -26,7 +24,7 @@ module FlakeySpecCatcher
26
24
 
27
25
  private
28
26
 
29
- # rubocop:disable Metrics/AbcSize
27
+ # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
30
28
  def apply_env_var_settings
31
29
  @repeat_factor = initialize_repeat_factor(ENV['FSC_REPEAT_FACTOR'])
32
30
  @ignore_files = env_var_string_to_array(ENV['FSC_IGNORE_FILES'])
@@ -34,6 +32,7 @@ module FlakeySpecCatcher
34
32
  @silent_mode = env_var_string_to_bool(ENV['FSC_SILENT_MODE'])
35
33
  @rerun_file_only = env_var_string_to_bool(ENV['FSC_RERUN_FILE_ONLY'])
36
34
  @list_child_specs = env_var_string_to_bool(ENV['FSC_LIST_CHILD_SPECS'])
35
+ @random_timing = env_var_string_to_bool(ENV['FSC_RANDOM_TIMING'])
37
36
  @rspec_usage_patterns = env_var_string_to_pairs(ENV['FSC_USAGE_PATTERNS'])
38
37
  @excluded_tags = env_var_string_to_tags(ENV['FSC_EXCLUDED_TAGS'])
39
38
  @output_file = ENV['FSC_OUTPUT_FILE']
@@ -48,6 +47,7 @@ module FlakeySpecCatcher
48
47
  @repeat_factor = @cli_override.repeat_factor if @cli_override.repeat_factor.to_i.positive?
49
48
  @break_on_first_failure = @cli_override.break_on_first_failure
50
49
  @list_child_specs = @cli_override.list_child_specs unless @cli_override.list_child_specs.nil?
50
+ @random_timing = @cli_override.random_timing unless @cli_override.random_timing.nil?
51
51
  @enable_runs = @cli_override.enable_runs
52
52
  @dry_run = @cli_override.dry_run
53
53
  @split_nodes = @cli_override.split_nodes unless @cli_override.split_nodes.nil?
@@ -61,7 +61,7 @@ module FlakeySpecCatcher
61
61
  @verbose = @cli_override.verbose
62
62
  @test_options = @cli_override.test_options
63
63
  end
64
- # rubocop:enable Metrics/AbcSize
64
+ # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
65
65
 
66
66
  def override_settings
67
67
  apply_cli_override
@@ -109,11 +109,7 @@ module FlakeySpecCatcher
109
109
  end
110
110
 
111
111
  def env_var_string_to_bool(env_var)
112
- if env_var.to_s.casecmp('true').zero?
113
- true
114
- else
115
- false
116
- end
112
+ env_var.to_s.casecmp('true').zero?
117
113
  end
118
114
 
119
115
  def env_var_string_to_pairs(env_var)
@@ -187,6 +183,8 @@ module FlakeySpecCatcher
187
183
  @rerun_file_only = env_var_string_to_bool(env_value)
188
184
  when 'FSC_LIST_CHILD_SPECS'
189
185
  @list_child_specs = env_var_string_to_bool(env_value)
186
+ when 'FSC_RANDOM_TIMING'
187
+ @random_timing = env_var_string_to_bool(env_value)
190
188
  when 'FSC_USAGE_PATTERNS'
191
189
  @rspec_usage_patterns = env_var_string_to_pairs(env_value)
192
190
  when 'FSC_EXCLUDED_TAGS'
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module FlakeySpecCatcher
4
- VERSION = '0.11.2'
4
+ VERSION = '0.12.0'
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: flakey_spec_catcher
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.11.2
4
+ version: 0.12.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Brian Watson
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2021-10-15 00:00:00.000000000 Z
13
+ date: 2021-11-08 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: rspec
@@ -26,6 +26,20 @@ dependencies:
26
26
  - - "~>"
27
27
  - !ruby/object:Gem::Version
28
28
  version: '3.10'
29
+ - !ruby/object:Gem::Dependency
30
+ name: timecop
31
+ requirement: !ruby/object:Gem::Requirement
32
+ requirements:
33
+ - - "~>"
34
+ - !ruby/object:Gem::Version
35
+ version: '0.9'
36
+ type: :runtime
37
+ prerelease: false
38
+ version_requirements: !ruby/object:Gem::Requirement
39
+ requirements:
40
+ - - "~>"
41
+ - !ruby/object:Gem::Version
42
+ version: '0.9'
29
43
  - !ruby/object:Gem::Dependency
30
44
  name: byebug
31
45
  requirement: !ruby/object:Gem::Requirement
@@ -126,6 +140,7 @@ files:
126
140
  - lib/flakey_spec_catcher/rspec_result.rb
127
141
  - lib/flakey_spec_catcher/rspec_result_manager.rb
128
142
  - lib/flakey_spec_catcher/runner.rb
143
+ - lib/flakey_spec_catcher/timecop_manager.rb
129
144
  - lib/flakey_spec_catcher/user_config.rb
130
145
  - lib/flakey_spec_catcher/version.rb
131
146
  - lib/helpers/colorize.rb