flakey_spec_catcher 0.11.2 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: adf31fd04e49611a2c10622b19350ab444224589815ac5f54948ec6aad53fc3a
-  data.tar.gz: 23526cc9694ac22402e6b47fbe956197af5be8b5648abba785e822884e1cd747
+  metadata.gz: 306e789630ec567368871033c6334208c08a77e69bf901d76d6a2f0f1135470d
+  data.tar.gz: c80514cf76ad55fceda815f217e070f67659feff6bdd6ab923356cd42c722267
 SHA512:
-  metadata.gz: 9e15abca29cc1a0186ceadcb83f6238c3a847b4f59747e8c8bc0ef23387ed2a8b5cf3777675d32690bfb529b8e7aba6e6e8ba9970d3d9628628392cfa346506b
-  data.tar.gz: ce915b99014c9eee3210e55663dec289b20dcb39936a28ffd299fc7c0bb003561ef3d67e4a620bf28057e8ae4fe60aca1ed17f2e66436aa6d0f0fdae928e2629
+  metadata.gz: 87c7f748c9577946ab735549729b5695978853225e0b10b01c205ffcbaab28db4140dada4e498eed2d169bcc26e2b940ac169002eda5bc92718159c6201996d2
+  data.tar.gz: 1f10d9cd60b5dd635c8e8759a54df20d54e467882c3c5e95ef2dad6ba3530ee99719371162032aa6775d11e3827db02cb2e1139cbf376763faa18006d98a8bd0

data/flakey_spec_catcher.gemspec

@@ -31,6 +31,7 @@ Gem::Specification.new do |gem|
   gem.required_ruby_version         = '>= 2.6'
 
   gem.add_dependency 'rspec', '~> 3.10'
+  gem.add_dependency 'timecop', '~> 0.9'
   gem.add_development_dependency 'byebug', '~> 11.1'
   gem.add_development_dependency 'climate_control', '~> 0.2'
   gem.add_development_dependency 'rake', '~> 13.0'

data/lib/flakey_spec_catcher/cli_override.rb

@@ -9,7 +9,7 @@ module FlakeySpecCatcher
   class CliOverride
     attr_reader :rerun_patterns, :rerun_usage, :repeat_factor, :enable_runs, :excluded_tags, :use_parent, :dry_run
     attr_reader :output_file, :split_nodes, :split_index, :verbose, :test_options, :break_on_first_failure
-    attr_reader :list_child_specs
+    attr_reader :list_child_specs, :random_timing
 
     def initialize
       @dry_run = false
@@ -65,6 +65,10 @@ module FlakeySpecCatcher
           @list_child_specs = list_child_specs
         end
 
+        opts.on('--random-timing', 'Run Specs at random times') do |random_timing|
+          @random_timing = random_timing
+        end
+
         opts.on('-e', '--excluded-tags=EXCLUDED_TAGS',
                 'Specify tags to exclude in a comma separated list') do |tags|
           @excluded_tags = parse_tags(tags)

data/lib/flakey_spec_catcher/rspec_result.rb

@@ -13,13 +13,15 @@ module FlakeySpecCatcher
   # This class will then organize and output the results accordingly.
   class RspecResult
     attr_accessor :description, :location, :total_times_run, :total_failures
+    attr_reader :spec_start_times, :failures
 
-    def initialize(description, location, exception_message = nil)
+    def initialize(description, location, spec_start_times, exception_message = nil)
       @description = description
       @location = location
       @total_times_run = 1
       @total_failures = exception_message ? 1 : 0
       @failures = []
+      @spec_start_times = spec_start_times
       add_failure(exception_message) if exception_message
     end
 
@@ -34,30 +36,47 @@ module FlakeySpecCatcher
 
     def add_failure(exception_message)
       failure = @failures.find { |f| f.exception_message == exception_message }
-      failure ? failure.add_failure : @failures.push(RSpecFailure.new(exception_message))
+      if failure
+        failure.add_failure(current_spec_start_time)
+      else
+        @failures.push(RSpecFailure.new(exception_message, current_spec_start_time))
+      end
+    end
+
+    def current_spec_start_time
+      @spec_start_times[@total_times_run - 1]
     end
 
     def print_results
       puts "\n#{@description.yellow} (#{location})
             \nFAILED #{total_failures} / #{total_times_run} times"
 
-      @failures.each do |f|
-        puts "#{f.count.to_s.indent(2)} times with exception message:"
-        puts f.exception_message.indent(4).red.to_s
-      end
+      @failures.each { |failure| puts failure.failure_summary }
     end
 
     # Simple class to contain failed example data
     class RSpecFailure
-      attr_reader :exception_message, :count
+      attr_reader :exception_message, :count, :spec_start_times
 
-      def initialize(exception_message)
+      def initialize(exception_message, spec_start_time = nil)
         @exception_message = exception_message
         @count = 1
+        @spec_start_times = [spec_start_time].compact
       end
 
-      def add_failure
+      def add_failure(spec_start_time = nil)
         @count += 1
+        @spec_start_times.push(spec_start_time) unless spec_start_time.nil?
+      end
+
+      def failure_summary
+        summary = "#{count.to_s.indent(2)} times with exception message:\n"
+        summary += exception_message.indent(4).red.to_s
+        return summary if spec_start_times.empty?
+
+        summary += "\n\nFailed at the following times:\n".indent(2)
+        summary += spec_start_times.map { |time| time.indent(4).yellow.to_s }.join("\n").to_s
+        summary
       end
     end
   end

data/lib/flakey_spec_catcher/rspec_result_manager.rb

@@ -12,14 +12,25 @@ module FlakeySpecCatcher
   # distinct example. It also provides helpers for adding new results,
   # displaying aggregate results, and checking the state of the collection.
   class RspecResultManager
+    attr_reader :results
+
     def initialize(rspec_result_class)
       @result_class = rspec_result_class
       @results = []
+      @spec_start_times = []
+    end
+
+    def track_spec_start_times(spec_times)
+      @spec_start_times = spec_times
     end
 
     def add_result(desc, location, message = nil)
       result = @results.find { |r| r.location == location }
-      result ? result.add_run(message) : @results.push(@result_class.new(desc, location, message))
+      if result
+        result.add_run(message)
+      else
+        @results.push(@result_class.new(desc, location, @spec_start_times, message))
+      end
     end
 
     def print_results

data/lib/flakey_spec_catcher/runner.rb

@@ -7,10 +7,11 @@ require_relative './user_config'
 require_relative './rerun_manager'
 require_relative './rspec_result_manager'
 require_relative './event_listener.rb'
+require_relative './timecop_manager.rb'
 
 module FlakeySpecCatcher
   class Runner
-    attr_reader :user_config, :rerun_manager, :git_controller, :test_run_count
+    attr_reader :user_config, :rerun_manager, :git_controller, :test_run_count, :random_dates
 
     def initialize(test_mode: false,
       user_config: FlakeySpecCatcher::UserConfig.new,
@@ -18,17 +19,17 @@ module FlakeySpecCatcher
       result_manager: FlakeySpecCatcher::RspecResultManager.new(FlakeySpecCatcher::RspecResult),
       rerun_manager: FlakeySpecCatcher::RerunManager.new(git_controller: git_controller,
                                                          user_config: user_config))
-
       @git_controller = git_controller
       @user_config = user_config
       @rerun_manager = rerun_manager
       @rspec_result_manager = result_manager
       @test_run_count = 0
       @temp_output_file = @user_config.output_file + 'temp' unless @user_config.output_file == File::NULL
+      enable_random_timing
     end
 
     # Debug Methods
-    # rubocop:disable Metrics/AbcSize
+    # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity
     def show_settings
       puts 'Flakey Spec Catcher Settings:'
       puts "  Current Branch: #{@git_controller.branch}" unless @user_config.use_parent
@@ -39,12 +40,13 @@ module FlakeySpecCatcher
       puts "  Break on first failure: #{@user_config.break_on_first_failure}" if @user_config.break_on_first_failure
       puts "  Node Total: #{@user_config.split_nodes}" if @user_config.split_nodes
       puts "  Node Index: #{@user_config.split_index}" if @user_config.split_index
+      puts '  Random Timing: Enabled' if @user_config.random_timing
       puts "  Changed Specs Detected: #{@git_controller.changed_examples}"
       return if @user_config.output_file == File::NULL
 
       puts "  Verbose Output Path: #{@user_config.output_file}"
     end
-    # rubocop:enable Metrics/AbcSize
+    # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity
 
     def rerun_preview
       puts "\n********************************************"
@@ -73,8 +75,8 @@ module FlakeySpecCatcher
 
       status = 0
       @rerun_manager.rerun_capsules.sort.each do |capsule|
-        @user_config.repeat_factor.times do
-          iteration_status = handle_capsule_rerun(capsule)
+        @user_config.repeat_factor.times do |iteration|
+          iteration_status = timecop_wrapper(capsule, iteration)
           status = [status, iteration_status].max
           break if @user_config.break_on_first_failure && !status.zero?
         end
@@ -127,6 +129,23 @@ module FlakeySpecCatcher
       end
     end
 
+    def enable_random_timing
+      @random_dates = if @user_config.random_timing
+        FlakeySpecCatcher::TimecopManager.generate_dates(@user_config.repeat_factor)
+      else
+        []
+      end
+      @rspec_result_manager.track_spec_start_times(@random_dates)
+    end
+
+    def timecop_wrapper(capsule, iteration)
+      if @user_config.random_timing
+        FlakeySpecCatcher::TimecopManager.randomly_travel_in_time(@random_dates[iteration]) { handle_capsule_rerun(capsule) }
+      else
+        handle_capsule_rerun(capsule)
+      end
+    end
+
     def print_flakey_specs_detected_message
       puts "\n**********************************************".magenta
       puts '          Flakiness Detected!'.magenta

data/lib/flakey_spec_catcher/timecop_manager.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require 'date'
+require 'time'
+require 'timecop'
+
+module FlakeySpecCatcher
+  class TimecopManager
+    def self.randomly_travel_in_time(date)
+      status = 0
+      Timecop.travel(date) do
+        status = yield
+      end
+      status
+    end
+
+    def self.current_month
+      Time.now.month
+    end
+
+    def self.current_year
+      Time.now.year
+    end
+
+    def self.random_month
+      (current_month..12).to_a.sample
+    end
+
+    def self.random_seconds
+      [0, 59].sample
+    end
+
+    def self.random_minutes
+      [0, (1..58).to_a.sample, 59].sample
+    end
+
+    def self.random_day(year, month)
+      last_day = last_day_of_month(year, month).day
+      [1, (1..last_day).to_a.sample, last_day].sample
+    end
+
+    def self.last_day_of_month(year, month)
+      day = Date.new(year, month, -1).day
+      Time.local(year, month, day, 23, 59, 59)
+    end
+
+    def self.last_day_of_year(year)
+      Time.local(year, 12, 31, 11, 59, 59)
+    end
+
+    def self.random_hour
+      [0, (1..22).to_a.sample, 23].sample
+    end
+
+    def self.random_date
+      year = current_year
+      month = random_month
+      day = random_day(year, month)
+      Time.local(year, month, day, random_hour, random_minutes, random_seconds)
+    end
+
+    def self.prioritized_dates
+      [last_day_of_month(current_year, current_month), last_day_of_year(current_year)]
+    end
+
+    def self.generate_dates(count)
+      dates = prioritized_dates
+      (count - prioritized_dates.count).times { dates << random_date }
+      dates.map(&:iso8601).slice(0, count)
+    end
+  end
+end

data/lib/flakey_spec_catcher/user_config.rb

@@ -3,20 +3,18 @@
 require_relative './cli_override'
 module FlakeySpecCatcher
   # UserConfig class
-  #
   # Captures user-defined settings to configure RSpec re-run settings.
 
   class UserConfig
-    attr_reader :repeat_factor, :ignore_files, :ignore_branches, :silent_mode
-    attr_reader :rerun_file_only, :rspec_usage_patterns, :excluded_tags
-    attr_reader :manual_rerun_patterns, :manual_rerun_usage
-    attr_reader :enable_runs, :output_file, :use_parent, :dry_run
-    attr_reader :split_nodes, :split_index, :verbose, :test_options
-    attr_reader :break_on_first_failure, :list_child_specs
+    attr_reader :repeat_factor, :ignore_files, :ignore_branches, :silent_mode, :rerun_file_only, :rspec_usage_patterns
+    attr_reader :excluded_tags, :manual_rerun_patterns, :manual_rerun_usage, :enable_runs, :output_file
+    attr_reader :use_parent, :dry_run, :split_nodes, :split_index, :verbose, :test_options, :break_on_first_failure
+    attr_reader :list_child_specs, :random_timing
 
     USER_CONFIG_ENV_VARS = %w[FSC_REPEAT_FACTOR FSC_IGNORE_FILES FSC_IGNORE_BRANCHES
                               FSC_SILENT_MODE FSC_RERUN_FILE_ONLY FSC_USAGE_PATTERNS
-                              FSC_EXCLUDED_TAGS FSC_OUTPUT_FILE FSC_LIST_CHILD_SPECS].freeze
+                              FSC_EXCLUDED_TAGS FSC_OUTPUT_FILE FSC_LIST_CHILD_SPECS
+                              FSC_RANDOM_TIMING].freeze
 
     def initialize(cli_override: CliOverride.new)
       apply_env_var_settings
@@ -26,7 +24,7 @@ module FlakeySpecCatcher
 
     private
 
-    # rubocop:disable Metrics/AbcSize
+    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
     def apply_env_var_settings
       @repeat_factor = initialize_repeat_factor(ENV['FSC_REPEAT_FACTOR'])
       @ignore_files = env_var_string_to_array(ENV['FSC_IGNORE_FILES'])
@@ -34,6 +32,7 @@ module FlakeySpecCatcher
       @silent_mode = env_var_string_to_bool(ENV['FSC_SILENT_MODE'])
       @rerun_file_only = env_var_string_to_bool(ENV['FSC_RERUN_FILE_ONLY'])
       @list_child_specs = env_var_string_to_bool(ENV['FSC_LIST_CHILD_SPECS'])
+      @random_timing = env_var_string_to_bool(ENV['FSC_RANDOM_TIMING'])
       @rspec_usage_patterns = env_var_string_to_pairs(ENV['FSC_USAGE_PATTERNS'])
       @excluded_tags = env_var_string_to_tags(ENV['FSC_EXCLUDED_TAGS'])
       @output_file = ENV['FSC_OUTPUT_FILE']
@@ -48,6 +47,7 @@ module FlakeySpecCatcher
       @repeat_factor = @cli_override.repeat_factor if @cli_override.repeat_factor.to_i.positive?
       @break_on_first_failure = @cli_override.break_on_first_failure
       @list_child_specs = @cli_override.list_child_specs unless @cli_override.list_child_specs.nil?
+      @random_timing = @cli_override.random_timing unless @cli_override.random_timing.nil?
       @enable_runs = @cli_override.enable_runs
       @dry_run = @cli_override.dry_run
       @split_nodes = @cli_override.split_nodes unless @cli_override.split_nodes.nil?
@@ -61,7 +61,7 @@ module FlakeySpecCatcher
       @verbose = @cli_override.verbose
       @test_options = @cli_override.test_options
     end
-    # rubocop:enable Metrics/AbcSize
+    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
     def override_settings
       apply_cli_override
@@ -109,11 +109,7 @@ module FlakeySpecCatcher
     end
 
     def env_var_string_to_bool(env_var)
-      if env_var.to_s.casecmp('true').zero?
-        true
-      else
-        false
-      end
+      env_var.to_s.casecmp('true').zero?
     end
 
     def env_var_string_to_pairs(env_var)
@@ -187,6 +183,8 @@ module FlakeySpecCatcher
         @rerun_file_only = env_var_string_to_bool(env_value)
       when 'FSC_LIST_CHILD_SPECS'
         @list_child_specs = env_var_string_to_bool(env_value)
+      when 'FSC_RANDOM_TIMING'
+        @random_timing = env_var_string_to_bool(env_value)
       when 'FSC_USAGE_PATTERNS'
         @rspec_usage_patterns = env_var_string_to_pairs(env_value)
       when 'FSC_EXCLUDED_TAGS'

data/lib/flakey_spec_catcher/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FlakeySpecCatcher
-  VERSION = '0.11.2'
+  VERSION = '0.12.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flakey_spec_catcher
 version: !ruby/object:Gem::Version
-  version: 0.11.2
+  version: 0.12.0
 platform: ruby
 authors:
 - Brian Watson
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-10-15 00:00:00.000000000 Z
+date: 2021-11-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -26,6 +26,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.10'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -126,6 +140,7 @@ files:
 - lib/flakey_spec_catcher/rspec_result.rb
 - lib/flakey_spec_catcher/rspec_result_manager.rb
 - lib/flakey_spec_catcher/runner.rb
+- lib/flakey_spec_catcher/timecop_manager.rb
 - lib/flakey_spec_catcher/user_config.rb
 - lib/flakey_spec_catcher/version.rb
 - lib/helpers/colorize.rb