flakey_spec_catcher 0.10.0 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9894969be42fdb7dfafa961942ce16b1313ca3cffdc746d9613ebe1398d39069
-  data.tar.gz: 75238a79f25ddb8cec6fd60064e9f06d5f5651b8bb3c27e423f317f653f65774
+  metadata.gz: 306e789630ec567368871033c6334208c08a77e69bf901d76d6a2f0f1135470d
+  data.tar.gz: c80514cf76ad55fceda815f217e070f67659feff6bdd6ab923356cd42c722267
 SHA512:
-  metadata.gz: d3698a10fd2c5d4ae75f3fe0250d7578a82906eb6ad478c759756911501b8f53125a63f1ced325e51c6115a9f76c7ac2264ae28a42c8b65273376186e8e7aa34
-  data.tar.gz: 440e5ef180775556f3292548330cf9572db13d82394921fbbca6053b37ed4c088aa5c37b9dcb06ab2fb4049879c7fe2a093a60cf05da649abb950bf01d0af6cf
+  metadata.gz: 87c7f748c9577946ab735549729b5695978853225e0b10b01c205ffcbaab28db4140dada4e498eed2d169bcc26e2b940ac169002eda5bc92718159c6201996d2
+  data.tar.gz: 1f10d9cd60b5dd635c8e8759a54df20d54e467882c3c5e95ef2dad6ba3530ee99719371162032aa6775d11e3827db02cb2e1139cbf376763faa18006d98a8bd0

data/flakey_spec_catcher.gemspec

@@ -31,6 +31,7 @@ Gem::Specification.new do |gem|
   gem.required_ruby_version         = '>= 2.6'
 
   gem.add_dependency 'rspec', '~> 3.10'
+  gem.add_dependency 'timecop', '~> 0.9'
   gem.add_development_dependency 'byebug', '~> 11.1'
   gem.add_development_dependency 'climate_control', '~> 0.2'
   gem.add_development_dependency 'rake', '~> 13.0'

data/lib/flakey_spec_catcher/change_capsule.rb

@@ -8,7 +8,7 @@ module FlakeySpecCatcher
   # A ChangeCapsule object will represent the changes made to a block of code. It
   # accomplishes this using ChangeContext and ChangeSummary objects.
   class ChangeCapsule
-    attr_reader :file_name, :change_summary, :change_contexts
+    attr_reader :file_name, :change_summary, :change_contexts, :spec_tree
     SCOPE_SPECIFIERS = %w[it context describe scenario].freeze
     SHARED_EXAMPLES = %w[include_examples it_behaves_like it_should_behave_like matching].freeze
 
@@ -16,6 +16,7 @@ module FlakeySpecCatcher
       @file_name = file_name
       @change_summary = change_summary
       @change_contexts = []
+      @spec_tree = {}
       handle_initial_change_contexts(change_contexts)
     end
 
@@ -23,23 +24,30 @@ module FlakeySpecCatcher
       @change_contexts.map(&:rerun_info)
     end
 
+    # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
     def fill_contexts
       change_context_stack = []
       ignore_scope_closure = 0
       lines_in_file = File.read(@file_name).split("\n")
       lines_in_file.each_with_index do |line, index|
+        shared_example_identified = false
+
         # Check if line matches an rspec example or examplegroup format
         if line =~ spec_scope
           handle_change_context(line, index, change_context_stack)
         # Else, ignore other blocks that might pollute context stack
+        elsif line =~ shared_example_scope
+          handle_change_context(line, index, change_context_stack)
+          shared_example_identified = true
         elsif line_matches_method_or_block(line)
           ignore_scope_closure += 1
         end
 
         fill_context(line, index, change_context_stack)
 
-        # Note - some things use do-like loops and we need to be able to ignore those
-        if line =~ pop_scope
+        if shared_example_identified
+          change_context_stack.pop
+        elsif line =~ pop_scope
           if ignore_scope_closure.positive?
             ignore_scope_closure -= 1
           else
@@ -48,6 +56,7 @@ module FlakeySpecCatcher
         end
       end
     end
+    # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
 
     private
 
@@ -69,14 +78,15 @@ module FlakeySpecCatcher
     end
 
     def spec_scope
-      # Not sure if we need to check for description in quotes
-      # spec_scope = /^\s*(#{SCOPE_SPECIFIERS.join("|")})\s*('.*'|".*").*do\s*$/
-
       /\s*(#{SCOPE_SPECIFIERS.join("|")}).*\s+do.*$/
     end
 
+    def shared_example_scope
+      /\s*(#{SHARED_EXAMPLES.join("|")}).*\s+.*$/
+    end
+
     def line_matches_method_or_block(line)
-      return true if line =~ /\s*do(\s+|$)/ || line =~ /^\s*def\s+/ || line =~ /^\s*if\s+/
+      return true if line =~ /\s*do(\s+|$)/ || line =~ /^\s*def\s+/ || line =~ /^\s*if\s+/ || line =~ /^\s*class\s+/
 
       false
     end
@@ -112,6 +122,24 @@ module FlakeySpecCatcher
 
       change_context_stack.push(change_context) unless
         change_context_stack.any? { |c| c == change_context }
+
+      build_spec_tree(change_context_stack)
+    end
+
+    def build_spec_tree(change_contexts)
+      return unless change_contexts.count > 1
+
+      current_scope = change_contexts[-1].rerun_info
+      parent_scope = change_contexts[-2].rerun_info
+
+      # Don't build the file level context into spec tree
+      return unless parent_scope =~ /:/
+
+      @spec_tree[parent_scope] = if @spec_tree[parent_scope]
+        @spec_tree[parent_scope] << current_scope
+      else
+        [current_scope]
+      end
     end
   end
 end

data/lib/flakey_spec_catcher/cli_override.rb

@@ -9,6 +9,7 @@ module FlakeySpecCatcher
   class CliOverride
     attr_reader :rerun_patterns, :rerun_usage, :repeat_factor, :enable_runs, :excluded_tags, :use_parent, :dry_run
     attr_reader :output_file, :split_nodes, :split_index, :verbose, :test_options, :break_on_first_failure
+    attr_reader :list_child_specs, :random_timing
 
     def initialize
       @dry_run = false
@@ -60,6 +61,14 @@ module FlakeySpecCatcher
           @break_on_first_failure = break_on_first_failure
         end
 
+        opts.on('--list-child-specs', 'List Child Specs (Verbose Spec Listing') do |list_child_specs|
+          @list_child_specs = list_child_specs
+        end
+
+        opts.on('--random-timing', 'Run Specs at random times') do |random_timing|
+          @random_timing = random_timing
+        end
+
         opts.on('-e', '--excluded-tags=EXCLUDED_TAGS',
                 'Specify tags to exclude in a comma separated list') do |tags|
           @excluded_tags = parse_tags(tags)

data/lib/flakey_spec_catcher/rerun_manager.rb

@@ -39,6 +39,7 @@ module FlakeySpecCatcher
       else
         all_non_example_groups - identify_tag_excluded_reruns
       end
+      tests = transform_parent_specs(tests).flatten if @user_config.list_child_specs
       filter_reruns_by_ignore_files(tests)
     end
 
@@ -171,5 +172,34 @@ module FlakeySpecCatcher
         @rerun_capsules.push(FlakeySpecCatcher::RerunCapsule.new(testcase: testcase, usage: usage))
       end
     end
+
+    def transform_file_level_change(test, tree)
+      # Map file level changes to the highest level Example Group in the tree
+      if test =~ /:/
+        test
+      else
+        tree.keys.first
+      end
+    end
+
+    def transform_parent_specs(tests)
+      transformed_tests = []
+
+      tests.each do |test|
+        capsule = @git_controller.capsule_manager.change_capsules.find { |c| c.file_name == test.split(':')[0] }
+        spec_tree = capsule.spec_tree
+        test = transform_file_level_change(test, spec_tree)
+        # If test is a key in the spec_tree, it's an example group, so we queue up its descendant examples
+        if spec_tree[test]
+          spec_tree[test].each do |child|
+            transformed_tests << transform_parent_specs([child]).flatten
+          end
+        # Else test is just an example
+        else
+          transformed_tests << test
+        end
+      end
+      transformed_tests
+    end
   end
 end

data/lib/flakey_spec_catcher/rspec_result.rb

@@ -13,13 +13,15 @@ module FlakeySpecCatcher
   # This class will then organize and output the results accordingly.
   class RspecResult
     attr_accessor :description, :location, :total_times_run, :total_failures
+    attr_reader :spec_start_times, :failures
 
-    def initialize(description, location, exception_message = nil)
+    def initialize(description, location, spec_start_times, exception_message = nil)
       @description = description
       @location = location
       @total_times_run = 1
       @total_failures = exception_message ? 1 : 0
       @failures = []
+      @spec_start_times = spec_start_times
       add_failure(exception_message) if exception_message
     end
 
@@ -34,30 +36,47 @@ module FlakeySpecCatcher
 
     def add_failure(exception_message)
       failure = @failures.find { |f| f.exception_message == exception_message }
-      failure ? failure.add_failure : @failures.push(RSpecFailure.new(exception_message))
+      if failure
+        failure.add_failure(current_spec_start_time)
+      else
+        @failures.push(RSpecFailure.new(exception_message, current_spec_start_time))
+      end
+    end
+
+    def current_spec_start_time
+      @spec_start_times[@total_times_run - 1]
     end
 
     def print_results
       puts "\n#{@description.yellow} (#{location})
             \nFAILED #{total_failures} / #{total_times_run} times"
 
-      @failures.each do |f|
-        puts "#{f.count.to_s.indent(2)} times with exception message:"
-        puts f.exception_message.indent(4).red.to_s
-      end
+      @failures.each { |failure| puts failure.failure_summary }
     end
 
     # Simple class to contain failed example data
     class RSpecFailure
-      attr_reader :exception_message, :count
+      attr_reader :exception_message, :count, :spec_start_times
 
-      def initialize(exception_message)
+      def initialize(exception_message, spec_start_time = nil)
         @exception_message = exception_message
         @count = 1
+        @spec_start_times = [spec_start_time].compact
       end
 
-      def add_failure
+      def add_failure(spec_start_time = nil)
         @count += 1
+        @spec_start_times.push(spec_start_time) unless spec_start_time.nil?
+      end
+
+      def failure_summary
+        summary = "#{count.to_s.indent(2)} times with exception message:\n"
+        summary += exception_message.indent(4).red.to_s
+        return summary if spec_start_times.empty?
+
+        summary += "\n\nFailed at the following times:\n".indent(2)
+        summary += spec_start_times.map { |time| time.indent(4).yellow.to_s }.join("\n").to_s
+        summary
       end
     end
   end

data/lib/flakey_spec_catcher/rspec_result_manager.rb

@@ -12,14 +12,25 @@ module FlakeySpecCatcher
   # distinct example. It also provides helpers for adding new results,
   # displaying aggregate results, and checking the state of the collection.
   class RspecResultManager
+    attr_reader :results
+
     def initialize(rspec_result_class)
       @result_class = rspec_result_class
       @results = []
+      @spec_start_times = []
+    end
+
+    def track_spec_start_times(spec_times)
+      @spec_start_times = spec_times
     end
 
     def add_result(desc, location, message = nil)
       result = @results.find { |r| r.location == location }
-      result ? result.add_run(message) : @results.push(@result_class.new(desc, location, message))
+      if result
+        result.add_run(message)
+      else
+        @results.push(@result_class.new(desc, location, @spec_start_times, message))
+      end
     end
 
     def print_results

data/lib/flakey_spec_catcher/runner.rb

@@ -7,10 +7,11 @@ require_relative './user_config'
 require_relative './rerun_manager'
 require_relative './rspec_result_manager'
 require_relative './event_listener.rb'
+require_relative './timecop_manager.rb'
 
 module FlakeySpecCatcher
   class Runner
-    attr_reader :user_config, :rerun_manager, :git_controller, :test_run_count
+    attr_reader :user_config, :rerun_manager, :git_controller, :test_run_count, :random_dates
 
     def initialize(test_mode: false,
       user_config: FlakeySpecCatcher::UserConfig.new,
@@ -18,17 +19,17 @@ module FlakeySpecCatcher
       result_manager: FlakeySpecCatcher::RspecResultManager.new(FlakeySpecCatcher::RspecResult),
       rerun_manager: FlakeySpecCatcher::RerunManager.new(git_controller: git_controller,
                                                          user_config: user_config))
-
       @git_controller = git_controller
       @user_config = user_config
       @rerun_manager = rerun_manager
       @rspec_result_manager = result_manager
       @test_run_count = 0
       @temp_output_file = @user_config.output_file + 'temp' unless @user_config.output_file == File::NULL
+      enable_random_timing
     end
 
     # Debug Methods
-    # rubocop:disable Metrics/AbcSize
+    # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity
     def show_settings
       puts 'Flakey Spec Catcher Settings:'
       puts "  Current Branch: #{@git_controller.branch}" unless @user_config.use_parent
@@ -39,12 +40,13 @@ module FlakeySpecCatcher
       puts "  Break on first failure: #{@user_config.break_on_first_failure}" if @user_config.break_on_first_failure
       puts "  Node Total: #{@user_config.split_nodes}" if @user_config.split_nodes
       puts "  Node Index: #{@user_config.split_index}" if @user_config.split_index
+      puts '  Random Timing: Enabled' if @user_config.random_timing
       puts "  Changed Specs Detected: #{@git_controller.changed_examples}"
       return if @user_config.output_file == File::NULL
 
       puts "  Verbose Output Path: #{@user_config.output_file}"
     end
-    # rubocop:enable Metrics/AbcSize
+    # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity
 
     def rerun_preview
       puts "\n********************************************"
@@ -73,8 +75,8 @@ module FlakeySpecCatcher
 
       status = 0
       @rerun_manager.rerun_capsules.sort.each do |capsule|
-        @user_config.repeat_factor.times do
-          iteration_status = handle_capsule_rerun(capsule)
+        @user_config.repeat_factor.times do |iteration|
+          iteration_status = timecop_wrapper(capsule, iteration)
           status = [status, iteration_status].max
           break if @user_config.break_on_first_failure && !status.zero?
         end
@@ -127,6 +129,23 @@ module FlakeySpecCatcher
       end
     end
 
+    def enable_random_timing
+      @random_dates = if @user_config.random_timing
+        FlakeySpecCatcher::TimecopManager.generate_dates(@user_config.repeat_factor)
+      else
+        []
+      end
+      @rspec_result_manager.track_spec_start_times(@random_dates)
+    end
+
+    def timecop_wrapper(capsule, iteration)
+      if @user_config.random_timing
+        FlakeySpecCatcher::TimecopManager.randomly_travel_in_time(@random_dates[iteration]) { handle_capsule_rerun(capsule) }
+      else
+        handle_capsule_rerun(capsule)
+      end
+    end
+
     def print_flakey_specs_detected_message
       puts "\n**********************************************".magenta
       puts '          Flakiness Detected!'.magenta

data/lib/flakey_spec_catcher/timecop_manager.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require 'date'
+require 'time'
+require 'timecop'
+
+module FlakeySpecCatcher
+  class TimecopManager
+    def self.randomly_travel_in_time(date)
+      status = 0
+      Timecop.travel(date) do
+        status = yield
+      end
+      status
+    end
+
+    def self.current_month
+      Time.now.month
+    end
+
+    def self.current_year
+      Time.now.year
+    end
+
+    def self.random_month
+      (current_month..12).to_a.sample
+    end
+
+    def self.random_seconds
+      [0, 59].sample
+    end
+
+    def self.random_minutes
+      [0, (1..58).to_a.sample, 59].sample
+    end
+
+    def self.random_day(year, month)
+      last_day = last_day_of_month(year, month).day
+      [1, (1..last_day).to_a.sample, last_day].sample
+    end
+
+    def self.last_day_of_month(year, month)
+      day = Date.new(year, month, -1).day
+      Time.local(year, month, day, 23, 59, 59)
+    end
+
+    def self.last_day_of_year(year)
+      Time.local(year, 12, 31, 11, 59, 59)
+    end
+
+    def self.random_hour
+      [0, (1..22).to_a.sample, 23].sample
+    end
+
+    def self.random_date
+      year = current_year
+      month = random_month
+      day = random_day(year, month)
+      Time.local(year, month, day, random_hour, random_minutes, random_seconds)
+    end
+
+    def self.prioritized_dates
+      [last_day_of_month(current_year, current_month), last_day_of_year(current_year)]
+    end
+
+    def self.generate_dates(count)
+      dates = prioritized_dates
+      (count - prioritized_dates.count).times { dates << random_date }
+      dates.map(&:iso8601).slice(0, count)
+    end
+  end
+end

data/lib/flakey_spec_catcher/user_config.rb

@@ -3,20 +3,18 @@
 require_relative './cli_override'
 module FlakeySpecCatcher
   # UserConfig class
-  #
   # Captures user-defined settings to configure RSpec re-run settings.
 
   class UserConfig
-    attr_reader :repeat_factor, :ignore_files, :ignore_branches, :silent_mode
-    attr_reader :rerun_file_only, :rspec_usage_patterns, :excluded_tags
-    attr_reader :manual_rerun_patterns, :manual_rerun_usage
-    attr_reader :enable_runs, :output_file, :use_parent, :dry_run
-    attr_reader :split_nodes, :split_index, :verbose, :test_options
-    attr_reader :break_on_first_failure
+    attr_reader :repeat_factor, :ignore_files, :ignore_branches, :silent_mode, :rerun_file_only, :rspec_usage_patterns
+    attr_reader :excluded_tags, :manual_rerun_patterns, :manual_rerun_usage, :enable_runs, :output_file
+    attr_reader :use_parent, :dry_run, :split_nodes, :split_index, :verbose, :test_options, :break_on_first_failure
+    attr_reader :list_child_specs, :random_timing
 
     USER_CONFIG_ENV_VARS = %w[FSC_REPEAT_FACTOR FSC_IGNORE_FILES FSC_IGNORE_BRANCHES
                               FSC_SILENT_MODE FSC_RERUN_FILE_ONLY FSC_USAGE_PATTERNS
-                              FSC_EXCLUDED_TAGS FSC_OUTPUT_FILE].freeze
+                              FSC_EXCLUDED_TAGS FSC_OUTPUT_FILE FSC_LIST_CHILD_SPECS
+                              FSC_RANDOM_TIMING].freeze
 
     def initialize(cli_override: CliOverride.new)
       apply_env_var_settings
@@ -26,13 +24,15 @@ module FlakeySpecCatcher
 
     private
 
-    # rubocop:disable Metrics/AbcSize
+    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
     def apply_env_var_settings
       @repeat_factor = initialize_repeat_factor(ENV['FSC_REPEAT_FACTOR'])
       @ignore_files = env_var_string_to_array(ENV['FSC_IGNORE_FILES'])
       @ignore_branches = env_var_string_to_array(ENV['FSC_IGNORE_BRANCHES'])
       @silent_mode = env_var_string_to_bool(ENV['FSC_SILENT_MODE'])
       @rerun_file_only = env_var_string_to_bool(ENV['FSC_RERUN_FILE_ONLY'])
+      @list_child_specs = env_var_string_to_bool(ENV['FSC_LIST_CHILD_SPECS'])
+      @random_timing = env_var_string_to_bool(ENV['FSC_RANDOM_TIMING'])
       @rspec_usage_patterns = env_var_string_to_pairs(ENV['FSC_USAGE_PATTERNS'])
       @excluded_tags = env_var_string_to_tags(ENV['FSC_EXCLUDED_TAGS'])
       @output_file = ENV['FSC_OUTPUT_FILE']
@@ -46,6 +46,8 @@ module FlakeySpecCatcher
       @use_parent = @cli_override.use_parent
       @repeat_factor = @cli_override.repeat_factor if @cli_override.repeat_factor.to_i.positive?
       @break_on_first_failure = @cli_override.break_on_first_failure
+      @list_child_specs = @cli_override.list_child_specs unless @cli_override.list_child_specs.nil?
+      @random_timing = @cli_override.random_timing unless @cli_override.random_timing.nil?
       @enable_runs = @cli_override.enable_runs
       @dry_run = @cli_override.dry_run
       @split_nodes = @cli_override.split_nodes unless @cli_override.split_nodes.nil?
@@ -59,7 +61,7 @@ module FlakeySpecCatcher
       @verbose = @cli_override.verbose
       @test_options = @cli_override.test_options
     end
-    # rubocop:enable Metrics/AbcSize
+    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
     def override_settings
       apply_cli_override
@@ -107,11 +109,7 @@ module FlakeySpecCatcher
     end
 
     def env_var_string_to_bool(env_var)
-      if env_var.to_s.casecmp('true').zero?
-        true
-      else
-        false
-      end
+      env_var.to_s.casecmp('true').zero?
     end
 
     def env_var_string_to_pairs(env_var)
@@ -170,7 +168,7 @@ module FlakeySpecCatcher
       end
     end
 
-    # rubocop:disable Metrics/CyclomaticComplexity
+    # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength
     def override_user_config(env_var, env_value)
       case env_var
       when 'FSC_REPEAT_FACTOR'
@@ -183,6 +181,10 @@ module FlakeySpecCatcher
         @silent_mode = env_var_string_to_bool(env_value)
       when 'FSC_RERUN_FILE_ONLY'
         @rerun_file_only = env_var_string_to_bool(env_value)
+      when 'FSC_LIST_CHILD_SPECS'
+        @list_child_specs = env_var_string_to_bool(env_value)
+      when 'FSC_RANDOM_TIMING'
+        @random_timing = env_var_string_to_bool(env_value)
       when 'FSC_USAGE_PATTERNS'
         @rspec_usage_patterns = env_var_string_to_pairs(env_value)
       when 'FSC_EXCLUDED_TAGS'
@@ -191,6 +193,6 @@ module FlakeySpecCatcher
         @output_file = env_value
       end
     end
-    # rubocop:enable Metrics/CyclomaticComplexity
+    # rubocop:enable Metrics/CyclomaticComplexity, Metrics/MethodLength
   end
 end

data/lib/flakey_spec_catcher/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FlakeySpecCatcher
-  VERSION = '0.10.0'
+  VERSION = '0.12.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flakey_spec_catcher
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.12.0
 platform: ruby
 authors:
 - Brian Watson
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-28 00:00:00.000000000 Z
+date: 2021-11-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -26,6 +26,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.10'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -126,6 +140,7 @@ files:
 - lib/flakey_spec_catcher/rspec_result.rb
 - lib/flakey_spec_catcher/rspec_result_manager.rb
 - lib/flakey_spec_catcher/runner.rb
+- lib/flakey_spec_catcher/timecop_manager.rb
 - lib/flakey_spec_catcher/user_config.rb
 - lib/flakey_spec_catcher/version.rb
 - lib/helpers/colorize.rb