flakey_spec_catcher 0.11.0 → 0.12.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1be370079a94e8f578e03be6e8683cd76ff8d1f5656422427c8ca3920856b38c
-  data.tar.gz: 6dddff4adf5e7f452a366bff7b67a50e8e4a04e64a9165c80e9318235afcae33
+  metadata.gz: 7c372506dc001281381c2da00adfbb57630541aec3ef03a537becb9330da6a47
+  data.tar.gz: '041469dd05911ac067396527eed5bc5de0c480c3e981a1d6688eb4070abe188d'
 SHA512:
-  metadata.gz: b49520ad8fe8a9115ae04bf90c93159694c0bcbbae087acaa9953aac26927b61efb68e3531d9097ea86b5b60d8bf6f84df685baf6798c95bcd0f5e560c752af5
-  data.tar.gz: 927b24f3283dfdd8575c3210a560c21a63f315bc98ada713390b15ac1d3dea0773534d761bf51ca5216eaee03567f7602e50615dc85fdab52620ab78ea456070
+  metadata.gz: 4390187c9364ebe61355e0de7f821bfac2331baa85435ec7afceffe02500baea8b79ccc126d448c5f97cb20093e4bcbf097ba281e81fdc7461f18c06bacca337
+  data.tar.gz: 7599cda6777957328931ba99f50369ca6c567e8f1d947c85862971ac504170e256617ac61de68e099b22516d5af133c0736044d3809d711f2df891dff282c481

data/flakey_spec_catcher.gemspec

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# rubocop:disable Layout/ExtraSpacing, Layout/SpaceAroundOperators
+# rubocop:disable Layout/ExtraSpacing, Layout/SpaceAroundOperators, Metrics/BlockLength
 require './lib/flakey_spec_catcher/version'
 
 Gem::Specification.new do |gem|
@@ -27,14 +27,19 @@ Gem::Specification.new do |gem|
   gem.executables   = 'flakey_spec_catcher'
   gem.require_paths = ['lib']
 
-  gem.metadata['allowed_push_host'] = 'https://rubygems.org'
+  gem.metadata = {
+    'allowed_push_host' => 'https://rubygems.org',
+    'rubygems_mfa_required' => 'true'
+  }
+
   gem.required_ruby_version         = '>= 2.6'
 
   gem.add_dependency 'rspec', '~> 3.10'
+  gem.add_dependency 'timecop', '~> 0.9'
   gem.add_development_dependency 'byebug', '~> 11.1'
   gem.add_development_dependency 'climate_control', '~> 0.2'
   gem.add_development_dependency 'rake', '~> 13.0'
   gem.add_development_dependency 'rubocop', '~> 0.93.1'
   gem.add_development_dependency 'simplecov', '~> 0.19'
 end
-# rubocop:enable Layout/ExtraSpacing, Layout/SpaceAroundOperators
+# rubocop:enable Layout/ExtraSpacing, Layout/SpaceAroundOperators, Metrics/BlockLength

data/lib/flakey_spec_catcher/change_capsule.rb

@@ -24,23 +24,30 @@ module FlakeySpecCatcher
       @change_contexts.map(&:rerun_info)
     end
 
+    # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
     def fill_contexts
       change_context_stack = []
       ignore_scope_closure = 0
       lines_in_file = File.read(@file_name).split("\n")
       lines_in_file.each_with_index do |line, index|
+        shared_example_identified = false
+
         # Check if line matches an rspec example or examplegroup format
         if line =~ spec_scope
           handle_change_context(line, index, change_context_stack)
         # Else, ignore other blocks that might pollute context stack
+        elsif line =~ shared_example_scope
+          handle_change_context(line, index, change_context_stack)
+          shared_example_identified = true
         elsif line_matches_method_or_block(line)
           ignore_scope_closure += 1
         end
 
         fill_context(line, index, change_context_stack)
 
-        # Note - some things use do-like loops and we need to be able to ignore those
-        if line =~ pop_scope
+        if shared_example_identified
+          change_context_stack.pop
+        elsif line =~ pop_scope
           if ignore_scope_closure.positive?
             ignore_scope_closure -= 1
           else
@@ -49,6 +56,7 @@ module FlakeySpecCatcher
         end
       end
     end
+    # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
 
     private
 
@@ -70,14 +78,15 @@ module FlakeySpecCatcher
     end
 
     def spec_scope
-      # Not sure if we need to check for description in quotes
-      # spec_scope = /^\s*(#{SCOPE_SPECIFIERS.join("|")})\s*('.*'|".*").*do\s*$/
-
       /\s*(#{SCOPE_SPECIFIERS.join("|")}).*\s+do.*$/
     end
 
+    def shared_example_scope
+      /\s*(#{SHARED_EXAMPLES.join("|")}).*\s+.*$/
+    end
+
     def line_matches_method_or_block(line)
-      return true if line =~ /\s*do(\s+|$)/ || line =~ /^\s*def\s+/ || line =~ /^\s*if\s+/
+      return true if line =~ /\s*do(\s+|$)/ || line =~ /^\s*def\s+/ || line =~ /^\s*if\s+/ || line =~ /^\s*class\s+/
 
       false
     end
@@ -122,6 +131,10 @@ module FlakeySpecCatcher
 
       current_scope = change_contexts[-1].rerun_info
       parent_scope = change_contexts[-2].rerun_info
+
+      # Don't build the file level context into spec tree
+      return unless parent_scope =~ /:/
+
       @spec_tree[parent_scope] = if @spec_tree[parent_scope]
         @spec_tree[parent_scope] << current_scope
       else

data/lib/flakey_spec_catcher/cli_override.rb

@@ -9,7 +9,7 @@ module FlakeySpecCatcher
   class CliOverride
     attr_reader :rerun_patterns, :rerun_usage, :repeat_factor, :enable_runs, :excluded_tags, :use_parent, :dry_run
     attr_reader :output_file, :split_nodes, :split_index, :verbose, :test_options, :break_on_first_failure
-    attr_reader :list_child_specs
+    attr_reader :list_child_specs, :random_timing
 
     def initialize
       @dry_run = false
@@ -65,6 +65,10 @@ module FlakeySpecCatcher
           @list_child_specs = list_child_specs
         end
 
+        opts.on('--random-timing', 'Run Specs at random times') do |random_timing|
+          @random_timing = random_timing
+        end
+
         opts.on('-e', '--excluded-tags=EXCLUDED_TAGS',
                 'Specify tags to exclude in a comma separated list') do |tags|
           @excluded_tags = parse_tags(tags)

data/lib/flakey_spec_catcher/rerun_manager.rb

@@ -173,11 +173,22 @@ module FlakeySpecCatcher
       end
     end
 
+    def transform_file_level_change(test, tree)
+      # Map file level changes to the highest level Example Group in the tree
+      if test =~ /:/
+        test
+      else
+        tree.keys.first
+      end
+    end
+
     def transform_parent_specs(tests)
       transformed_tests = []
+
       tests.each do |test|
         capsule = @git_controller.capsule_manager.change_capsules.find { |c| c.file_name == test.split(':')[0] }
         spec_tree = capsule.spec_tree
+        test = transform_file_level_change(test, spec_tree)
         # If test is a key in the spec_tree, it's an example group, so we queue up its descendant examples
         if spec_tree[test]
           spec_tree[test].each do |child|

data/lib/flakey_spec_catcher/rspec_result.rb

@@ -13,13 +13,15 @@ module FlakeySpecCatcher
   # This class will then organize and output the results accordingly.
   class RspecResult
     attr_accessor :description, :location, :total_times_run, :total_failures
+    attr_reader :spec_start_times, :failures
 
-    def initialize(description, location, exception_message = nil)
+    def initialize(description, location, spec_start_times, exception_message = nil)
       @description = description
       @location = location
       @total_times_run = 1
       @total_failures = exception_message ? 1 : 0
       @failures = []
+      @spec_start_times = spec_start_times
       add_failure(exception_message) if exception_message
     end
 
@@ -34,30 +36,47 @@ module FlakeySpecCatcher
 
     def add_failure(exception_message)
       failure = @failures.find { |f| f.exception_message == exception_message }
-      failure ? failure.add_failure : @failures.push(RSpecFailure.new(exception_message))
+      if failure
+        failure.add_failure(current_spec_start_time)
+      else
+        @failures.push(RSpecFailure.new(exception_message, current_spec_start_time))
+      end
+    end
+
+    def current_spec_start_time
+      @spec_start_times[@total_times_run - 1]
     end
 
     def print_results
       puts "\n#{@description.yellow} (#{location})
             \nFAILED #{total_failures} / #{total_times_run} times"
 
-      @failures.each do |f|
-        puts "#{f.count.to_s.indent(2)} times with exception message:"
-        puts f.exception_message.indent(4).red.to_s
-      end
+      @failures.each { |failure| puts failure.failure_summary }
     end
 
     # Simple class to contain failed example data
     class RSpecFailure
-      attr_reader :exception_message, :count
+      attr_reader :exception_message, :count, :spec_start_times
 
-      def initialize(exception_message)
+      def initialize(exception_message, spec_start_time = nil)
         @exception_message = exception_message
         @count = 1
+        @spec_start_times = [spec_start_time].compact
       end
 
-      def add_failure
+      def add_failure(spec_start_time = nil)
         @count += 1
+        @spec_start_times.push(spec_start_time) unless spec_start_time.nil?
+      end
+
+      def failure_summary
+        summary = "#{count.to_s.indent(2)} times with exception message:\n"
+        summary += exception_message.indent(4).red.to_s
+        return summary if spec_start_times.empty?
+
+        summary += "\n\nFailed at the following times:\n".indent(2)
+        summary += spec_start_times.map { |time| time.indent(4).yellow.to_s }.join("\n").to_s
+        summary
       end
     end
   end

data/lib/flakey_spec_catcher/rspec_result_manager.rb

@@ -12,14 +12,25 @@ module FlakeySpecCatcher
   # distinct example. It also provides helpers for adding new results,
   # displaying aggregate results, and checking the state of the collection.
   class RspecResultManager
+    attr_reader :results
+
     def initialize(rspec_result_class)
       @result_class = rspec_result_class
       @results = []
+      @spec_start_times = []
+    end
+
+    def track_spec_start_times(spec_times)
+      @spec_start_times = spec_times
     end
 
     def add_result(desc, location, message = nil)
       result = @results.find { |r| r.location == location }
-      result ? result.add_run(message) : @results.push(@result_class.new(desc, location, message))
+      if result
+        result.add_run(message)
+      else
+        @results.push(@result_class.new(desc, location, @spec_start_times, message))
+      end
     end
 
     def print_results

data/lib/flakey_spec_catcher/runner.rb

@@ -7,10 +7,11 @@ require_relative './user_config'
 require_relative './rerun_manager'
 require_relative './rspec_result_manager'
 require_relative './event_listener.rb'
+require_relative './timecop_manager.rb'
 
 module FlakeySpecCatcher
   class Runner
-    attr_reader :user_config, :rerun_manager, :git_controller, :test_run_count
+    attr_reader :user_config, :rerun_manager, :git_controller, :test_run_count, :random_dates
 
     def initialize(test_mode: false,
       user_config: FlakeySpecCatcher::UserConfig.new,
@@ -18,17 +19,17 @@ module FlakeySpecCatcher
       result_manager: FlakeySpecCatcher::RspecResultManager.new(FlakeySpecCatcher::RspecResult),
       rerun_manager: FlakeySpecCatcher::RerunManager.new(git_controller: git_controller,
                                                          user_config: user_config))
-
       @git_controller = git_controller
       @user_config = user_config
       @rerun_manager = rerun_manager
       @rspec_result_manager = result_manager
       @test_run_count = 0
       @temp_output_file = @user_config.output_file + 'temp' unless @user_config.output_file == File::NULL
+      enable_random_timing
     end
 
     # Debug Methods
-    # rubocop:disable Metrics/AbcSize
+    # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity
     def show_settings
       puts 'Flakey Spec Catcher Settings:'
       puts "  Current Branch: #{@git_controller.branch}" unless @user_config.use_parent
@@ -39,12 +40,13 @@ module FlakeySpecCatcher
       puts "  Break on first failure: #{@user_config.break_on_first_failure}" if @user_config.break_on_first_failure
       puts "  Node Total: #{@user_config.split_nodes}" if @user_config.split_nodes
       puts "  Node Index: #{@user_config.split_index}" if @user_config.split_index
+      puts '  Random Timing: Enabled' if @user_config.random_timing
       puts "  Changed Specs Detected: #{@git_controller.changed_examples}"
       return if @user_config.output_file == File::NULL
 
       puts "  Verbose Output Path: #{@user_config.output_file}"
     end
-    # rubocop:enable Metrics/AbcSize
+    # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity
 
     def rerun_preview
       puts "\n********************************************"
@@ -73,8 +75,8 @@ module FlakeySpecCatcher
 
       status = 0
       @rerun_manager.rerun_capsules.sort.each do |capsule|
-        @user_config.repeat_factor.times do
-          iteration_status = handle_capsule_rerun(capsule)
+        @user_config.repeat_factor.times do |iteration|
+          iteration_status = timecop_wrapper(capsule, iteration)
           status = [status, iteration_status].max
           break if @user_config.break_on_first_failure && !status.zero?
         end
@@ -127,6 +129,23 @@ module FlakeySpecCatcher
       end
     end
 
+    def enable_random_timing
+      @random_dates = if @user_config.random_timing
+        FlakeySpecCatcher::TimecopManager.generate_dates(@user_config.repeat_factor)
+      else
+        []
+      end
+      @rspec_result_manager.track_spec_start_times(@random_dates)
+    end
+
+    def timecop_wrapper(capsule, iteration)
+      if @user_config.random_timing
+        FlakeySpecCatcher::TimecopManager.randomly_travel_in_time(@random_dates[iteration]) { handle_capsule_rerun(capsule) }
+      else
+        handle_capsule_rerun(capsule)
+      end
+    end
+
     def print_flakey_specs_detected_message
       puts "\n**********************************************".magenta
       puts '          Flakiness Detected!'.magenta

data/lib/flakey_spec_catcher/timecop_manager.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require 'date'
+require 'time'
+require 'timecop'
+
+module FlakeySpecCatcher
+  class TimecopManager
+    def self.randomly_travel_in_time(date)
+      status = 0
+      Timecop.travel(date) do
+        status = yield
+      end
+      status
+    end
+
+    def self.current_month
+      Time.now.month
+    end
+
+    def self.current_year
+      Time.now.year
+    end
+
+    def self.random_month
+      (current_month..12).to_a.sample
+    end
+
+    def self.random_seconds
+      [0, 59].sample
+    end
+
+    def self.random_minutes
+      [0, (1..58).to_a.sample, 59].sample
+    end
+
+    def self.random_day(year, month)
+      last_day = last_day_of_month(year, month).day
+      [1, (1..last_day).to_a.sample, last_day].sample
+    end
+
+    def self.last_day_of_month(year, month)
+      day = Date.new(year, month, -1).day
+      Time.local(year, month, day, 23, 59, 59)
+    end
+
+    def self.last_day_of_year(year)
+      Time.local(year, 12, 31, 11, 59, 59)
+    end
+
+    def self.random_hour
+      [0, (1..22).to_a.sample, 23].sample
+    end
+
+    def self.random_date
+      year = current_year
+      month = random_month
+      day = random_day(year, month)
+      Time.local(year, month, day, random_hour, random_minutes, random_seconds)
+    end
+
+    def self.prioritized_dates
+      [last_day_of_month(current_year, current_month), last_day_of_year(current_year)]
+    end
+
+    def self.generate_dates(count)
+      dates = prioritized_dates
+      (count - prioritized_dates.count).times { dates << random_date }
+      dates.map(&:iso8601).slice(0, count)
+    end
+  end
+end

data/lib/flakey_spec_catcher/user_config.rb

@@ -3,20 +3,18 @@
 require_relative './cli_override'
 module FlakeySpecCatcher
   # UserConfig class
-  #
   # Captures user-defined settings to configure RSpec re-run settings.
 
   class UserConfig
-    attr_reader :repeat_factor, :ignore_files, :ignore_branches, :silent_mode
-    attr_reader :rerun_file_only, :rspec_usage_patterns, :excluded_tags
-    attr_reader :manual_rerun_patterns, :manual_rerun_usage
-    attr_reader :enable_runs, :output_file, :use_parent, :dry_run
-    attr_reader :split_nodes, :split_index, :verbose, :test_options
-    attr_reader :break_on_first_failure, :list_child_specs
+    attr_reader :repeat_factor, :ignore_files, :ignore_branches, :silent_mode, :rerun_file_only, :rspec_usage_patterns
+    attr_reader :excluded_tags, :manual_rerun_patterns, :manual_rerun_usage, :enable_runs, :output_file
+    attr_reader :use_parent, :dry_run, :split_nodes, :split_index, :verbose, :test_options, :break_on_first_failure
+    attr_reader :list_child_specs, :random_timing
 
     USER_CONFIG_ENV_VARS = %w[FSC_REPEAT_FACTOR FSC_IGNORE_FILES FSC_IGNORE_BRANCHES
                               FSC_SILENT_MODE FSC_RERUN_FILE_ONLY FSC_USAGE_PATTERNS
-                              FSC_EXCLUDED_TAGS FSC_OUTPUT_FILE FSC_LIST_CHILD_SPECS].freeze
+                              FSC_EXCLUDED_TAGS FSC_OUTPUT_FILE FSC_LIST_CHILD_SPECS
+                              FSC_RANDOM_TIMING].freeze
 
     def initialize(cli_override: CliOverride.new)
       apply_env_var_settings
@@ -26,7 +24,7 @@ module FlakeySpecCatcher
 
     private
 
-    # rubocop:disable Metrics/AbcSize
+    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
     def apply_env_var_settings
       @repeat_factor = initialize_repeat_factor(ENV['FSC_REPEAT_FACTOR'])
       @ignore_files = env_var_string_to_array(ENV['FSC_IGNORE_FILES'])
@@ -34,6 +32,7 @@ module FlakeySpecCatcher
       @silent_mode = env_var_string_to_bool(ENV['FSC_SILENT_MODE'])
       @rerun_file_only = env_var_string_to_bool(ENV['FSC_RERUN_FILE_ONLY'])
       @list_child_specs = env_var_string_to_bool(ENV['FSC_LIST_CHILD_SPECS'])
+      @random_timing = env_var_string_to_bool(ENV['FSC_RANDOM_TIMING'])
       @rspec_usage_patterns = env_var_string_to_pairs(ENV['FSC_USAGE_PATTERNS'])
       @excluded_tags = env_var_string_to_tags(ENV['FSC_EXCLUDED_TAGS'])
       @output_file = ENV['FSC_OUTPUT_FILE']
@@ -48,6 +47,7 @@ module FlakeySpecCatcher
       @repeat_factor = @cli_override.repeat_factor if @cli_override.repeat_factor.to_i.positive?
       @break_on_first_failure = @cli_override.break_on_first_failure
       @list_child_specs = @cli_override.list_child_specs unless @cli_override.list_child_specs.nil?
+      @random_timing = @cli_override.random_timing unless @cli_override.random_timing.nil?
       @enable_runs = @cli_override.enable_runs
       @dry_run = @cli_override.dry_run
       @split_nodes = @cli_override.split_nodes unless @cli_override.split_nodes.nil?
@@ -61,7 +61,7 @@ module FlakeySpecCatcher
       @verbose = @cli_override.verbose
       @test_options = @cli_override.test_options
     end
-    # rubocop:enable Metrics/AbcSize
+    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
     def override_settings
       apply_cli_override
@@ -109,11 +109,7 @@ module FlakeySpecCatcher
     end
 
     def env_var_string_to_bool(env_var)
-      if env_var.to_s.casecmp('true').zero?
-        true
-      else
-        false
-      end
+      env_var.to_s.casecmp('true').zero?
     end
 
     def env_var_string_to_pairs(env_var)
@@ -187,6 +183,8 @@ module FlakeySpecCatcher
         @rerun_file_only = env_var_string_to_bool(env_value)
       when 'FSC_LIST_CHILD_SPECS'
         @list_child_specs = env_var_string_to_bool(env_value)
+      when 'FSC_RANDOM_TIMING'
+        @random_timing = env_var_string_to_bool(env_value)
       when 'FSC_USAGE_PATTERNS'
         @rspec_usage_patterns = env_var_string_to_pairs(env_value)
       when 'FSC_EXCLUDED_TAGS'

data/lib/flakey_spec_catcher/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FlakeySpecCatcher
-  VERSION = '0.11.0'
+  VERSION = '0.12.1'
 end

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: flakey_spec_catcher
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.12.1
 platform: ruby
 authors:
 - Brian Watson
 - Mikey Hargiss
 - Ben Nelson
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-10-12 00:00:00.000000000 Z
+date: 2021-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -26,6 +26,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.10'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -96,7 +110,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.19'
-description:
+description: 
 email:
 - bwatson@instructure.com
 - mhargiss@instructure.com
@@ -126,16 +140,18 @@ files:
 - lib/flakey_spec_catcher/rspec_result.rb
 - lib/flakey_spec_catcher/rspec_result_manager.rb
 - lib/flakey_spec_catcher/runner.rb
+- lib/flakey_spec_catcher/timecop_manager.rb
 - lib/flakey_spec_catcher/user_config.rb
 - lib/flakey_spec_catcher/version.rb
 - lib/helpers/colorize.rb
 - lib/helpers/indent_string.rb
-homepage:
+homepage: 
 licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message:
+  rubygems_mfa_required: 'true'
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -150,8 +166,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.1
-signing_key:
+rubygems_version: 3.1.6
+signing_key: 
 specification_version: 4
 summary: Run new or changed specs many times to prevent unreliable specs
 test_files: []