flakey_spec_catcher 0.3.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 71dd2fe6a58fc01c2647d06f6aa20f9c904f6ced4d1a54278d96344c40534713
-  data.tar.gz: cc060a637c1e14e04e7fd914b84c0b6c854912512a649fcf95696f57069444e1
+  metadata.gz: 1c2c74dc190189f31023ee816f41a7856aceee56ccca2de3f385b3559932cfde
+  data.tar.gz: 15b1cf40884ea015014a2d0b4b96ea223f46f9381751282d370e21641a0456da
 SHA512:
-  metadata.gz: 28114d12131611ae6405bd0c3d066b44c16c3efece45633e46b1c8f5a263eec7189c45b46efbf112897e5e7cbeecf110f493ee499c79d76647f64e2da4a99abe
-  data.tar.gz: be57cbc9ab2052f770507effd1b2e985124607bc5bfa11b0c659279726969aea9390df6b352501bdd31a1e179cb3c554fd448dc24c705f4974087772f502bd00
+  metadata.gz: f3eea41fa5d50fd29d292810a5b1fbf0c25452e57c85c81999fb257063a02f823657e15bd2cb9e23c63fe90923367d89b9098e5aef56142dcf51f9638412d86e
+  data.tar.gz: 7fbc79e679dfc5c4f38c995c377ab119eae0d4e890d1d5b7b97ee9516215f60e094439f246dfd7e97c2b4fa3f5c2dd5293979d230d26da7fc19174d1192ce327

data/README.md

@@ -75,6 +75,20 @@ FSC_USAGE_PATTERNS = '{ spec/ui => bundle exec rspec }, { spec/api => parallel_r
 
 ```
 
+### Manually re-running a test:
+
+Once the gem has been installed, you may manually specify a test to run along with a custom usage
+(if none is provided, the specified test will run with RSpec::Core::Runner) and a repeat_factor.
+If no repeat_factor is provided, FSC will check ENV['FSC_REPEAT_FACTOR'] to see if a value
+has been configured. If none was provided, it will fall back on the default of  20 re-runs
+
+Usage Examples:
+```
+flakey_spec_catcher --test='api/spec/user_spec.rb:3' --repeat='15'
+flakey_spec_catcher --test='api/spec/*_spec.rb' --usage='bundle exec parallel_rspec'
+flakey_spec_catcher --test='api/spec/admin_spec.rb' --repeat='10' --usage='rspec'
+```
+
 ### How FSC works:
 
   1. GitController runs a git diff and creates a ChangeSummary object to

data/lib/flakey_spec_catcher/cli_override.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'optparse'
+
+module FlakeySpecCatcher
+  # CliOverride class
+  #
+  # Captures command line arguments for manual re-runs
+  class CliOverride
+    attr_reader :rerun_pattern, :rerun_usage, :repeat_factor
+
+    def initialize
+      parse_command_line_args
+    end
+
+    private
+
+    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+    def parse_command_line_args
+      OptionParser.new do |opts|
+        opts.banner = 'Usage: flakey_spec_catcher [OPTIONS]'
+
+        opts.on('-t', '--test=TEST_NAME',
+                'Specify a spec as `path/to/file:line_number` or `file/path` to run') do |test|
+          @rerun_pattern = remove_formatter_quotes(test)
+        end
+
+        opts.on('-u', '--usage=USAGE', 'Specify a re-run usage for the manual re-run') do |usage|
+          raise ArgumentError if @rerun_pattern.nil?
+
+          @rerun_usage = remove_formatter_quotes(usage)
+        end
+
+        opts.on('-r', '--repeat=REPEAT_FACTOR',
+                'Specify a repeat factor for the manual re-run(s)') do |repeat|
+          raise ArgumentError if @rerun_pattern.nil?
+
+          parsed_repeat_factor = remove_formatter_quotes(repeat).to_i
+          @repeat_factor = parsed_repeat_factor if parsed_repeat_factor.positive?
+        end
+
+        opts.on('-v', '--version', 'Prints current flakey_spec_catcher_version') do
+          puts "flakey_spec_catcher Version: #{FlakeySpecCatcher::VERSION}"
+        end
+
+        opts.on('-h', '--help', 'Displays available flakey_spec_catcher cli overrides') do
+          puts opts
+        end
+      end.parse!
+    end
+    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+
+    def remove_formatter_quotes(env_var)
+      env_var.tr('\'\"', '')
+    end
+  end
+end

data/lib/flakey_spec_catcher/event_listener.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'rspec'
+
+module FlakeySpecCatcher
+  # EventListener class
+  #
+  # Listens for events from RSpec and processes data from those events.
+  #
+  # This event listener will receive example_failed and example_passed events
+  # from RSpec and will log those results into the provided result manager
+  class EventListener
+    def initialize(result_manager)
+      @result_manager = result_manager
+    end
+
+    def example_failed(notification)
+      description = notification.example.full_description.to_s.strip
+      @result_manager.add_result(description, format_message(notification))
+    end
+
+    def example_passed(notification)
+      description = notification.example.full_description.to_s.strip
+      @result_manager.add_result(description)
+    end
+
+    private
+
+    def format_message(notification)
+      exception = notification.example.execution_result.exception
+      message = if exception.is_a?(RSpec::Core::MultipleExceptionError)
+        exception.all_exceptions.join("\n").strip
+      else
+        exception.to_s.strip
+      end
+
+      message
+    end
+  end
+end

data/lib/flakey_spec_catcher/rerun_manager.rb

@@ -16,7 +16,17 @@ module FlakeySpecCatcher
       @git_controller = git_controller
       @user_config = user_config
       @rerun_capsules = []
-      pair_reruns_with_usages
+      determine_rerun_usage
+    end
+
+    def determine_rerun_usage
+      if @user_config.manual_rerun_pattern.nil?
+        pair_reruns_with_usages
+      else
+        @rerun_capsules.clear
+        inject_manual_reruns(@user_config.manual_rerun_pattern,
+                             @user_config.manual_rerun_usage)
+      end
     end
 
     def tests_for_rerun
@@ -52,6 +62,37 @@ module FlakeySpecCatcher
       end
     end
 
+    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+    def inject_manual_reruns(pattern, usage)
+      # Check if file exists first and handle if user supplies testcase:line_number
+      file_name = pattern.split(':')[0]
+      line_number_present = pattern.split(':').count > 1
+      matching_files = Dir.glob(file_name)
+
+      # If no file matches are run, don't queue up re-runs
+      if matching_files.count.zero?
+        puts "Specified pattern #{pattern} did not match an existing file"
+        raise ArgumentError
+      end
+
+      # It won't make sense to have multiple files to run with one specific line number
+      if line_number_present
+        if matching_files.count > 1
+          puts "Specified pattern #{pattern} matched multiple files but a line number was specified"
+          raise ArgumentError
+        else
+          add_rerun_capsule(testcase: pattern, usage: usage)
+        end
+
+      # No line numbers, queue up all matching files
+      else
+        matching_files.each do |file|
+          add_rerun_capsule(testcase: file, usage: usage)
+        end
+      end
+    end
+    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+
     private
 
     def filter_reruns_by_ignore_files(reruns)

data/lib/flakey_spec_catcher/rspec_result.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require_relative '../helpers/colorize'
+require_relative '../helpers/indent_string'
+
 module FlakeySpecCatcher
   # RspecResult class
   #
@@ -9,116 +12,50 @@ module FlakeySpecCatcher
   # re-run of that spec, the results will be pushed to the RspecResult object.
   # This class will then organize and output the results accordingly.
   class RspecResult
-    attr_reader :file_name, :total_examples_run, :total_failures, :failure_summary
+    attr_accessor :description, :total_times_run, :total_failures
 
-    def initialize(file_name)
-      @file_name = file_name
-      @total_examples_run = 0
-      @total_failures = 0
-      @failure_summary = []
+    def initialize(description, exception_message = nil)
+      @description = description
+      @total_times_run = 1
+      @total_failures = exception_message ? 1 : 0
+      @failures = []
+      add_failure(exception_message) if exception_message
     end
 
-    def info
-      ratio = "#{@total_examples_run - @total_failures}/#{@total_examples_run}"
-      puts "\nFile Name: #{@file_name} passed #{ratio} examples"
+    def add_run(exception_message = nil)
+      @total_times_run += 1
 
-      return unless @total_failures.positive?
+      return unless exception_message
 
-      puts '  Failure Summary:'
-      @failure_summary.each do |example, _occurrences|
-        example.info
-      end
+      @total_failures += 1
+      add_failure(exception_message)
     end
 
-    def add_run(results)
-      parse_totals(results)
-
-      # Parse the failed examples and add them into the failure_summary
-      if @failure_summary.empty?
-        # Add the initial runs
-        @failure_summary = parse_failed_examples(results)
-      else
-        populate_failure_summary(parse_failed_examples(results))
-      end
+    def add_failure(exception_message)
+      failure = @failures.find { |f| f.exception_message == exception_message }
+      failure ? failure.add_failure : @failures.push(RSpecFailure.new(exception_message))
     end
 
-    private
-
-    def parse_failed_examples(results)
-      failures = []
-      # Currently we're only going to return the name of the example that failed
-      # Example:
-      # 1) Flakey spec Passes some of the time
-      # Failure/Error: expect(1).to eq(rand(3))
-      #
-      #   expected: 0
-      #        got: 1
-      #
-      #   (compared using ==)
-      #  ./spec/flakey_example_spec.rb:6:in `block (2 levels) in <top (required)>'
-
-      # Would return 'Flakey spec Passes some of the time'
-
-      # Get the line that contains the failed example - Denoted by '1)' in the examples
-      #  and strip the example ennumerators
-      example_lines = results.scan(/^\s*\d\).*$/)
-      failed_examples = example_lines.map { |example| example.strip.sub!(/\d\)\s*/, '') }
-
-      # Use the first line of the failed_examples to get the actual failures
-      failed_examples.each do |example|
-        # Go from the spec example description until '#)' or 'Finished in'
-        result = results[/#{Regexp.escape(example)}(.*?)(Finished in| \d\))/m, 1]
-        failures << RSpecFailure.new(example, result.strip)
+    def print_results
+      puts "\n#{@description.yellow}\nFAILED #{total_failures} / #{total_times_run} times"
+      @failures.each do |f|
+        puts "#{f.count.to_s.indent(2)} times with exception message:"
+        puts f.exception_message.indent(4).red.to_s
       end
-
-      # Return the array of RSpecFailures
-      failures
     end
 
-    def parse_totals(results)
-      # Get summary line to determine number of run results.
-      # Will appear as "3 examples, 0 failures"
-      summary = results.split("\n").find { |line| line =~ /example.*failure/ }
-      pass_count, fail_count = summary.split(' ').select { |x| x[/\d/] }
-      @total_examples_run += pass_count.to_i
-      @total_failures += fail_count.to_i
-    end
+    # Simple class to contain failed example data
+    class RSpecFailure
+      attr_reader :exception_message, :count
 
-    def populate_failure_summary(failures)
-      # For each failure, if that example is in @failure_summary, add the results to it
-      failures.each do |failure|
-        match_found = false
-        @failure_summary.each do |example|
-          # If example is already in the summary, add the expect/actual to the example's failures
-          if example.example_name == failure.example_name
-            example.add_case(failure.failure_details.join(''))
-            match_found = true
-          end
-        end
-        # If example is not in the summary, add the whole object to the array of examples
-        @failure_summary << failure unless match_found
+      def initialize(exception_message)
+        @exception_message = exception_message
+        @count = 1
       end
-    end
-  end
 
-  # Simple class to contain failed example data
-  class RSpecFailure
-    attr_reader :example_name, :failure_details
-
-    def initialize(example_name, results)
-      @example_name = example_name
-      @failure_count = 1
-      @failure_details = [results]
-    end
-
-    def add_case(results)
-      @failure_count += 1
-      @failure_details << results
-    end
-
-    def info
-      puts "\n    '#{@example_name}' failed #{@failure_count} time(s)"
-      @failure_details.each { |failure| puts "\n       #{failure}" }
+      def add_failure
+        @count += 1
+      end
     end
   end
 end

data/lib/flakey_spec_catcher/rspec_result_manager.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require_relative '../helpers/colorize'
+require_relative './rspec_result'
+
+module FlakeySpecCatcher
+  # RspecResultManager class
+  #
+  # Holds a collection of RSpec results and provides helper methods
+  #
+  # An RspecResultManager will hold a collection of results, one for each
+  # distinct example. It also provides helpers for adding new results,
+  # displaying aggregate results, and checking the state of the collection.
+  class RspecResultManager
+    def initialize(rspec_result_class)
+      @result_class = rspec_result_class
+      @results = []
+    end
+
+    def add_result(description, message = nil)
+      result = @results.find { |r| r.description == description }
+      result ? result.add_run(message) : @results.push(@result_class.new(description, message))
+    end
+
+    def print_results
+      puts "\n********** SUMMARY **********"
+      print_successes_count(successes)
+      failures.each(&:print_results) if failures.any?
+    end
+
+    def failures
+      @results.select { |r| r.total_failures.positive? }
+    end
+
+    def successes
+      @results.select { |r| r.total_failures.zero? }
+    end
+
+    def empty?
+      @results.empty?
+    end
+
+    private
+
+    def print_successes_count(successes)
+      puts "#{successes.count} example(s) ran without any failures".green if successes.any?
+    end
+  end
+end

data/lib/flakey_spec_catcher/runner.rb

@@ -1,27 +1,28 @@
 # frozen_string_literal: true
 
 require 'rspec'
-require_relative './rspec_result'
 require_relative './git_controller'
 require_relative './capsule_manager'
 require_relative './user_config'
 require_relative './rerun_manager'
+require_relative './rspec_result_manager'
+require_relative './event_listener.rb'
 
 module FlakeySpecCatcher
   class Runner
     attr_reader :user_config, :rerun_manager, :git_controller
-    attr_reader :rspec_results
 
     def initialize(test_mode: false,
       git_controller: FlakeySpecCatcher::GitController.new(test_mode: test_mode),
       user_config: FlakeySpecCatcher::UserConfig.new,
+      result_manager: FlakeySpecCatcher::RspecResultManager.new(FlakeySpecCatcher::RspecResult),
       rerun_manager: FlakeySpecCatcher::RerunManager.new(git_controller: git_controller,
                                                          user_config: user_config))
 
-      @rspec_results = []
       @git_controller = git_controller
       @user_config = user_config
       @rerun_manager = rerun_manager
+      @rspec_result_manager = result_manager
     end
 
     # Debug Methods
@@ -51,47 +52,61 @@ module FlakeySpecCatcher
       status = 0
       @user_config.repeat_factor.times do
         @rerun_manager.rerun_capsules.sort.each do |capsule|
-          iteration_status = 0
-          if capsule.default_usage?
-            iteration_status = invoke_rspec_runner(capsule.testcase)
-          else
-            `#{capsule.usage} #{capsule.testcase}`
-            iteration_status = $?.exitstatus # rubocop:disable Style/SpecialGlobalVars
-          end
+          iteration_status = handle_capsule_rerun(capsule)
           status = [status, iteration_status].max
         end
       end
 
+      display_results(status)
+
       # Always return 0 if silent_mode is enabled
-      if @user_config.silent_mode
-        0
-      else
-        status
-      end
+      @user_config.silent_mode ? 0 : status
     end
 
     private
 
+    def display_results(status)
+      @rspec_result_manager.print_results unless @rspec_result_manager.empty?
+      status.zero? ? print_no_flakey_specs_detected_message : print_flakey_specs_detected_message
+    end
+
     def invoke_rspec_runner(test)
-      return_status = RSpec::Core::Runner.run([test])
+      configure_listener
+      # Pass in CLI options to suppress normal output, and only run the specified test
+      return_status = RSpec::Core::Runner.run(['--out', '/dev/null', test])
       RSpec.clear_examples
       return_status
     end
 
-    def print_rspec_results
-      @rspec_results.map(&:get_info)
+    def invoke_custom_rspec_runner(usage, testcase)
+      `#{usage} #{testcase}`
+      $?.exitstatus # rubocop:disable Style/SpecialGlobalVars
+    end
 
-      # Determine the exit code based on the parsed results
-      if @rspec_results.any? { |r| r.total_failures.positive? }
-        puts "\n********************************************"
-        puts 'Flakiness Detected! Exiting with status code 1'
-        puts "********************************************\n"
-        return 1
+    def handle_capsule_rerun(capsule)
+      if capsule.default_usage?
+        invoke_rspec_runner(capsule.testcase)
       else
-        puts "\n***********************************************"
-        puts 'No Flakiness Detected! Exiting with status code 0'
-        puts "***********************************************\n"
-        return 0
+        invoke_custom_rspec_runner(capsule.usage, capsule.testcase)
+      end
+    end
+
+    def print_flakey_specs_detected_message
+      puts "\n**********************************************".magenta
+      puts '          Flakiness Detected!'.magenta
+      puts "**********************************************\n".magenta
+    end
+
+    def print_no_flakey_specs_detected_message
+      puts "\n*************************************************".green
+      puts '            No Flakiness Detected!'.green
+      puts "*************************************************\n".green
+    end
+
+    def configure_listener
+      RSpec.configure do |c|
+        c.reporter.register_listener EventListener.new(@rspec_result_manager),
+                                     :example_failed, :example_passed
       end
     end
   end

data/lib/flakey_spec_catcher/user_config.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative './cli_override'
 module FlakeySpecCatcher
   # UserConfig class
   #
@@ -7,21 +8,40 @@ module FlakeySpecCatcher
   class UserConfig
     attr_reader :repeat_factor, :ignore_files, :ignore_branches, :silent_mode
     attr_reader :rerun_file_only, :rspec_usage_patterns
+    attr_reader :manual_rerun_pattern, :manual_rerun_usage
     USER_CONFIG_ENV_VARS = %w[FSC_REPEAT_FACTOR FSC_IGNORE_FILES FSC_IGNORE_BRANCHES
                               FSC_SILENT_MODE FSC_RERUN_FILE_ONLY FSC_USAGE_PATTERNS].freeze
 
-    def initialize
+    def initialize(cli_override: CliOverride.new)
       @repeat_factor = initialize_repeat_factor(ENV['FSC_REPEAT_FACTOR'])
       @ignore_files = env_var_string_to_array(ENV['FSC_IGNORE_FILES'])
       @ignore_branches = env_var_string_to_array(ENV['FSC_IGNORE_BRANCHES'])
       @silent_mode = env_var_string_to_bool(ENV['FSC_SILENT_MODE'])
       @rerun_file_only = env_var_string_to_bool(ENV['FSC_RERUN_FILE_ONLY'])
       @rspec_usage_patterns = env_var_string_to_pairs(ENV['FSC_USAGE_PATTERNS'])
-      parse_commit_message
+      @cli_override = cli_override
+      override_settings
     end
 
     private
 
+    def override_settings
+      apply_cli_override
+      return unless @manual_rerun_pattern.nil?
+
+      parse_commit_message
+    end
+
+    def apply_cli_override
+      @manual_rerun_pattern = @cli_override.rerun_pattern
+      @manual_rerun_usage = @cli_override.rerun_usage
+      @repeat_factor = @cli_override.repeat_factor if @cli_override.repeat_factor.to_i.positive?
+    end
+
+    def remove_formatter_quotes(env_var)
+      env_var.tr('\'\"', '')
+    end
+
     def initialize_repeat_factor(env_var)
       env_var.to_i.positive? ? env_var.to_i : 20
     end
@@ -60,7 +80,7 @@ module FlakeySpecCatcher
 
       # For each array of matches, flatten the individual match group sub-array
       #  then remove the formatter quotes we require the values to be wrapped in
-      matches.map { |m| m.join('').tr("'", '') }
+      matches.map { |m| remove_formatter_quotes(m.join('')) }
     end
 
     def parse_commit_message

data/lib/flakey_spec_catcher/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FlakeySpecCatcher
-  VERSION = '0.3.1'
+  VERSION = '0.5.0'
 end

data/lib/helpers/colorize.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# This is an extension of the Ruby String class.
+# Example usage: puts "This message is red".red
+# From http://stackoverflow.com/a/11482430/3038677
+class String
+  module Colors
+    RED     = 31
+    GREEN   = 32
+    YELLOW  = 33
+    MAGENTA = 35
+  end
+
+  def red
+    colorize(Colors::RED)
+  end
+
+  def green
+    colorize(Colors::GREEN)
+  end
+
+  def yellow
+    colorize(Colors::YELLOW)
+  end
+
+  def magenta
+    colorize(Colors::MAGENTA)
+  end
+
+  private
+
+  def colorize(color_code)
+    "\e[#{color_code}m#{self}\e[0m"
+  end
+end

data/lib/helpers/indent_string.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: false
+
+String.class_eval do
+  def indent(count, char = ' ')
+    gsub(/([^\n]*)(\n|$)/) do |_match|
+      last_iteration = (Regexp.last_match(1) == '' && Regexp.last_match(2) == '')
+      line = ''
+      line << (char * count) unless last_iteration
+      line << Regexp.last_match(1)
+      line << Regexp.last_match(2)
+      line
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flakey_spec_catcher
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Brian Watson
@@ -90,13 +90,18 @@ files:
 - lib/flakey_spec_catcher/change_capsule.rb
 - lib/flakey_spec_catcher/change_context.rb
 - lib/flakey_spec_catcher/change_summary.rb
+- lib/flakey_spec_catcher/cli_override.rb
+- lib/flakey_spec_catcher/event_listener.rb
 - lib/flakey_spec_catcher/git_controller.rb
 - lib/flakey_spec_catcher/rerun_capsule.rb
 - lib/flakey_spec_catcher/rerun_manager.rb
 - lib/flakey_spec_catcher/rspec_result.rb
+- lib/flakey_spec_catcher/rspec_result_manager.rb
 - lib/flakey_spec_catcher/runner.rb
 - lib/flakey_spec_catcher/user_config.rb
 - lib/flakey_spec_catcher/version.rb
+- lib/helpers/colorize.rb
+- lib/helpers/indent_string.rb
 homepage: 
 licenses:
 - MIT
@@ -117,7 +122,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.1
+rubyforge_project: 
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Run new or changed specs many times to prevent unreliable specs