flakey_spec_catcher 0.9.2 → 0.9.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 924bcd5b7fe71399bc43663478b9b238731e9920e4fcc8c54ba8977a345ee065
-  data.tar.gz: '080682849736c9d46acd6eeeb8ccc8e98115fb9558004e821f4beed33dfd455e'
+  metadata.gz: fce0bbfeef6c7ba89586f7d59b64084329d685108396037f77dec4276cf7a402
+  data.tar.gz: 015d979e822fb914c0eaecbe1a518eb2971fba7d1a1d41c13e5ffb83e184b0f3
 SHA512:
-  metadata.gz: 68cf5c4b41ae6b158a501bab2a6134a40dd1d0ca3d7ce26452948b986ddf713d443458643d2a27a8ff565b9856c647867630015c78dc2f9e7784bf31b0d772e4
-  data.tar.gz: cee42fcbf243eaa03e56745491515ae7580e2e7974d751faef5f2387c40f07cb20b9f225d188e04009950b7cfd339720e406b789f4e7fa2ec537c2c109b81c18
+  metadata.gz: fa94174f28662dc5d1e10be69879852c9b7f1947dfa7acfcafc462916a9fa405a1fd8807a79a81f9cbf83fd91bc930370bae54491f892da7f90e9b3c1b2440c5
+  data.tar.gz: 2ce6f81244240c34a64a1c8319d95e56a50d24f270642f77ad5edf1f33464e8b2df601c2e6d20ca6705eca94f683e30caa81138b8909f57e0b301b5a897a09c4

data/README.md

@@ -152,11 +152,18 @@ FSC_USAGE_PATTERNS = '{ spec/ui => bundle exec rspec }, { spec/api => parallel_r
 ```
   -t, --test=TEST_NAME             Specify one or more specs in comma separated list
   -u, --usage=USAGE                Specify a re-run usage for the manual re-run
+  --use-parent                     Use the parent of commit for git diff instead of the Head of SCM
   -r, --repeat=REPEAT_FACTOR       Specify a repeat factor for the manual re-run(s)
   -e, --excluded-tags=EXCLUDED     Specify tags to exclude in a comma separated list
   -o, --output=PATH_TO_OUTPUT      Direct all re-run output to a specific file
+  --node-total                     Specify now many nodes the run is being split into
+  --node-index                     Specify the index this node represents in a split run
+  -d, --dry-run                    Performs all setup but doesn't run any tests
+  --dry-run-quiet                  Prints list of tests to be run
+  --verbose                        Send all output from running tests to stdout
   -v, --version                    Prints current flakey_spec_catcher_version
   -h, --help                       Displays available flakey_spec_catcher cli overrides
+  --rspec-options '[OPTIONS]'      Execute default usage with rspec options
 ```
 
 Examples:
@@ -195,6 +202,36 @@ flakey_spec_catcher --test='api/spec/*_spec.rb' --usage='bundle exec parallel_rs
 flakey_spec_catcher --test='api/spec/admin_spec.rb' --repeat='10' --usage='rspec'
 ```
 
+### Running on multiple nodes:
+
+If there are a lot of tests required to be checked, then you may want to run the
+tests on multiple nodes. this can be done with the combination of the `--node-total`
+and `--node-index` arguments.
+
+Usage Examples:
+```sh
+# if you have 30 tests to run
+
+# this will be tests 0-9
+bundle exec flakey_spec_catcher --node-total=3 --node-index=0
+# this will be tests 10-19
+bundle exec flakey_spec_catcher --node-total=3 --node-index=1
+# this will be tests 20-29
+bundle exec flakey_spec_catcher --node-total=3 --node-index=2
+```
+
+Note that the last node may have a different number of tests that it runs:
+```sh
+# if you have 20 tests to run
+
+# this will be tests 0-6 (7 tests)
+bundle exec flakey_spec_catcher --node-total=3 --node-index=0
+# this will be tests 7-13 (7 tests)
+bundle exec flakey_spec_catcher --node-total=3 --node-index=1
+# this will be tests 14-19 (6 tests)
+bundle exec flakey_spec_catcher --node-total=3 --node-index=2
+```
+
 ### Additional Git Detection Examples
 
 ```sh

data/bin/flakey_spec_catcher

@@ -9,4 +9,6 @@ if app.user_config.enable_runs
   app.show_settings
   app.rerun_preview
   exit app.run_specs
+elsif app.user_config.dry_run
+  app.show_test_list
 end

data/lib/flakey_spec_catcher/change_capsule.rb

@@ -76,7 +76,7 @@ module FlakeySpecCatcher
     end
 
     def line_matches_method_or_block(line)
-      return true if line =~ /\s*do(\s+|$)/ || line =~ /^\s*def\s+/
+      return true if line =~ /\s*do(\s+|$)/ || line =~ /^\s*def\s+/ || line =~ /^\s*if\s+/
 
       false
     end

data/lib/flakey_spec_catcher/cli_override.rb

@@ -7,14 +7,18 @@ module FlakeySpecCatcher
   #
   # Captures command line arguments for manual re-runs
   class CliOverride
-    attr_reader :rerun_patterns, :rerun_usage, :repeat_factor, :enable_runs, :excluded_tags, :use_parent
-    attr_reader :output_file
+    attr_reader :rerun_patterns, :rerun_usage, :repeat_factor, :enable_runs, :excluded_tags, :use_parent, :dry_run
+    attr_reader :output_file, :split_nodes, :split_index, :verbose, :test_options
 
     def initialize
+      @dry_run = false
       @enable_runs = true
       @excluded_tags = []
       @use_parent = false
+      @verbose = false
+      @test_options = []
       parse_command_line_args
+      validate_arguments
     end
 
     private
@@ -36,15 +40,20 @@ module FlakeySpecCatcher
         end
 
         opts.on('-u', '--usage=USAGE', 'Specify a re-run usage for the manual re-run') do |usage|
-          raise ArgumentError if @rerun_patterns.nil?
-
           @rerun_usage = remove_formatter_quotes(usage)
         end
 
+        opts.on('--node-total=SPLIT_NODES', 'Specify now many nodes the run is being split into') do |nodes|
+          @split_nodes = remove_formatter_quotes(nodes).to_i
+        end
+
+        opts.on('--node-index=SPLIT_INDEX', 'Specify the index this node represents in a split run') do |index|
+          @split_index = remove_formatter_quotes(index).to_i
+        end
+
         opts.on('-r', '--repeat=REPEAT_FACTOR',
                 'Specify a repeat factor for the manual re-run(s)') do |repeat|
-          parsed_repeat_factor = remove_formatter_quotes(repeat).to_i
-          @repeat_factor = parsed_repeat_factor if parsed_repeat_factor.positive?
+          @repeat_factor = remove_formatter_quotes(repeat).to_i
         end
 
         opts.on('-e', '--excluded-tags=EXCLUDED_TAGS',
@@ -62,12 +71,44 @@ module FlakeySpecCatcher
           @enable_runs = false
         end
 
+        opts.on('-d', '--dry-run', "Performs all setup but doesn't run any tests") do
+          @dry_run = true
+        end
+
+        opts.on('--dry-run-quiet', 'Prints list of tests to be run') do
+          @enable_runs = false
+          @dry_run = true
+        end
+
+        opts.on('--verbose', 'Send all output from running tests to stdout') do
+          @verbose = true
+        end
+
         opts.on('-h', '--help', 'Displays available flakey_spec_catcher cli overrides') do
           puts opts
           @enable_runs = false
         end
+
+        opts.on("--rspec-options '[OPTIONS]'", 'execute default usage with rspec options') do |arg|
+          @test_options = arg.split(/[ ](?=(?:[^"]*"[^"]*")*[^"]*$)(?=(?:[^']*'[^']*')*[^']*$)/)
+        end
       end.parse!
     end
+
+    # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+    def validate_arguments
+      if !@rerun_usage.nil? && @rerun_patterns.nil?
+        raise ArgumentError, 'rerun usage can only be specified with rerun patterns'
+      end
+      raise ArgumentError, 'repeat factor must be positive' if !@repeat_factor.nil? && @repeat_factor.negative?
+      raise ArgumentError, 'split index and split nodes must be specified together' if @split_nodes.nil? != @split_index.nil?
+
+      splitting = !@split_nodes.nil? && !@split_index.nil?
+      raise ArgumentError, 'split total must be positive' if splitting && @split_nodes.negative?
+      raise ArgumentError, 'split index must be positive' if splitting && @split_index.negative?
+      raise ArgumentError, 'split index must be less than split nodes' if splitting && @split_index >= @split_nodes
+    end
+    # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
     # rubocop:enable Metrics/AbcSize, Metrics/MethodLength, Metrics/BlockLength
 
     def remove_formatter_quotes(env_var)

data/lib/flakey_spec_catcher/git_controller.rb

@@ -26,6 +26,9 @@ module FlakeySpecCatcher
       @remote = find_git_remote
       @branch = find_remote_branch
       @capsule_manager = capsule_manager
+      # we don't need to do any git comparisons if we have manually specified tests to run.
+      return unless @user_config.manual_rerun_patterns.nil? || @user_config.manual_rerun_patterns.empty?
+
       initialize_git_comparison(test_mode)
       parse_changes
       identify_change_contexts
@@ -105,10 +108,12 @@ module FlakeySpecCatcher
       @git_comparison = "#{@base_commit_sha}..#{@working_commit_sha}"
     end
 
+    # rubocop:disable Metrics/AbcSize
     def parse_changes
       # For each file, get the change block
       diff_files.each do |filename|
         next unless filename =~ /_spec.rb/
+        next unless File.file?(filename)
 
         # Get diff pairs [ /path/to/file,  "@@ -19 +19,4 @@" ]
         diff = `git diff --unified=0 #{@git_comparison} -- #{filename}`
@@ -123,6 +128,7 @@ module FlakeySpecCatcher
         end
       end
     end
+    # rubocop:enable Metrics/AbcSize
 
     def identify_change_contexts
       @capsule_manager.change_capsules.each(&:fill_contexts)

data/lib/flakey_spec_catcher/rerun_manager.rb

@@ -17,6 +17,7 @@ module FlakeySpecCatcher
       @user_config = user_config
       @rerun_capsules = []
       determine_rerun_usage
+      split!
     end
 
     def determine_rerun_usage
@@ -110,6 +111,37 @@ module FlakeySpecCatcher
 
     private
 
+    # rubocop:disable Metrics/AbcSize
+    def split!
+      return unless !@user_config.split_nodes.nil? && !@user_config.split_index.nil?
+
+      # we create this restriction for simplicity. if we have multiple capsules then
+      # we have to split while being aware of splitting across capsules.
+      # if we absolutely need to split with different usage types, then we can implement that later..
+      raise ArgumentError('can only split on one usage type') if @rerun_capsules.size != 1
+
+      # ensure everything is sorted consistently before splitting
+      capsule = @rerun_capsules[0]
+      capsule.testcase.sort!
+
+      # compute the total tests that are changing
+      test_count = capsule.testcase.size
+
+      # the last node is going to be different from the previous nodes.
+      # for instance, if there are 13 test to run and you want 3 nodes:
+      # node1 -> 5
+      # node2 -> 5
+      # node3 -> 3
+      # it may be more efficient to split differently, but for now this will work.
+      node_test_count = (test_count / @user_config.split_nodes.to_f).ceil
+
+      skipping = @user_config.split_index * node_test_count
+      skipping_until = (@user_config.split_index + 1) * node_test_count
+      split_tests = capsule.testcase[skipping...skipping_until]
+      @rerun_capsules = [FlakeySpecCatcher::RerunCapsule.new(usage: capsule.usage, testcase: split_tests)]
+    end
+    # rubocop:enable Metrics/AbcSize
+
     def filter_reruns_by_ignore_files(reruns)
       return reruns if @user_config.ignore_files.count.zero?
 

data/lib/flakey_spec_catcher/runner.rb

@@ -10,7 +10,7 @@ require_relative './event_listener.rb'
 
 module FlakeySpecCatcher
   class Runner
-    attr_reader :user_config, :rerun_manager, :git_controller
+    attr_reader :user_config, :rerun_manager, :git_controller, :test_run_count
 
     def initialize(test_mode: false,
       user_config: FlakeySpecCatcher::UserConfig.new,
@@ -23,6 +23,8 @@ module FlakeySpecCatcher
       @user_config = user_config
       @rerun_manager = rerun_manager
       @rspec_result_manager = result_manager
+      @test_run_count = 0
+      @temp_output_file = @user_config.output_file + 'temp' unless @user_config.output_file == File::NULL
     end
 
     # Debug Methods
@@ -34,8 +36,10 @@ module FlakeySpecCatcher
       puts "  Current Sha: #{@git_controller.working_commit_sha}"
       puts "  Base Sha: #{@git_controller.base_commit_sha}"
       puts "  Repeat factor: #{@user_config.repeat_factor}"
+      puts "  Node Total: #{@user_config.split_nodes}" if @user_config.split_nodes
+      puts "  Node Index: #{@user_config.split_index}" if @user_config.split_index
       puts "  Changed Specs Detected: #{@git_controller.changed_examples}"
-      return if @user_config.output_file == '/dev/null'
+      return if @user_config.output_file == File::NULL
 
       puts "  Verbose Output Path: #{@user_config.output_file}"
     end
@@ -55,7 +59,17 @@ module FlakeySpecCatcher
     end
     # end Debug methods
 
+    def show_test_list
+      @rerun_manager.rerun_capsules.each do |capsule|
+        capsule.testcase.each do |test|
+          puts test
+        end
+      end
+    end
+
     def run_specs
+      return 0 if @user_config.dry_run
+
       status = 0
       @rerun_manager.rerun_capsules.sort.each do |capsule|
         @user_config.repeat_factor.times do
@@ -65,6 +79,7 @@ module FlakeySpecCatcher
       end
 
       display_results(status)
+      copy_to_user_output unless @user_config.output_file == File::NULL
 
       # Always return 0 if silent_mode is enabled
       @user_config.silent_mode ? 0 : status
@@ -80,21 +95,29 @@ module FlakeySpecCatcher
     def invoke_rspec_runner(test)
       configure_listener
       # Pass in CLI options to suppress normal output, and only run the specified test
-      rspec_args = ['--out', @user_config.output_file, test]
-      return_status = RSpec::Core::Runner.run(rspec_args)
+      rspec_args = ['--format', 'documentation', '--out', @user_config.output_file, test]
+      # Rspec output sent to stdout if verbose option is true
+      rspec_args << '-fd' if @user_config.verbose
+      return_status = RSpec::Core::Runner.run(@user_config.test_options.concat(rspec_args))
       RSpec.clear_examples
+      copy_output_to_temp_file unless @user_config.output_file == File::NULL
       return_status
     end
 
     def invoke_custom_rspec_runner(usage, testcase)
-      custom_usage_output = `#{usage} #{testcase.join(' ')}`
+      if @user_config.verbose
+        $stdout << custom_usage_output = `#{usage} #{testcase.join(' ')}`
+      else
+        custom_usage_output = `#{usage} #{testcase.join(' ')}`
+      end
 
-      File.open(@user_config.output_file, 'a') { |f| f.puts custom_usage_output } if @user_config.output_file != '/dev/null'
+      File.open(@user_config.output_file, 'a') { |f| f.puts custom_usage_output } if @user_config.output_file != File::NULL
 
       $?.exitstatus # rubocop:disable Style/SpecialGlobalVars
     end
 
     def handle_capsule_rerun(capsule)
+      @test_run_count += 1
       if capsule.default_usage?
         invoke_rspec_runner(capsule.testcase)
       else
@@ -120,5 +143,18 @@ module FlakeySpecCatcher
                                      :example_failed, :example_passed
       end
     end
+
+    def copy_output_to_temp_file
+      # copy contents of output file, it will get overwritten in RSpec::Core::Runner
+      File.open(@temp_output_file, 'a') do |f|
+        f.puts IO.readlines(@user_config.output_file)
+      end
+    end
+
+    def copy_to_user_output
+      # copy all appended output to original output file, delete the temp output file
+      IO.copy_stream(@temp_output_file, @user_config.output_file) if File.exist?(@temp_output_file)
+      File.delete(@temp_output_file) if File.exist?(@temp_output_file)
+    end
   end
 end

data/lib/flakey_spec_catcher/user_config.rb

@@ -10,7 +10,8 @@ module FlakeySpecCatcher
     attr_reader :repeat_factor, :ignore_files, :ignore_branches, :silent_mode
     attr_reader :rerun_file_only, :rspec_usage_patterns, :excluded_tags
     attr_reader :manual_rerun_patterns, :manual_rerun_usage
-    attr_reader :enable_runs, :output_file, :use_parent
+    attr_reader :enable_runs, :output_file, :use_parent, :dry_run
+    attr_reader :split_nodes, :split_index, :verbose, :test_options
 
     USER_CONFIG_ENV_VARS = %w[FSC_REPEAT_FACTOR FSC_IGNORE_FILES FSC_IGNORE_BRANCHES
                               FSC_SILENT_MODE FSC_RERUN_FILE_ONLY FSC_USAGE_PATTERNS
@@ -34,14 +35,8 @@ module FlakeySpecCatcher
       @rspec_usage_patterns = env_var_string_to_pairs(ENV['FSC_USAGE_PATTERNS'])
       @excluded_tags = env_var_string_to_tags(ENV['FSC_EXCLUDED_TAGS'])
       @output_file = ENV['FSC_OUTPUT_FILE']
-    end
-    # rubocop:enable Metrics/AbcSize
-
-    def override_settings
-      apply_cli_override
-      return unless @manual_rerun_patterns.nil?
-
-      parse_commit_message
+      @split_nodes = env_var_string_to_int_or_nil('FSC_NODE_TOTAL')
+      @split_index = env_var_string_to_int_or_nil('FSC_NODE_INDEX')
     end
 
     def apply_cli_override
@@ -50,19 +45,32 @@ module FlakeySpecCatcher
       @use_parent = @cli_override.use_parent
       @repeat_factor = @cli_override.repeat_factor if @cli_override.repeat_factor.to_i.positive?
       @enable_runs = @cli_override.enable_runs
+      @dry_run = @cli_override.dry_run
+      @split_nodes = @cli_override.split_nodes unless @cli_override.split_nodes.nil?
+      @split_index = @cli_override.split_index unless @cli_override.split_index.nil?
       @excluded_tags = if @cli_override.excluded_tags.empty?
         @excluded_tags
       else
         @cli_override.excluded_tags
       end
       @output_file = set_output_file
+      @verbose = @cli_override.verbose
+      @test_options = @cli_override.test_options
+    end
+    # rubocop:enable Metrics/AbcSize
+
+    def override_settings
+      apply_cli_override
+      return unless @manual_rerun_patterns.nil?
+
+      parse_commit_message
     end
 
     def set_output_file
       if !@cli_override.output_file.nil?
         @cli_override.output_file
       elsif @output_file.nil? || @output_file.strip.empty?
-        '/dev/null'
+        File::NULL
       else
         @output_file
       end
@@ -76,6 +84,18 @@ module FlakeySpecCatcher
       env_var.to_i.positive? ? env_var.to_i : 20
     end
 
+    def env_var_string_to_int_or_nil(env_var_name)
+      env_var = ENV[env_var_name]
+      if env_var.nil? || env_var.empty?
+        nil
+      else
+        check = env_var.to_i
+        raise ArgumentError, "#{env_var_name} must be positive" if check.negative?
+
+        check
+      end
+    end
+
     def env_var_string_to_array(env_var)
       if env_var.nil? || env_var.empty?
         []

data/lib/flakey_spec_catcher/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FlakeySpecCatcher
-  VERSION = '0.9.2'
+  VERSION = '0.9.8'
 end

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: flakey_spec_catcher
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 0.9.8
 platform: ruby
 authors:
 - Brian Watson
 - Mikey Hargiss
 - Ben Nelson
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-02-18 00:00:00.000000000 Z
+date: 2020-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -68,7 +68,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.17'
-description: 
+description:
 email:
 - bwatson@instructure.com
 - mhargiss@instructure.com
@@ -102,12 +102,12 @@ files:
 - lib/flakey_spec_catcher/version.rb
 - lib/helpers/colorize.rb
 - lib/helpers/indent_string.rb
-homepage: 
+homepage:
 licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -122,8 +122,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
-signing_key: 
+rubygems_version: 3.0.3
+signing_key:
 specification_version: 4
 summary: Run new or changed specs many times to prevent unreliable specs
 test_files: []