forking_test_runner 1.4.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 24a22080e7feec720e83c73e48241d6984b0c7d769cce74f7687f563f1b1e75d
-  data.tar.gz: 2a0b5a4bb46cad79e4f7de3cb9b37bdd1307d7b9754b27397686349278be9ae6
+  metadata.gz: fc4844e1ccea576d0e7992efedab19587cb803ec04df7db7f82061253bada8fe
+  data.tar.gz: 2a8a12c8446e35c410417ac5271cd401d525d508a9b0db2c1c0674bd81d9275f
 SHA512:
-  metadata.gz: ca2e09cb43119ea4c62260fa8487d7c93e1b3a935a2dbbc1c5a219166c7891cdec55f8b95cf30610f83319334efb17739e505f9132b4dbdc6e7b78e59346e3fb
-  data.tar.gz: 3385e2157a115e09810ff55cb50d5bb24e07be53effc1bf4b7362b434215ed6c4111ce26a379955dfbb540bf27a7e2ab954fff97080e8c6d9cc782e0ea8a222e
+  metadata.gz: 67e48c4c7e3ac01664bfb0b21bf8339875a1fed6b8a40a75f546cd849ad165df3346afaf641035317ff0be8d553ab745f2cc7e14fb3d747c5849db01d5bab1d9
+  data.tar.gz: 59c63fcf19c1a472804504987d31bfe01e85805453db87104068279227a7b83391e9cc74323f412033d1fbe83a5a9b4304805b249f275539ba2367a3478e2460

data/lib/forking_test_runner.rb

@@ -1,130 +1,74 @@
 require 'benchmark'
 require 'optparse'
 require 'forking_test_runner/version'
+require 'forking_test_runner/coverage_capture'
+require 'forking_test_runner/cli'
+require 'parallel'
+require 'tempfile'
 
 module ForkingTestRunner
   CLEAR = "------"
 
-  module CoverageCapture
-    def capture_coverage!
-      @capture_coverage = peek_result.dup
-    end
-
-    # override to add pre-fork captured coverage when someone asks for the results
-    def result
-      original = super
-      return original unless @capture_coverage
-      CoverageCapture.merge_coverage(original, @capture_coverage)
-    end
-
-    class << self
-      def merge_coverage(a, b)
-        merged = a.dup
-        b.each do |file, coverage|
-          orig = merged[file]
-          merged[file] = if orig
-            if coverage.is_a?(Array)
-              merge_lines_coverage(orig, coverage)
-            else
-              {
-                lines: merge_lines_coverage(orig.fetch(:lines), coverage.fetch(:lines)),
-                branches: merge_branches_coverage(orig.fetch(:branches), coverage.fetch(:branches))
-              }
-            end
-          else
-            coverage
-          end
-        end
-        merged
-      end
-
-      private
-
-      # assuming b has same or more keys since it comes from a fork
-      # [nil,1,0] + [nil,nil,2] -> [nil,1,2]
-      def merge_lines_coverage(a, b)
-        b.each_with_index.map do |b_count, i|
-          a_count = a[i]
-          (a_count.nil? && b_count.nil?) ? nil : a_count.to_i + b_count.to_i
-        end
-      end
-
-      # assuming b has same or more keys since it comes from a fork
-      # {foo: {bar: 0, baz: 1}} + {foo: {bar: 1, baz: 0}} -> {foo: {bar: 1, baz: 1}}
-      def merge_branches_coverage(a, b)
-        b.each_with_object({}) do |(branch, v), all|
-          vb = v.dup
-          if part = a[branch]
-            part.each do |nested, a_count|
-              vb[nested] = a_count + vb[nested].to_i
-            end
-          end
-          all[branch] = vb
-        end
-      end
-    end
-  end
-
   class << self
     def cli(argv)
-      @options, tests = parse_options(argv)
-
-      disable_test_autorun
-
-      load_test_env(@options.fetch(:helper))
+      @options, tests = CLI.parse_options(argv)
 
       # figure out what we need to run
       runtime_log = @options.fetch(:runtime_log)
-      group, group_count = find_group_args
-      tests = find_tests_for_group(group, group_count, tests, runtime_log)
+      groups, group_count = find_group_args
+      parallel = @options.fetch(:parallel)
+      test_groups =
+        if parallel && !@options.fetch(:group)
+          Array.new(parallel) { |i| find_tests_for_group(i + 1, parallel, tests, runtime_log) }
+        else
+          groups.map { |group| find_tests_for_group(group, group_count, tests, runtime_log) }
+        end
 
+      # say what we are running
+      all_tests = test_groups.flatten(1)
       if @options.fetch(:quiet)
-        puts "Running #{tests.size} test files"
+        puts "Running #{all_tests.size} test files"
       else
-        puts "Running tests #{tests.map(&:first).join(" ")}"
+        puts "Running tests #{all_tests.map(&:first).join(" ")}"
       end
 
-      if ar?
-        preload_fixtures
-        ActiveRecord::Base.connection.disconnect!
-      end
-
-      Coverage.capture_coverage! if @options.fetch(:merge_coverage)
-
       # run all the tests
-      results = tests.map do |file, expected|
-        puts "#{CLEAR} >>> #{file} "
-        time, success, output = benchmark { run_test(file) }
-
-        puts output if !success && @options.fetch(:quiet)
-
-        if runtime_log && !@options.fetch(:quiet)
-          puts "Time: expected #{expected.round(2)}, actual #{time.round(2)}"
-        end
+      results = with_lock do |lock|
+        Parallel.map_with_index(test_groups, in_processes: parallel || 0) do |tests, env_index|
+          if parallel
+            ENV["TEST_ENV_NUMBER"] = (env_index == 0 ? '' : (env_index + 1).to_s) # NOTE: does not support first_is_1 option
+          end
 
-        if !success || !@options.fetch(:quiet)
-          puts "#{CLEAR} <<< #{file} ---- #{success ? "OK" : "Failed"}"
-        end
+          reraise_clean_ar_error { load_test_env }
 
-        [file, time, expected, output, success]
+          tests.map do |file, expected|
+            print_started file unless parallel
+            result = [file, expected, *benchmark { run_test(file) }]
+            sync_stdout lock do
+              print_started file if parallel
+              print_finished *result
+            end
+            result
+          end
+        end.flatten(1)
       end
 
       unless @options.fetch(:quiet)
         # pretty print the results
         puts "\nResults:"
         puts results.
-          sort_by { |_,_,_,_,r| r ? 0 : 1 }. # failures should be last so they are easy to find
-          map { |f,_,_,_,r| "#{f}: #{r ? "OK" : "Fail"}"}
+          sort_by { |_,_,_,r,_| r ? 0 : 1 }. # failures should be last so they are easy to find
+          map { |f,_,_,r,_| "#{f}: #{r ? "OK" : "Fail"}"}
         puts
       end
 
-      success = results.map(&:last).all?
+      success = results.map { |r| r[3] }.all?
 
-      puts colorize(success, summarize_results(results.map { |r| r[3] }))
+      puts colorize(success, summarize_results(results.map { |r| r[4] }))
 
       if runtime_log
         # show how long they ran vs expected
-        diff = results.map { |_,time,expected| time - expected }.inject(:+).to_f
+        diff = results.map { |_, expected, time| time - expected }.inject(:+).to_f
         puts "Time: #{diff.round(2)} diff to expected"
       end
 
@@ -140,6 +84,38 @@ module ForkingTestRunner
 
     private
 
+    def with_lock(&block)
+      return yield unless @options.fetch(:parallel)
+      Tempfile.open"forking-test-runner-lock", &block
+    end
+
+    def sync_stdout(lock)
+      return yield unless @options.fetch(:parallel)
+      begin
+        lock.flock(File::LOCK_EX)
+        yield
+      ensure
+        lock.flock(File::LOCK_UN)
+      end
+    end
+
+    def print_started(file)
+      puts "#{CLEAR} >>> #{file}"
+    end
+
+    def print_finished(file, expected, time, success, stdout)
+      # print stdout if it was not shown before, but needs to be shown
+      puts stdout if (!success && @options.fetch(:quiet)) || (@options.fetch(:parallel) && !@options.fetch(:quiet))
+
+      if @options.fetch(:runtime_log) && !@options.fetch(:quiet)
+        puts "Time: expected #{expected.round(2)}, actual #{time.round(2)}"
+      end
+
+      if !success || !@options.fetch(:quiet)
+        puts "#{CLEAR} <<< #{file} ---- #{success ? "OK" : "Failed"}"
+      end
+    end
+
     def colorize(green, string)
       if $stdout.tty?
         "\e[#{green ? 32 : 31}m#{string}\e[0m"
@@ -162,15 +138,13 @@ module ForkingTestRunner
 
     def benchmark
       result = false
-      time = Benchmark.realtime do
-        result = yield
-      end
-      return [time, result].flatten
+      time = Benchmark.realtime { result = yield }
+      [time, *result]
     end
 
     # log runtime via dumping or curling it into the runtime log location
     def record_test_runtime(mode, results, log)
-      data = results.map { |test, time| "#{test}:#{time.round(2)}" }.join("\n") << "\n"
+      data = results.map { |test, _, time| "#{test}:#{time.round(2)}" }.join("\n") << "\n"
 
       case mode
       when 'simple'
@@ -198,21 +172,47 @@ module ForkingTestRunner
     end
 
     def find_group_args
-      if @options.fetch(:group) && @options.fetch(:groups)
+      group = @options.fetch(:group)
+      groups = @options.fetch(:groups)
+      if group && groups
         # delete options we want while leaving others as they are (-v / --seed etc)
-        group = @options.fetch(:group)
-        group_count = @options.fetch(:groups)
+        [group.split(",").map { |g| Integer(g) }, groups]
       else
-        group = 1
-        group_count = 1
+        [[1], 1]
+      end
+    end
+
+    def load_test_env
+      CoverageCapture.activate! if @options.fetch(:merge_coverage)
+
+      load_test_helper
+
+      if active_record?
+        preload_fixtures
+        ActiveRecord::Base.connection.disconnect!
       end
 
-      [group, group_count]
+      CoverageCapture.capture! if @options.fetch(:merge_coverage)
     end
 
-    def load_test_env(helper=nil)
+    def reraise_clean_ar_error
+      return yield unless @options.fetch(:parallel)
+
+      e = begin
+        yield
+        nil
+      rescue
+        $!
+      end
+
+      # needs to be done outside of the rescue block to avoid inheriting the cause
+      raise RuntimeError, "Re-raised error from test helper: #{e.message}", e.backtrace if e
+    end
+
+    def load_test_helper
+      disable_test_autorun
       require 'rspec/core' if @options.fetch(:rspec)
-      helper = helper || (@options.fetch(:rspec) ? "spec/spec_helper" : "test/test_helper")
+      helper =  @options.fetch(:helper) || (@options.fetch(:rspec) ? "spec/spec_helper" : "test/test_helper")
       require "./#{helper}"
     end
 
@@ -221,9 +221,8 @@ module ForkingTestRunner
     def preload_fixtures
       return if @options.fetch(:no_fixtures)
 
-      fixtures = (ActiveSupport::VERSION::MAJOR == 3 ? ActiveRecord::Fixtures : ActiveRecord::FixtureSet)
-
       # reuse our pre-loaded fixtures even if we have a different connection
+      fixtures = ActiveRecord::FixtureSet
       fixtures_eigenclass = class << fixtures; self; end
       fixtures_eigenclass.send(:define_method, :cache_for_connection) do |_connection|
         fixtures.class_variable_get(:@@all_cached_fixtures)[:unique]
@@ -247,7 +246,7 @@ module ForkingTestRunner
       toggle_test_autorun true, file
     end
 
-    def fork_with_captured_output(tee_to_stdout)
+    def fork_with_captured_stdout
       rpipe, wpipe = IO.pipe
 
       child = fork do
@@ -263,7 +262,7 @@ module ForkingTestRunner
 
       while ch = rpipe.read(1)
         buffer << ch
-        $stdout.write(ch) if tee_to_stdout
+        $stdout.write(ch) if !@options.fetch(:quiet) && !@options.fetch(:parallel) # tee
       end
 
       Process.wait(child)
@@ -271,10 +270,10 @@ module ForkingTestRunner
     end
 
     def run_test(file)
-      output = change_program_name_to file do
-        fork_with_captured_output(!@options.fetch(:quiet)) do
+      stdout = change_program_name_to file do
+        fork_with_captured_stdout do
           SimpleCov.pid = Process.pid if defined?(SimpleCov) && SimpleCov.respond_to?(:pid=) # trick simplecov into reporting in this fork
-          if ar?
+          if active_record?
             key = (ActiveRecord::VERSION::STRING >= "4.1.0" ? :test : "test")
             ActiveRecord::Base.establish_connection key
           end
@@ -282,14 +281,17 @@ module ForkingTestRunner
         end
       end
 
-      [$?.success?, output]
+      [$?.success?, stdout]
     end
 
     def change_program_name_to(name)
-      old, $0 = $0, name
-      yield
-    ensure
-      $0 = old
+      return yield if @options.fetch(:parallel)
+      begin
+        old, $0 = $0, name
+        yield
+      ensure
+        $0 = old
+      end
     end
 
     def find_tests_for_group(group, group_count, tests, runtime_log)
@@ -310,21 +312,15 @@ module ForkingTestRunner
       group.map { |test| [test, (tests[test] if group_by == :runtime)] }
     end
 
-    def ar?
+    def active_record?
       !@options.fetch(:no_ar) && defined?(ActiveRecord::Base)
     end
 
     def minitest_class
       @minitest_class ||= begin
         require 'bundler/setup'
-        gem 'minitest'
-        if Gem.loaded_specs["minitest"].version.segments.first == 4 # 4.x
-          require 'minitest/unit'
-          MiniTest::Unit
-        else
-          require 'minitest'
-          Minitest
-        end
+        require 'minitest'
+        Minitest
       end
     end
 
@@ -348,85 +344,5 @@ module ForkingTestRunner
         end
       end
     end
-
-    # we remove the args we understand and leave the rest alone
-    # so minitest / rspec can read their own options (--seed / -v ...)
-    #  - keep our options clear / unambiguous to avoid overriding
-    #  - read all serial non-flag arguments as tests and leave only unknown options behind
-    #  - use .fetch everywhere to make sure nothing is misspelled
-    # GOOD: test --ours --theirs
-    # OK: --ours test --theirs
-    # BAD: --theirs test --ours
-    def parse_options(argv)
-      arguments = [
-        [:rspec, "--rspec", "RSpec mode"],
-        [:helper, "--helper", "Helper file to load before tests start", String],
-        [:quiet, "--quiet", "Quiet"],
-        [:no_fixtures, "--no-fixtures", "Do not load fixtures"],
-        [:no_ar, "--no-ar", "Disable ActiveRecord logic"],
-        [:merge_coverage, "--merge-coverage", "Merge base code coverage into indvidual files coverage, great for SingleCov"],
-        [
-          :record_runtime,
-          "--record-runtime=MODE",
-          "\n      Record test runtime:\n" <<
-          "        simple = write to disk at --runtime-log)\n" <<
-          "        amend  = write from multiple remote workers via http://github.com/grosser/amend, needs TRAVIS_REPO_SLUG & TRAVIS_BUILD_NUMBER",
-          String
-        ],
-        [:runtime_log, "--runtime-log=FILE", "File to store runtime log in or runtime.log", String],
-        [:group, "--group=NUM", "What group this is (use with --groups / starts at 1)", Integer],
-        [:groups, "--groups=NUM", "How many groups there are in total (use with --group)", Integer],
-        [:version, "--version", "Show version"],
-        [:help, "--help", "Show help"]
-      ]
-
-      options = arguments.each_with_object({}) do |(setting, flag, _, type), all|
-        all[setting] = delete_argv(flag.split('=', 2)[0], argv, type: type)
-      end
-
-      # show version
-      if options.fetch(:version)
-        puts VERSION
-        exit 0
-      end
-
-      # # show help
-      if options[:help]
-        parser = OptionParser.new("forking-test-runner folder [options]", 32, '') do |opts|
-          arguments.each do |_, flag, desc, type|
-            opts.on(flag, desc, type)
-          end
-        end
-        puts parser
-        exit 0
-      end
-
-      # check if we can use merge_coverage
-      if options.fetch(:merge_coverage)
-        abort "merge_coverage does not work on ruby prior to 2.3" if RUBY_VERSION < "2.3.0"
-        require 'coverage'
-        klass = (class << Coverage; self; end)
-        klass.prepend CoverageCapture
-      end
-
-      # all remaining non-flag options until the next flag must be tests
-      next_flag = argv.index { |arg| arg.start_with?("-") } || argv.size
-      tests = argv.slice!(0, next_flag)
-      abort "No tests or folders found in arguments" if tests.empty?
-      tests.each { |t| abort "Unable to find #{t}" unless File.exist?(t) }
-
-      [options, tests]
-    end
-
-    def delete_argv(name, argv, type: nil)
-      return unless index = argv.index(name)
-      argv.delete_at(index)
-      if type
-        found = argv.delete_at(index) || raise("Missing argument for #{name}")
-        send(type.name, found) # case found
-      else
-        true
-      end
-    end
   end
 end

data/lib/forking_test_runner/cli.rb

@@ -0,0 +1,94 @@
+module ForkingTestRunner
+  # read and delete options we support and pass the rest through to the underlying test runner (-v / --seed etc)
+  module CLI
+    OPTIONS = [
+      [:rspec, "--rspec", "RSpec mode"],
+      [:helper, "--helper", "Helper file to load before tests start", String],
+      [:quiet, "--quiet", "Quiet"],
+      [:no_fixtures, "--no-fixtures", "Do not load fixtures"],
+      [:no_ar, "--no-ar", "Disable ActiveRecord logic"],
+      [:merge_coverage, "--merge-coverage", "Merge base code coverage into indvidual files coverage, great for SingleCov"],
+      [
+        :record_runtime,
+        "--record-runtime=MODE",
+        "\n      Record test runtime:\n" <<
+          "        simple = write to disk at --runtime-log)\n" <<
+          "        amend  = write from multiple remote workers via http://github.com/grosser/amend, needs TRAVIS_REPO_SLUG & TRAVIS_BUILD_NUMBER",
+        String
+      ],
+      [:runtime_log, "--runtime-log=FILE", "File to store runtime log in or runtime.log", String],
+      [:parallel, "--parallel=NUM", "Number of parallel groups to run at once", Integer],
+      [:group, "--group=NUM[,NUM]", "What group this is (use with --groups / starts at 1)", String],
+      [:groups, "--groups=NUM", "How many groups there are in total (use with --group)", Integer],
+      [:version, "--version", "Show version"],
+      [:help, "--help", "Show help"]
+    ]
+
+    class << self
+      def parse_options(argv)
+        options = OPTIONS.each_with_object({}) do |(setting, flag, _, type), all|
+          all[setting] = delete_argv(flag.split('=', 2)[0], argv, type: type)
+        end
+
+        # show version
+        if options.fetch(:version)
+          puts VERSION
+          exit 0
+        end
+
+        # show help
+        if options[:help]
+          puts help
+          exit 0
+        end
+
+        # check if we can use merge_coverage
+        if options.fetch(:merge_coverage)
+          abort "merge_coverage does not work on ruby prior to 2.3" if RUBY_VERSION < "2.3.0"
+        end
+
+        if !!options.fetch(:group) ^ !!options.fetch(:groups)
+          abort "use --group and --groups together"
+        end
+
+        # all remaining non-flag options until the next flag must be tests
+        next_flag = argv.index { |arg| arg.start_with?("-") } || argv.size
+        tests = argv.slice!(0, next_flag)
+        abort "No tests or folders found in arguments" if tests.empty?
+        tests.each { |t| abort "Unable to find #{t}" unless File.exist?(t) }
+
+        [options, tests]
+      end
+
+      private
+
+      # fake parser that will print nicely
+      def help
+        OptionParser.new("forking-test-runner folder [options]", 32, '') do |opts|
+          OPTIONS.each do |_, flag, desc, type|
+            opts.on(flag, desc, type)
+          end
+        end
+      end
+
+      # we remove the args we understand and leave the rest alone
+      # so minitest / rspec can read their own options (--seed / -v ...)
+      #  - keep our options clear / unambiguous to avoid overriding
+      #  - read all serial non-flag arguments as tests and leave only unknown options behind
+      #  - use .fetch everywhere to make sure nothing is misspelled
+      # GOOD: test --ours --theirs
+      # OK: --ours test --theirs
+      # BAD: --theirs test --ours
+      def delete_argv(name, argv, type: nil)
+        return unless index = argv.index(name)
+        argv.delete_at(index)
+        if type
+          found = argv.delete_at(index) || raise("Missing argument for #{name}")
+          send(type.name, found) # case found
+        else
+          true
+        end
+      end
+    end
+  end
+end

data/lib/forking_test_runner/coverage_capture.rb

@@ -0,0 +1,71 @@
+module ForkingTestRunner
+  module CoverageCapture
+    # override Coverage.result to add pre-fork captured coverage
+    def result
+      return super unless captured = CoverageCapture.coverage
+      CoverageCapture.merge_coverage(super, captured)
+    end
+
+    # deprecated, single_cov checks for this, so leave it here
+    def capture_coverage!
+    end
+
+    class << self
+      attr_accessor :coverage
+
+      def activate!
+        require 'coverage'
+        (class << Coverage; self; end).prepend self
+      end
+
+      def capture!
+        self.coverage = Coverage.peek_result.dup
+      end
+
+      def merge_coverage(a, b)
+        merged = a.dup
+        b.each do |file, coverage|
+          orig = merged[file]
+          merged[file] = if orig
+            if coverage.is_a?(Array)
+              merge_lines_coverage(orig, coverage)
+            else
+              {
+                lines: merge_lines_coverage(orig.fetch(:lines), coverage.fetch(:lines)),
+                branches: merge_branches_coverage(orig.fetch(:branches), coverage.fetch(:branches))
+              }
+            end
+          else
+            coverage
+          end
+        end
+        merged
+      end
+
+      private
+
+      # assuming b has same or more keys since it comes from a fork
+      # [nil,1,0] + [nil,nil,2] -> [nil,1,2]
+      def merge_lines_coverage(a, b)
+        b.each_with_index.map do |b_count, i|
+          a_count = a[i]
+          (a_count.nil? && b_count.nil?) ? nil : a_count.to_i + b_count.to_i
+        end
+      end
+
+      # assuming b has same or more keys since it comes from a fork
+      # {foo: {bar: 0, baz: 1}} + {foo: {bar: 1, baz: 0}} -> {foo: {bar: 1, baz: 1}}
+      def merge_branches_coverage(a, b)
+        b.each_with_object({}) do |(branch, v), all|
+          vb = v.dup
+          if part = a[branch]
+            part.each do |nested, a_count|
+              vb[nested] = a_count + vb[nested].to_i
+            end
+          end
+          all[branch] = vb
+        end
+      end
+    end
+  end
+end

data/lib/forking_test_runner/version.rb

@@ -1,3 +1,3 @@
 module ForkingTestRunner
-  VERSION = "1.4.0"
+  VERSION = "1.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: forking_test_runner
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-02-25 00:00:00.000000000 Z
+date: 2019-09-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: parallel_tests
@@ -118,6 +118,8 @@ files:
 - MIT-LICENSE
 - bin/forking-test-runner
 - lib/forking_test_runner.rb
+- lib/forking_test_runner/cli.rb
+- lib/forking_test_runner/coverage_capture.rb
 - lib/forking_test_runner/version.rb
 homepage: https://github.com/grosser/forking_test_runner
 licenses:
@@ -131,15 +133,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.0.0
+      version: 2.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Run every test in a fork to avoid pollution and get clean output per test