parallel_tests 3.3.0 → 4.2.0

data/lib/parallel_tests/test/runner.rb

@@ -1,8 +1,12 @@
+# frozen_string_literal: true
+require 'shellwords'
 require 'parallel_tests'
 
 module ParallelTests
   module Test
     class Runner
+      RuntimeLogTooSmallError = Class.new(StandardError)
+
       class << self
         # --- usually overwritten by other runners
 
@@ -14,13 +18,24 @@ module ParallelTests
           /_(test|spec).rb$/
         end
 
+        def default_test_folder
+          "test"
+        end
+
         def test_file_name
           "test"
         end
 
         def run_tests(test_files, process_number, num_processes, options)
-          require_list = test_files.map { |file| file.sub(" ", "\\ ") }.join(" ")
-          cmd = "#{executable} -Itest -e '%w[#{require_list}].each { |f| require %{./\#{f}} }' -- #{options[:test_options]}"
+          require_list = test_files.map { |file| file.gsub(" ", "\\ ") }.join(" ")
+          cmd = [
+            *executable,
+            '-Itest',
+            '-e',
+            "%w[#{require_list}].each { |f| require %{./\#{f}} }",
+            '--',
+            *options[:test_options]
+          ]
           execute_command(cmd, process_number, num_processes, options)
         end
 
@@ -32,7 +47,7 @@ module ParallelTests
         # --- usually used by other runners
 
         # finds all tests and partitions them into groups
-        def tests_in_groups(tests, num_groups, options={})
+        def tests_in_groups(tests, num_groups, options = {})
           tests = tests_with_size(tests, options)
           Grouper.in_even_groups_by_size(tests, num_groups, options)
         end
@@ -46,12 +61,19 @@ module ParallelTests
           when :filesize
             sort_by_filesize(tests)
           when :runtime
-            sort_by_runtime(tests, runtimes(tests, options), options.merge(allowed_missing: (options[:allowed_missing_percent] || 50) / 100.0))
+            sort_by_runtime(
+              tests, runtimes(tests, options),
+              options.merge(allowed_missing: (options[:allowed_missing_percent] || 50) / 100.0)
+            )
           when nil
             # use recorded test runtime if we got enough data
-            runtimes = runtimes(tests, options) rescue []
+            runtimes = begin
+              runtimes(tests, options)
+            rescue StandardError
+              []
+            end
             if runtimes.size * 1.5 > tests.size
-              puts "Using recorded test runtime"
+              puts "Using recorded test runtime" unless options[:quiet]
               sort_by_runtime(tests, runtimes)
             else
               sort_by_filesize(tests)
@@ -64,35 +86,44 @@ module ParallelTests
         end
 
         def execute_command(cmd, process_number, num_processes, options)
+          number = test_env_number(process_number, options).to_s
           env = (options[:env] || {}).merge(
-            "TEST_ENV_NUMBER" => test_env_number(process_number, options).to_s,
+            "TEST_ENV_NUMBER" => number,
             "PARALLEL_TEST_GROUPS" => num_processes.to_s,
-            "PARALLEL_PID_FILE" => ParallelTests.pid_file_path,
+            "PARALLEL_PID_FILE" => ParallelTests.pid_file_path
           )
-          cmd = "nice #{cmd}" if options[:nice]
-          cmd = "#{cmd} 2>&1" if options[:combine_stderr]
+          cmd = ["nice", *cmd] if options[:nice]
 
-          puts cmd if report_process_command?(options) && !options[:serialize_stdout]
+          # being able to run with for example `-output foo-$TEST_ENV_NUMBER` worked originally and is convenient
+          cmd.map! { |c| c.gsub("$TEST_ENV_NUMBER", number).gsub("${TEST_ENV_NUMBER}", number) }
+
+          print_command(cmd, env) if report_process_command?(options) && !options[:serialize_stdout]
 
           execute_command_and_capture_output(env, cmd, options)
         end
 
+        def print_command(command, env)
+          env_str = ['TEST_ENV_NUMBER', 'PARALLEL_TEST_GROUPS'].map { |e| "#{e}=#{env[e]}" }.join(' ')
+          puts [env_str, Shellwords.shelljoin(command)].compact.join(' ')
+        end
+
         def execute_command_and_capture_output(env, cmd, options)
+          popen_options = {} # do not add `pgroup: true`, it will break `binding.irb` inside the test
+          popen_options[:err] = [:child, :out] if options[:combine_stderr]
+
           pid = nil
-          output = IO.popen(env, cmd) do |io|
+          output = IO.popen(env, cmd, popen_options) do |io|
             pid = io.pid
             ParallelTests.pids.add(pid)
             capture_output(io, env, options)
           end
           ParallelTests.pids.delete(pid) if pid
           exitstatus = $?.exitstatus
-          seed = output[/seed (\d+)/,1]
+          seed = output[/seed (\d+)/, 1]
 
-          if report_process_command?(options) && options[:serialize_stdout]
-            output = [cmd, output].join("\n")
-          end
+          output = "#{Shellwords.shelljoin(cmd)}\n#{output}" if report_process_command?(options) && options[:serialize_stdout]
 
-          {:stdout => output, :exit_status => exitstatus, :command => cmd, :seed => seed}
+          { env: env, stdout: output, exit_status: exitstatus, command: cmd, seed: seed }
         end
 
         def find_results(test_output)
@@ -104,7 +135,7 @@ module ParallelTests
           end.compact
         end
 
-        def test_env_number(process_number, options={})
+        def test_env_number(process_number, options = {})
           if process_number == 0 && !options[:first_is_1]
             ''
           else
@@ -114,39 +145,42 @@ module ParallelTests
 
         def summarize_results(results)
           sums = sum_up_results(results)
-          sums.sort.map{|word, number|  "#{number} #{word}#{'s' if number != 1}" }.join(', ')
+          sums.sort.map { |word, number| "#{number} #{word}#{'s' if number != 1}" }.join(', ')
         end
 
         # remove old seed and add new seed
         def command_with_seed(cmd, seed)
-          clean = cmd.sub(/\s--seed\s+\d+\b/, '')
-          "#{clean} --seed #{seed}"
+          clean = remove_command_arguments(cmd, '--seed')
+          [*clean, '--seed', seed]
         end
 
         protected
 
         def executable
-          ENV['PARALLEL_TESTS_EXECUTABLE'] || determine_executable
+          if (executable = ENV['PARALLEL_TESTS_EXECUTABLE'])
+            [executable]
+          else
+            determine_executable
+          end
         end
 
         def determine_executable
-          "ruby"
+          ["ruby"]
         end
 
         def sum_up_results(results)
-          results = results.join(' ').gsub(/s\b/,'') # combine and singularize results
+          results = results.join(' ').gsub(/s\b/, '') # combine and singularize results
           counts = results.scan(/(\d+) (\w+)/)
-          counts.inject(Hash.new(0)) do |sum, (number, word)|
+          counts.each_with_object(Hash.new(0)) do |(number, word), sum|
             sum[word] += number.to_i
-            sum
           end
         end
 
         # read output of the process and print it in chunks
-        def capture_output(out, env, options={})
-          result = ""
-          loop do
-            begin
+        def capture_output(out, env, options = {})
+          result = +""
+          begin
+            loop do
               read = out.readpartial(1000000) # read whatever chunk we can get
               if Encoding.default_internal
                 read = read.force_encoding(Encoding.default_internal)
@@ -159,11 +193,13 @@ module ParallelTests
                 $stdout.flush
               end
             end
-          end rescue EOFError
+          rescue EOFError
+            nil
+          end
           result
         end
 
-        def sort_by_runtime(tests, runtimes, options={})
+        def sort_by_runtime(tests, runtimes, options = {})
           allowed_missing = options[:allowed_missing] || 1.0
           allowed_missing = tests.size * allowed_missing
 
@@ -173,14 +209,12 @@ module ParallelTests
             allowed_missing -= 1 unless time = runtimes[test]
             if allowed_missing < 0
               log = options[:runtime_log] || runtime_log
-              raise "Runtime log file '#{log}' does not contain sufficient data to sort #{tests.size} test files, please update or remove it."
+              raise RuntimeLogTooSmallError, "Runtime log file '#{log}' does not contain sufficient data to sort #{tests.size} test files, please update or remove it."
             end
             [test, time]
           end
 
-          if options[:verbose]
-            puts "Runtime found for #{tests.count(&:last)} of #{tests.size} tests"
-          end
+          puts "Runtime found for #{tests.count(&:last)} of #{tests.size} tests" if options[:verbose]
 
           set_unknown_runtime tests, options
         end
@@ -190,7 +224,7 @@ module ParallelTests
           lines = File.read(log).split("\n")
           lines.each_with_object({}) do |line, times|
             test, _, time = line.rpartition(':')
-            next unless test and time
+            next unless test && time
             times[test] = time.to_f if tests.include?(test)
           end
         end
@@ -217,7 +251,7 @@ module ParallelTests
           end.uniq
         end
 
-        def files_in_folder(folder, options={})
+        def files_in_folder(folder, options = {})
           pattern = if options[:symlinks] == false # not nil or true
             "**/*"
           else
@@ -225,7 +259,22 @@ module ParallelTests
             # http://stackoverflow.com/questions/357754/can-i-traverse-symlinked-directories-in-ruby-with-a-glob
             "**{,/*/**}/*"
           end
-          Dir[File.join(folder, pattern)].uniq
+          Dir[File.join(folder, pattern)].uniq.sort
+        end
+
+        def remove_command_arguments(command, *args)
+          remove_next = false
+          command.select do |arg|
+            if remove_next
+              remove_next = false
+              false
+            elsif args.include?(arg)
+              remove_next = true
+              false
+            else
+              true
+            end
+          end
         end
 
         private
@@ -236,12 +285,12 @@ module ParallelTests
           known, unknown = tests.partition(&:last)
           return if unknown.empty?
           unknown_runtime = options[:unknown_runtime] ||
-            (known.empty? ? 1 : known.map!(&:last).inject(:+) / known.size) # average
+            (known.empty? ? 1 : known.map!(&:last).sum / known.size) # average
           unknown.each { |set| set[1] = unknown_runtime }
         end
 
         def report_process_command?(options)
-          options[:verbose] || options[:verbose_process_command]
+          options[:verbose] || options[:verbose_command]
         end
       end
     end

data/lib/parallel_tests/test/runtime_logger.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'parallel_tests'
 require 'parallel_tests/test/runner'
 
@@ -22,7 +23,7 @@ module ParallelTests
             separator = "\n"
             groups = logfile.read.split(separator).map { |line| line.split(":") }.group_by(&:first)
             lines = groups.map do |file, times|
-              time = "%.2f" % times.map(&:last).map(&:to_f).inject(:+)
+              time = "%.2f" % times.map(&:last).map(&:to_f).sum
               "#{file}:#{time}"
             end
             logfile.rewind
@@ -34,7 +35,7 @@ module ParallelTests
         private
 
         def with_locked_log
-          File.open(logfile, File::RDWR|File::CREAT) do |logfile|
+          File.open(logfile, File::RDWR | File::CREAT) do |logfile|
             logfile.flock(File::LOCK_EX)
             yield logfile
           end
@@ -59,7 +60,7 @@ module ParallelTests
         end
 
         def message(test, delta)
-          return unless method = test.public_instance_methods(true).detect { |method| method =~ /^test_/ }
+          return unless method = test.public_instance_methods(true).detect { |m| m =~ /^test_/ }
           filename = test.instance_method(method).source_location.first.sub("#{Dir.pwd}/", "")
           "#{filename}:#{delta}"
         end
@@ -74,22 +75,26 @@ end
 
 if defined?(Minitest::Runnable) # Minitest 5
   class << Minitest::Runnable
-    prepend(Module.new do
-      def run(*)
-        ParallelTests::Test::RuntimeLogger.log_test_run(self) do
-          super
+    prepend(
+      Module.new do
+        def run(*)
+          ParallelTests::Test::RuntimeLogger.log_test_run(self) do
+            super
+          end
         end
       end
-    end)
+    )
   end
 
   class << Minitest
-    prepend(Module.new do
-      def run(*args)
-        result = super
-        ParallelTests::Test::RuntimeLogger.unique_log
-        result
+    prepend(
+      Module.new do
+        def run(*args)
+          result = super
+          ParallelTests::Test::RuntimeLogger.unique_log
+          result
+        end
       end
-    end)
+    )
   end
 end

data/lib/parallel_tests/version.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module ParallelTests
-  VERSION = Version = '3.3.0'
+  VERSION = '4.2.0'
 end

data/lib/parallel_tests.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require "parallel"
 require "parallel_tests/railtie" if defined? Rails::Railtie
 require "rbconfig"
@@ -17,21 +18,19 @@ module ParallelTests
         count,
         ENV["PARALLEL_TEST_PROCESSORS"],
         Parallel.processor_count
-      ].detect{|c| not c.to_s.strip.empty? }.to_i
+      ].detect { |c| !c.to_s.strip.empty? }.to_i
     end
 
     def with_pid_file
       Tempfile.open('parallel_tests-pidfile') do |f|
-        begin
-          ENV['PARALLEL_PID_FILE'] = f.path
-          # Pids object should be created before threads will start adding pids to it
-          # Otherwise we would have to use Mutex to prevent creation of several instances
-          @pids = pids
-          yield
-        ensure
-          ENV['PARALLEL_PID_FILE'] = nil
-          @pids = nil
-        end
+        ENV['PARALLEL_PID_FILE'] = f.path
+        # Pids object should be created before threads will start adding pids to it
+        # Otherwise we would have to use Mutex to prevent creation of several instances
+        @pids = pids
+        yield
+      ensure
+        ENV['PARALLEL_PID_FILE'] = nil
+        @pids = nil
       end
     end
 
@@ -57,7 +56,8 @@ module ParallelTests
       until !File.directory?(current) || current == previous
         filename = File.join(current, "Gemfile")
         return true if File.exist?(filename)
-        current, previous = File.expand_path("..", current), current
+        previous = current
+        current = File.expand_path("..", current)
       end
 
       false
@@ -76,7 +76,7 @@ module ParallelTests
     end
 
     def with_ruby_binary(command)
-      WINDOWS ? "#{RUBY_BINARY} -- #{command}" : command
+      WINDOWS ? [RUBY_BINARY, '--', command] : [command]
     end
 
     def wait_for_other_processes_to_finish

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parallel_tests
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 4.2.0
 platform: ruby
 authors:
 - Michael Grosser
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-16 00:00:00.000000000 Z
+date: 2023-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: parallel
@@ -24,7 +24,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email: michael@grosser.it
 executables:
 - parallel_spinach
@@ -68,10 +68,10 @@ licenses:
 - MIT
 metadata:
   bug_tracker_uri: https://github.com/grosser/parallel_tests/issues
-  documentation_uri: https://github.com/grosser/parallel_tests/blob/v3.3.0/Readme.md
-  source_code_uri: https://github.com/grosser/parallel_tests/tree/v3.3.0
+  documentation_uri: https://github.com/grosser/parallel_tests/blob/v4.2.0/Readme.md
+  source_code_uri: https://github.com/grosser/parallel_tests/tree/v4.2.0
   wiki_uri: https://github.com/grosser/parallel_tests/wiki
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -79,15 +79,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.4.0
+      version: 2.7.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.3
-signing_key: 
+rubygems_version: 3.3.3
+signing_key:
 specification_version: 4
 summary: Run Test::Unit / RSpec / Cucumber / Spinach in parallel
 test_files: []