ci-queue 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 97f69595ec09509df32703213fd2d882db14f017
-  data.tar.gz: cae08e13e586bd5c7cc8927057225084b0ee0d74
+  metadata.gz: d71f2dd528eeda6758bc37bb9142cac9814150ea
+  data.tar.gz: 3042fc3632751cc3ef1834dd749978dff2c628a6
 SHA512:
-  metadata.gz: 742b3d39caf903b480950a86eafb687776e8b4d1a9465d28e1ce29ff37d3242f3127a40d7115e51eddec03b6e22322cef0108ef2a64f6a5744b642b3e1b14ecf
-  data.tar.gz: 8f753085f6408c98e698c0a75c06b77a427e066cbad9aa7c1db102613cdcf4b6d25d27e93145e86928bd5ae956b54adeb44245f36ee2ceee3eb3e9646c45363b
+  metadata.gz: bc012267f7470bc5527ae5081ebfe2e268acf6d777db6c8c50d218e9aa21fa7a3dba46e2f1af467a9f7fcbf5ca98512322128dea0115ce4a047cd6a09aa09253
+  data.tar.gz: 80445ac4bb1f43a2a930269708c7f97178921d5da60b198af42ffdeecf07e0e0c3242cb4c065c6b220ee4ef19b6a7935716330875df1aa1b62b665f891641ae9

data/README.md

@@ -39,10 +39,16 @@ Additionally you can configure the requeue settings (see main README) with `--ma
 
 If you'd like to centralize the error reporting you can do so with:
 
-```
+```bash
 minitest-queue --queue redis://example.com --timeout 600 report
 ```
 
+The runner also comes with a tool to investigate leaky tests:
+
+```bash
+minitest-queue --queue path/to/test_order.log --failing-test 'SomeTest#test_something' bisect -Itest test/**/*_test.rb
+```
+
 ### RSpec
 
 The RSpec integration is not implemented yet.

data/lib/ci/queue.rb

@@ -7,6 +7,7 @@ require 'ci/queue/index'
 require 'ci/queue/configuration'
 require 'ci/queue/static'
 require 'ci/queue/file'
+require 'ci/queue/bisect'
 
 module CI
   module Queue

data/lib/ci/queue/bisect.rb

@@ -0,0 +1,59 @@
+module CI
+  module Queue
+    class Bisect
+      def initialize(path, config)
+        @tests = ::File.readlines(path).map(&:strip).reject(&:empty?).take_while { |t| t != config.failing_test }
+        @config = config
+      end
+
+      def size
+        @tests.size
+      end
+
+      def populate(all_tests, &test_indexer)
+        @all_tests = all_tests
+        @test_indexer = test_indexer
+      end
+
+      def to_a
+        @tests + [config.failing_test]
+      end
+
+      def suspects_left
+        @tests.size
+      end
+
+      def failing_test
+        Static.new([config.failing_test], config).populate(@all_tests, &@test_indexer)
+      end
+
+      def candidates
+        Static.new(first_half + [config.failing_test], config).populate(@all_tests, &@test_indexer)
+      end
+
+      def failed!
+        @tests = first_half
+      end
+
+      def succeeded!
+        @tests = second_half
+      end
+
+      private
+
+      attr_reader :config
+
+      def slices
+        @tests.each_slice((@tests.size / 2.0).ceil).to_a
+      end
+
+      def first_half
+        slices.first
+      end
+
+      def second_half
+        slices.last
+      end
+    end
+  end
+end

data/lib/ci/queue/configuration.rb

@@ -1,7 +1,7 @@
 module CI
   module Queue
     class Configuration
-      attr_accessor :timeout, :build_id, :worker_id, :max_requeues, :requeue_tolerance, :namespace, :seed
+      attr_accessor :timeout, :build_id, :worker_id, :max_requeues, :requeue_tolerance, :namespace, :seed, :failing_test
 
       class << self
         def from_env(env)

data/lib/ci/queue/file.rb

@@ -10,7 +10,8 @@ module CI
       end
 
       def initialize(path, *args)
-        super(::File.readlines(path).map(&:strip).reject(&:empty?), *args)
+        io = path == '-' ? STDIN : ::File.open(path)
+        super(io.each_line.map(&:strip).reject(&:empty?), *args)
       end
     end
   end

data/lib/ci/queue/version.rb

@@ -1,6 +1,6 @@
 module CI
   module Queue
-    VERSION = '0.7.0'
+    VERSION = '0.8.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue.rb

@@ -58,7 +58,8 @@ module Minitest
 
     def queue_reporters=(reporters)
       @queue_reporters ||= []
-      Reporters.reporters = ((Reporters.reporters || []) - @queue_reporters) + reporters
+      Reporters.use!(((Reporters.reporters || []) - @queue_reporters) + reporters)
+      Minitest.backtrace_filter.add_filter(%r{exe/minitest-queue|lib/ci/queue/})
       @queue_reporters = reporters
     end
 

data/lib/minitest/queue/runner.rb

@@ -49,6 +49,52 @@ module Minitest
         # Let minitest's at_exit hook trigger
       end
 
+      def bisect_command
+        invalid_usage! "Missing the FAILING_TEST argument." unless queue_config.failing_test
+
+        @queue = CI::Queue::Bisect.new(queue_url, queue_config)
+        Minitest.queue = queue
+        set_load_path
+        load_tests
+        populate_queue
+
+        step("Testing the failing test in isolation")
+        unless run_tests_in_fork(queue.failing_test)
+          puts reopen_previous_step
+          puts red("The test fail when run alone, no need to bisect.")
+          exit! 0
+        end
+
+        run_index = 0
+        while queue.suspects_left > 1
+          run_index += 1
+          step("Run ##{run_index}, #{queue.suspects_left} suspects left")
+          if run_tests_in_fork(queue.candidates)
+            queue.succeeded!
+          else
+            queue.failed!
+          end
+          puts
+        end
+
+        failing_order = queue.candidates
+        step("Final validation")
+        status = if run_tests_in_fork(failing_order)
+          step(yellow("The bisection was inconclusive, there might not be any leaky test here."))
+          exit! 1
+        else
+          step(green('The following command should reproduce the leak on your machine:'), collapsed: false)
+          command = %w(bundle exec minitest-queue --queue - run)
+          command << "-I#{load_paths}" if load_paths
+          command += argv
+
+          puts
+          puts "cat <<EOF |\n#{failing_order.to_a.join("\n")}\nEOF\n#{command.join(' ')}"
+          puts
+          exit! 0
+        end
+      end
+
       def report_command
         supervisor = begin
           queue.supervisor
@@ -73,7 +119,6 @@ module Minitest
           reporter.report
         end
 
-        STDOUT.flush
         exit! success ? 0 : 1
       end
 
@@ -82,6 +127,17 @@ module Minitest
       attr_reader :queue_config, :options, :command, :argv
       attr_accessor :queue, :queue_url, :load_paths
 
+      def run_tests_in_fork(queue)
+        child_pid = fork do
+          Minitest.queue = queue
+          Minitest::Reporters.use!([Minitest::Reporters::SpecReporter.new])
+          exit # let minitest excute its at_exit
+        end
+
+        _, status = Process.wait2(child_pid)
+        return status.success?
+      end
+
       def populate_queue
         Minitest.queue.populate(shuffle(Minitest.loaded_tests), &:to_s) # TODO: stop serializing
       end
@@ -210,6 +266,16 @@ module Minitest
 
           opts.separator ""
           opts.separator "    report: Wait for all workers to complete and summarize the test failures."
+
+          opts.separator ""
+          opts.separator "    bisect: bisect a test suite to find global state leaks."
+          help = split_heredoc(<<-EOS)
+            The identifier of the failing test.
+          EOS
+          opts.separator ""
+          opts.on('--failing-test TEST_IDENTIFIER') do |identifier|
+            queue_config.failing_test = identifier
+          end
         end
       end
 
@@ -218,6 +284,7 @@ module Minitest
       end
 
       def shuffle(tests)
+        return tests unless queue_config.seed
         random = Random.new(Digest::MD5.hexdigest(queue_config.seed).to_i(16))
         tests.shuffle(random: random)
       end
@@ -233,6 +300,12 @@ module Minitest
         exit! 1 # exit! is required to avoid minitest at_exit callback
       end
 
+      def exit!(*)
+        STDOUT.flush
+        STDERR.flush
+        super
+      end
+
       def abort!(message)
         reopen_previous_step
         puts red(message)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-11-22 00:00:00.000000000 Z
+date: 2017-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -114,6 +114,7 @@ files:
 - dev.yml
 - exe/minitest-queue
 - lib/ci/queue.rb
+- lib/ci/queue/bisect.rb
 - lib/ci/queue/configuration.rb
 - lib/ci/queue/file.rb
 - lib/ci/queue/index.rb
@@ -155,7 +156,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.10
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Distribute tests over many workers using a queue