ci-queue 0.10.1 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9ccbb5a14a7d65abe7ff69cac03dd08789351090
-  data.tar.gz: 1bcbaf620cc579174d1b4bcb5c53b9e8c49ba5f2
+  metadata.gz: 14313109ee765675cfb5babf40a14366888cedcd
+  data.tar.gz: 79d6f2168cb41a110d0e029dccc5306b61fbce02
 SHA512:
-  metadata.gz: 48b6fbbb41d76f883a84c9e989e01fae5810ea4a06b4685d79fe006362b2db391d4d7b3ebc0d8506cbe7eb56cb0642da37dc31cb1c25d1dbf61252c74a05724c
-  data.tar.gz: 63d66696121227cdc1767ac13f8afbe1d437e2f13c7991026f0a3a7189589503de9203ba1f40122b84b20836f21aad336a721d27e1c05fc166c11449be68ea2b
+  metadata.gz: e29790ab693de057e62a2db788d2fade577eb6cb3e7f6bbe5c0081d74327e47a8db66cf7ae7dc64fcbce8fa974641e3eabea58b97a024adafd98479e7bcae3cc
+  data.tar.gz: 51c1f832932265c19ee882c6a1d0861b07b09052ca0719771f8f7f545dc39c0a67db1e8e3f8648e6077561d688d5224d25df14d4f779bd8d853788870f9d1671

data/README.md

@@ -51,18 +51,17 @@ minitest-queue --queue path/to/test_order.log --failing-test 'SomeTest#test_some
 
 ### RSpec
 
-The RSpec integration is still missing some features, but is already usable:
+Assuming you use one of the supported CI providers, the command can be as simple as:
 
 ```bash
-rspec-queue --queue redis://example.com --build XXX --worker XXX
+rspec-queue --queue redis://example.com
 ```
 
-#### Missing features
-
-To be implemented:
+If you'd like to centralize the error reporting you can do so with:
 
-  - Requeueing
-  - Centralized reporting
+```bash
+rspec-queue --queue redis://example.com --timeout 600 --report
+```
 
 #### Limitations
 

data/lib/ci/queue.rb

@@ -4,6 +4,7 @@ require 'cgi'
 require 'ci/queue/version'
 require 'ci/queue/output_helpers'
 require 'ci/queue/configuration'
+require 'ci/queue/build_record'
 require 'ci/queue/static'
 require 'ci/queue/file'
 require 'ci/queue/bisect'

data/lib/ci/queue/build_record.rb

@@ -0,0 +1,44 @@
+module CI
+  module Queue
+    class BuildRecord
+      attr_reader :error_reports
+
+      def initialize(queue)
+        @queue = queue
+        @error_reports = {}
+        @stats = {}
+      end
+
+      def progress
+        @queue.progress
+      end
+
+      def queue_exhausted?
+        @queue.exhausted?
+      end
+
+      def record_error(id, payload, stats: nil)
+        error_reports[id] = payload
+        record_stats(stats)
+      end
+
+      def record_success(id, stats: nil)
+        error_reports.delete(id)
+        record_stats(stats)
+      end
+
+      def fetch_stats(stat_names)
+        stat_names.zip(stats.values_at(*stat_names).map(&:to_f))
+      end
+
+      private
+
+      attr_reader :stats
+
+      def record_stats(builds_stats)
+        return unless builds_stats
+        stats.merge!(builds_stats)
+      end
+    end
+  end
+end

data/lib/ci/queue/redis.rb

@@ -1,4 +1,5 @@
 require 'redis'
+require 'ci/queue/redis/build_record'
 require 'ci/queue/redis/base'
 require 'ci/queue/redis/worker'
 require 'ci/queue/redis/retry'

data/lib/ci/queue/redis/build_record.rb

@@ -0,0 +1,63 @@
+module CI
+  module Queue
+    module Redis
+      class BuildRecord
+        def initialize(queue, redis, config)
+          @queue = queue
+          @redis = redis
+          @config = config
+        end
+
+        def progress
+          @queue.progress
+        end
+
+        def queue_exhausted?
+          @queue.exhausted?
+        end
+
+        def record_error(id, payload, stats: nil)
+          redis.pipelined do
+            redis.hset(key('error-reports'), id, payload)
+            record_stats(stats)
+          end
+          nil
+        end
+
+        def record_success(id, stats: nil)
+          redis.pipelined do
+            redis.hdel(key('error-reports'), id)
+            record_stats(stats)
+          end
+          nil
+        end
+
+        def error_reports
+          redis.hgetall(key('error-reports'))
+        end
+
+        def fetch_stats(stat_names)
+          counts = redis.pipelined do
+            stat_names.each { |c| redis.hvals(key(c)) }
+          end
+          stat_names.zip(counts.map { |values| values.map(&:to_f).inject(:+).to_f }).to_h
+        end
+
+        private
+
+        attr_reader :config, :redis
+
+        def record_stats(stats)
+          return unless stats
+          stats.each do |stat_name, stat_value|
+            redis.hset(key(stat_name), config.worker_id, stat_value)
+          end
+        end
+
+        def key(*args)
+          ['build', config.build_id, *args].join(':')
+        end
+      end
+    end
+  end
+end

data/lib/ci/queue/redis/retry.rb

@@ -7,17 +7,8 @@ module CI
           super(tests, config)
         end
 
-        def minitest_reporters
-          require 'minitest/reporters/queue_reporter'
-          require 'minitest/reporters/redis_reporter'
-          @minitest_reporters ||= [
-            Minitest::Reporters::QueueReporter.new,
-            Minitest::Reporters::RedisReporter::Worker.new(
-              redis: redis,
-              build_id: config.build_id,
-              worker_id: config.worker_id,
-            )
-          ]
+        def build
+          @build ||= CI::Queue::Redis::BuildRecord.new(self, redis, config)
         end
 
         private

data/lib/ci/queue/redis/supervisor.rb

@@ -6,14 +6,13 @@ module CI
           false
         end
 
-        def minitest_reporters
-          require 'minitest/reporters/redis_reporter'
-          @reporters ||= [
-            Minitest::Reporters::RedisReporter::Summary.new(
-              build_id: build_id,
-              redis: redis,
-            )
-          ]
+        def total
+          wait_for_master(timeout: config.timeout)
+          redis.get(key('total')).to_i
+        end
+
+        def build
+          @build ||= CI::Queue::Redis::BuildRecord.new(self, redis, config)
         end
 
         def wait_for_workers

data/lib/ci/queue/redis/worker.rb

@@ -63,17 +63,8 @@ module CI
           Supervisor.new(redis_url, config)
         end
 
-        def minitest_reporters
-          require 'minitest/reporters/queue_reporter'
-          require 'minitest/reporters/redis_reporter'
-          @minitest_reporters ||= [
-            Minitest::Reporters::QueueReporter.new,
-            Minitest::Reporters::RedisReporter::Worker.new(
-              redis: redis,
-              build_id: build_id,
-              worker_id: worker_id,
-            )
-          ]
+        def build
+          @build ||= CI::Queue::Redis::BuildRecord.new(self, redis, config)
         end
 
         def acknowledge(test)
@@ -89,11 +80,12 @@ module CI
         def requeue(test, offset: Redis.requeue_offset)
           test_key = test.id
           raise_on_mismatching_test(test_key)
+          global_max_requeues = config.global_max_requeues(total)
 
-          requeued = eval_script(
+          requeued = config.max_requeues > 0 && global_max_requeues > 0 && eval_script(
             :requeue,
             keys: [key('processed'), key('requeues-count'), key('queue'), key('running')],
-            argv: [config.max_requeues, config.global_max_requeues(total), test_key, offset],
+            argv: [config.max_requeues, global_max_requeues, test_key, offset],
           ) == 1
 
           @reserved_test = test_key unless requeued

data/lib/ci/queue/static.rb

@@ -17,11 +17,8 @@ module CI
         @total = tests.size
       end
 
-      def minitest_reporters
-        require 'minitest/reporters/queue_reporter'
-        @minitest_reporters ||= [
-          Minitest::Reporters::QueueReporter.new,
-        ]
+      def build
+        @build ||= BuildRecord.new(self)
       end
 
       def supervisor

data/lib/ci/queue/version.rb

@@ -1,6 +1,6 @@
 module CI
   module Queue
-    VERSION = '0.10.1'
+    VERSION = '0.11.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue.rb

@@ -1,8 +1,14 @@
 require 'minitest'
-
 gem 'minitest-reporters', '~> 1.1'
 require 'minitest/reporters'
 
+require 'minitest/queue/failure_formatter'
+require 'minitest/queue/error_report'
+require 'minitest/queue/local_requeue_reporter'
+require 'minitest/queue/build_status_recorder'
+require 'minitest/queue/build_status_reporter'
+
+
 module Minitest
   class Requeue < Skip
     attr_reader :failure
@@ -68,11 +74,6 @@ module Minitest
 
     def queue=(queue)
       @queue = queue
-      if queue.respond_to?(:minitest_reporters)
-        self.queue_reporters = queue.minitest_reporters
-      else
-        self.queue_reporters = []
-      end
     end
 
     def queue_reporters=(reporters)

data/lib/minitest/queue/build_status_recorder.rb

@@ -0,0 +1,68 @@
+require 'minitest/reporters'
+
+module Minitest
+  module Queue
+    class BuildStatusRecorder < Minitest::Reporters::BaseReporter
+      COUNTERS = %w(
+        assertions
+        errors
+        failures
+        skips
+        requeues
+        total_time
+      ).freeze
+
+      class << self
+        attr_accessor :failure_formatter
+      end
+      self.failure_formatter = FailureFormatter
+
+      attr_accessor :requeues
+
+      def initialize(build:, **options)
+        super(options)
+
+        @build = build
+        self.failures = 0
+        self.errors = 0
+        self.skips = 0
+        self.requeues = 0
+      end
+
+      def report
+        # noop
+      end
+
+      def record(test)
+        super
+
+        self.total_time = Minitest.clock_time - start_time
+        if test.requeued?
+          self.requeues += 1
+        elsif test.skipped?
+          self.skips += 1
+        elsif test.error?
+          self.errors += 1
+        elsif test.failure
+          self.failures += 1
+        end
+
+
+        stats = COUNTERS.zip(COUNTERS.map { |c| send(c) })
+        if (test.failure || test.error?) && !test.skipped?
+          build.record_error("#{test.klass}##{test.name}", dump(test), stats: stats)
+        else
+          build.record_success("#{test.klass}##{test.name}", stats: stats)
+        end
+      end
+
+      private
+
+      def dump(test)
+        ErrorReport.new(self.class.failure_formatter.new(test).to_h).dump
+      end
+
+      attr_reader :build
+    end
+  end
+end

data/lib/minitest/queue/build_status_reporter.rb

@@ -0,0 +1,84 @@
+module Minitest
+  module Queue
+    class BuildStatusReporter < Minitest::Reporters::BaseReporter
+      include ::CI::Queue::OutputHelpers
+
+      def initialize(build:, **options)
+        @build = build
+        super(options)
+      end
+
+      def completed?
+        build.queue_exhausted?
+      end
+
+      def error_reports
+        build.error_reports.sort_by(&:first).map { |k, v| ErrorReport.load(v) }
+      end
+
+      def report
+        puts aggregates
+        errors = error_reports
+        puts errors
+
+        errors.empty?
+      end
+
+      def success?
+        errors == 0 && failures == 0
+      end
+
+      def record(*)
+        raise NotImplementedError
+      end
+
+      def failures
+        fetch_summary['failures'].to_i
+      end
+
+      def errors
+        fetch_summary['errors'].to_i
+      end
+
+      def assertions
+        fetch_summary['assertions'].to_i
+      end
+
+      def skips
+        fetch_summary['skips'].to_i
+      end
+
+      def requeues
+        fetch_summary['requeues'].to_i
+      end
+
+      def total_time
+        fetch_summary['total_time'].to_f
+      end
+
+      def progress
+        build.progress
+      end
+
+      private
+
+      attr_reader :build
+
+      def aggregates
+        success = failures.zero? && errors.zero?
+        failures_count = "#{failures} failures, #{errors} errors,"
+
+        step([
+          'Ran %d tests, %d assertions,' % [progress, assertions],
+          success ? green(failures_count) : red(failures_count),
+          yellow("#{skips} skips, #{requeues} requeues"),
+          'in %.2fs (aggregated)' % total_time,
+        ].join(' '), collapsed: success)
+      end
+
+      def fetch_summary
+        @summary ||= build.fetch_stats(BuildStatusRecorder::COUNTERS)
+      end
+    end
+  end
+end

data/lib/minitest/queue/error_report.rb

@@ -0,0 +1,69 @@
+module Minitest
+  module Queue
+    class ErrorReport
+      class << self
+        attr_accessor :coder
+
+        def load(payload)
+          new(coder.load(payload))
+        end
+      end
+
+      self.coder = Marshal
+
+      begin
+        require 'snappy'
+        require 'msgpack'
+        require 'stringio'
+
+        module SnappyPack
+          extend self
+
+          MSGPACK = MessagePack::Factory.new
+          MSGPACK.register_type(0x00, Symbol)
+
+          def load(payload)
+            io = StringIO.new(Snappy.inflate(payload))
+            MSGPACK.unpacker(io).unpack
+          end
+
+          def dump(object)
+            io = StringIO.new
+            packer = MSGPACK.packer(io)
+            packer.pack(object)
+            packer.flush
+            io.rewind
+            Snappy.deflate(io.string).force_encoding(Encoding::UTF_8)
+          end
+        end
+
+        self.coder = SnappyPack
+      rescue LoadError
+      end
+
+      def initialize(data)
+        @data = data
+      end
+
+      def dump
+        self.class.coder.dump(@data)
+      end
+
+      def test_name
+        @data[:test_name]
+      end
+
+      def test_and_module_name
+        @data[:test_and_module_name]
+      end
+
+      def to_s
+        output
+      end
+
+      def output
+        @data[:output]
+      end
+    end
+  end
+end

data/lib/minitest/{reporters → queue}/failure_formatter.rb

@@ -1,7 +1,8 @@
 require 'delegate'
+require 'ansi'
 
 module Minitest
-  module Reporters
+  module Queue
     class FailureFormatter < SimpleDelegator
       include ANSI::Code
 

data/lib/minitest/{reporters/queue_reporter.rb → queue/local_requeue_reporter.rb}

@@ -2,8 +2,8 @@ require 'ci/queue/output_helpers'
 require 'minitest/reporters'
 
 module Minitest
-  module Reporters
-    class QueueReporter < BaseReporter
+  module Queue
+    class LocalRequeueReporter < Minitest::Reporters::BaseReporter
       include ::CI::Queue::OutputHelpers
       attr_accessor :requeues
 

data/lib/minitest/queue/runner.rb

@@ -43,6 +43,11 @@ module Minitest
       def run_command
         set_load_path
         Minitest.queue = queue
+        Minitest.queue_reporters = [
+          Minitest::Queue::LocalRequeueReporter.new,
+          Minitest::Queue::BuildStatusRecorder.new(build: queue.build)
+        ]
+
         trap('TERM') { Minitest.queue.shutdown! }
         trap('INT') { Minitest.queue.shutdown! }
         load_tests
@@ -115,12 +120,9 @@ module Minitest
           end
         end
 
-        success = supervisor.minitest_reporters.all?(&:success?)
-        supervisor.minitest_reporters.each do |reporter|
-          reporter.report
-        end
-
-        exit! success ? 0 : 1
+        reporter = BuildStatusReporter.new(build: supervisor.build)
+        reporter.report
+        exit! reporter.success? ? 0 : 1
       end
 
       private

data/lib/minitest/reporters/redis_reporter.rb

@@ -1,5 +1,4 @@
 require 'minitest/reporters'
-require 'minitest/reporters/failure_formatter'
 
 module Minitest
   module Reporters
@@ -20,177 +19,32 @@ module Minitest
       end
       self.failure_formatter = FailureFormatter
 
-      class Error
-        class << self
-          attr_accessor :coder
-
-          def load(payload)
-            new(coder.load(payload))
-          end
-        end
-
-        self.coder = Marshal
-
-        begin
-          require 'snappy'
-          require 'msgpack'
-          require 'stringio'
-
-          module SnappyPack
-            extend self
-
-            MSGPACK = MessagePack::Factory.new
-            MSGPACK.register_type(0x00, Symbol)
-
-            def load(payload)
-              io = StringIO.new(Snappy.inflate(payload))
-              MSGPACK.unpacker(io).unpack
-            end
-
-            def dump(object)
-              io = StringIO.new
-              packer = MSGPACK.packer(io)
-              packer.pack(object)
-              packer.flush
-              io.rewind
-              Snappy.deflate(io.string).force_encoding(Encoding::UTF_8)
-            end
-          end
-
-          self.coder = SnappyPack
-        rescue LoadError
-        end
-
-        def initialize(data)
-          @data = data
-        end
-
-        def dump
-          self.class.coder.dump(@data)
-        end
-
-        def test_name
-          @data[:test_name]
-        end
-
-        def test_and_module_name
-          @data[:test_and_module_name]
-        end
-
-        def to_s
-          output
-        end
-
-        def output
-          @data[:output]
-        end
-      end
-
       class Base < BaseReporter
-        def initialize(redis:, build_id:, **options)
-          @redis = redis
-          @key = "build:#{build_id}"
+        def initialize(build:, **options)
+          @build = build
           super(options)
         end
 
-        def total
-          redis.get(key('total')).to_i
-        end
-
-        def processed
-          redis.scard(key('processed')).to_i
-        end
-
         def completed?
-          total > 0 && total == processed
+          build.queue_exhausted?
         end
 
         def error_reports
-          redis.hgetall(key('error-reports')).sort_by(&:first).map { |k, v| Error.load(v) }
+          build.error_reports.sort_by(&:first).map { |k, v| ErrorReport.load(v) }
         end
 
         private
 
-        attr_reader :redis
-
-        def key(*args)
-          [@key, *args].join(':')
-        end
       end
 
       class Summary < Base
-        include ::CI::Queue::OutputHelpers
-
-        def report
-          puts aggregates
-          errors = error_reports
-          puts errors
-
-          errors.empty?
-        end
-
-        def success?
-          errors == 0 && failures == 0
-        end
-
-        def record(*)
-          raise NotImplementedError
-        end
-
-        def failures
-          fetch_summary['failures'].to_i
-        end
-
-        def errors
-          fetch_summary['errors'].to_i
-        end
-
-        def assertions
-          fetch_summary['assertions'].to_i
-        end
-
-        def skips
-          fetch_summary['skips'].to_i
-        end
-
-        def requeues
-          fetch_summary['requeues'].to_i
-        end
-
-        def total_time
-          fetch_summary['total_time'].to_f
-        end
-
-        private
-
-        def aggregates
-          success = failures.zero? && errors.zero?
-          failures_count = "#{failures} failures, #{errors} errors,"
-
-          step([
-            'Ran %d tests, %d assertions,' % [processed, assertions],
-            success ? green(failures_count) : red(failures_count),
-            yellow("#{skips} skips, #{requeues} requeues"),
-            'in %.2fs (aggregated)' % total_time,
-          ].join(' '), collapsed: success)
-        end
-
-        def fetch_summary
-          @summary ||= begin
-            counts = redis.pipelined do
-              COUNTERS.each { |c| redis.hgetall(key(c)) }
-            end
-            COUNTERS.zip(counts.map { |h| h.values.map(&:to_f).inject(:+).to_f }).to_h
-          end
-        end
       end
 
       class Worker < Base
         attr_accessor :requeues
 
-        def initialize(worker_id:, **options)
+        def initialize(*)
           super
-          @worker_id = worker_id
           self.failures = 0
           self.errors = 0
           self.skips = 0
@@ -215,25 +69,22 @@ module Minitest
             self.failures += 1
           end
 
-          redis.multi do
-            if (test.failure || test.error?) && !test.skipped?
-              redis.hset(key('error-reports'), "#{test.klass}##{test.name}", dump(test))
-            else
-              redis.hdel(key('error-reports'), "#{test.klass}##{test.name}")
-            end
-            COUNTERS.each do |counter|
-              redis.hset(key(counter), worker_id, send(counter))
-            end
+
+          stats = COUNTERS.zip(COUNTERS.map { |c| send(c) })
+          if (test.failure || test.error?) && !test.skipped?
+            build.record_error("#{test.klass}##{test.name}", dump(test), stats: stats)
+          else
+            build.record_success("#{test.klass}##{test.name}", stats: stats)
           end
         end
 
         private
 
         def dump(test)
-          Error.new(RedisReporter.failure_formatter.new(test).to_h).dump
+          ErrorReport.new(RedisReporter.failure_formatter.new(test).to_h).dump
         end
 
-        attr_reader :worker_id, :aggregates
+        attr_reader :aggregates
       end
     end
   end

data/lib/rspec/queue.rb

@@ -1,5 +1,7 @@
+require 'delegate'
 require 'rspec/core'
 require 'ci/queue'
+require 'rspec/queue/build_status_recorder'
 
 module RSpec
   module Queue
@@ -9,6 +11,34 @@ module RSpec
       end
     end
 
+    module RunnerHelpers
+      private
+
+      def queue_url
+        configuration.queue_url || ENV['CI_QUEUE_URL']
+      end
+
+      def invalid_usage!(message)
+        reopen_previous_step
+        puts red(message)
+        puts
+        puts 'Please use --help for a listing of valid options'
+        exit! 1 # exit! is required to avoid at_exit callback
+      end
+
+      def exit!(*)
+        STDOUT.flush
+        STDERR.flush
+        super
+      end
+
+      def abort!(message)
+        reopen_previous_step
+        puts red(message)
+        exit! 1 # exit! is required to avoid at_exit callback
+      end
+    end
+
     module ConfigurationExtension
       private
 
@@ -43,6 +73,11 @@ module RSpec
           options[:queue_url] = url
         end
 
+        parser.on('--report', *help) do |url|
+          options[:report] = true
+          options[:runner] = RSpec::Queue::ReportRunner.new
+        end
+
         help = split_heredoc(<<-EOS)
           Unique identifier for the workload. All workers working on the same suite of tests must have the same build identifier.
           If the build is tried again, or another revision is built, this value must be different.
@@ -114,6 +149,69 @@ module RSpec
 
     RSpec::Core::Parser.prepend(ParserExtension)
 
+    module ExampleExtension
+      protected
+
+      def mark_as_requeued!(reporter)
+        @metadata = @metadata.dup # Avoid mutating the @metadata hash of the original Example instance
+        @metadata[:execution_result] = execution_result.dup
+
+
+        failure_notification = RSpec::Core::Notifications::FailedExampleNotification.new(self)
+        execution_result.exception = @exception
+        execution_result.status = :failed
+
+        presenter = RSpec::Core::Formatters::ExceptionPresenter::Factory.new(self).build
+        error_message = presenter.fully_formatted_lines(nil, ::RSpec::Core::Formatters::ConsoleCodes)
+        error_message.delete_at(1) # remove the example description
+
+        @exception = nil
+        execution_result.exception = nil
+        execution_result.status = :pending
+        execution_result.pending_message = [
+          "The example failed, but another attempt will be done to rule out flakiness",
+          *error_message.map { |l| l.empty? ? l : "   " + l },
+        ].join("\n")
+
+        # Ensure the example is recorded as ran, so it's visible to formatters
+        reporter.example_started(self)
+        finish(reporter, acknowledge: false)
+      end
+
+      private
+
+      def start(*)
+        reset! # In case that example was already ran but got requeued
+        super
+      end
+
+      def finish(reporter, acknowledge: true)
+        if acknowledge && reporter.respond_to?(:requeue)
+          if @exception && reporter.requeue
+            reporter.cancel_run!
+            dup.mark_as_requeued!(reporter)
+            return
+          elsif reporter.acknowledge || !@exception
+            # If the test was already acknowledged by another worker (we timed out)
+            # Then we only record it if it is successful.
+            super(reporter)
+          else
+            reporter.cancel_run!
+            return
+          end
+        else
+          super(reporter)
+        end
+      end
+
+      def reset!
+        @exception = nil
+        @metadata[:execution_result] = RSpec::Core::Example::ExecutionResult.new
+      end
+    end
+
+    RSpec::Core::Example.prepend(ExampleExtension)
+
     class SingleExample
       attr_reader :example_group, :example
 
@@ -134,16 +232,94 @@ module RSpec
         return if RSpec.world.wants_to_quit
         instance = example_group.new(example.inspect_output)
         example_group.set_ivars(instance, example_group.before_context_ivars)
-        succeeded = example.run(instance, reporter)
-        if !succeeded && reporter.fail_fast_limit_met?
-          RSpec.world.wants_to_quit = true
+        example.run(instance, reporter)
+      end
+    end
+
+    class ReportRunner
+      include RunnerHelpers
+      include CI::Queue::OutputHelpers
+
+      def call(options, stdout, stderr)
+        setup(options, stdout, stderr)
+
+        queue = CI::Queue.from_uri(queue_url, RSpec::Queue.config)
+
+        supervisor = begin
+          queue.supervisor
+        rescue NotImplementedError => error
+          abort! error.message
+        end
+
+        step("Waiting for workers to complete")
+
+        unless supervisor.wait_for_workers
+          unless supervisor.queue_initialized?
+            abort! "No master was elected. Did all workers crash?"
+          end
+
+          unless supervisor.exhausted?
+            abort! "#{supervisor.size} tests weren't run."
+          end
+        end
+
+        # TODO: better reporting
+        errors = supervisor.build.error_reports.sort_by(&:first).map(&:last)
+        if errors.empty?
+          step(green('No errors found'))
+          0
+        else
+          message = errors.size == 1 ? "1 error found" : "#{errors.size} errors found"
+          step(red(message), collapsed: false)
+          puts errors
+          1
         end
-        succeeded
+      end
+
+      private
+
+      attr_reader :configuration
+
+      def setup(options, out, err)
+        @options       = options
+        @configuration = RSpec.configuration
+        @world         = RSpec.world
+        @configuration.error_stream = err
+        @configuration.output_stream = out if @configuration.output_stream == $stdout
+        @options.options.delete(:requires) # Prevent loading of spec_helper so the app doesn't need to boot
+        @options.configure(@configuration)
+
+        invalid_usage!('Missing --queue parameter') unless queue_url
+        invalid_usage!('Missing --build parameter') unless RSpec::Queue.config.build_id
+      end
+    end
+
+    class QueueReporter < SimpleDelegator
+      def initialize(reporter, queue, example)
+        @queue = queue
+        @example = example
+        super(reporter)
+      end
+
+      def requeue
+        @queue.requeue(@example)
+      end
+
+      def cancel_run!
+        # Remove the requeued example from the list of examples ran
+        # Otherwise some formatters might break because the example state is reset
+        examples.pop
+        nil
+      end
+
+      def acknowledge
+        @queue.acknowledge(@example)
       end
     end
 
     class Runner < ::RSpec::Core::Runner
       include CI::Queue::OutputHelpers
+      include RunnerHelpers
 
       def setup(err, out)
         super
@@ -160,14 +336,16 @@ module RSpec
         end
 
         queue = CI::Queue.from_uri(queue_url, RSpec::Queue.config)
+        BuildStatusRecorder.build = queue.build
         queue.populate(examples, random: ordering_seed, &:id)
         examples_count = examples.size # TODO: figure out which stub value would be best
         success = true
         @configuration.reporter.report(examples_count) do |reporter|
+          @configuration.add_formatter(BuildStatusRecorder)
+
           @configuration.with_suite_hooks do
             queue.poll do |example|
-              success &= example.run(reporter)
-              queue.acknowledge(example)
+              success &= example.run(QueueReporter.new(reporter, queue, example))
             end
           end
         end
@@ -185,30 +363,6 @@ module RSpec
           Random.new
         end
       end
-
-      def queue_url
-        configuration.queue_url || ENV['CI_QUEUE_URL']
-      end
-
-      def invalid_usage!(message)
-        reopen_previous_step
-        puts red(message)
-        puts
-        puts 'Please use --help for a listing of valid options'
-        exit! 1 # exit! is required to avoid minitest at_exit callback
-      end
-
-      def exit!(*)
-        STDOUT.flush
-        STDERR.flush
-        super
-      end
-
-      def abort!(message)
-        reopen_previous_step
-        puts red(message)
-        exit! 1 # exit! is required to avoid minitest at_exit callback
-      end
     end
   end
 end

data/lib/rspec/queue/build_status_recorder.rb

@@ -0,0 +1,38 @@
+module RSpec
+  module Queue
+    class BuildStatusRecorder
+      ::RSpec::Core::Formatters.register self, :example_passed, :example_failed
+
+      class << self
+        attr_accessor :build
+      end
+
+      def initialize(*)
+      end
+
+      def example_passed(notification)
+        example = notification.example
+        build.record_success(example.id)
+      end
+
+      def example_failed(notification)
+        example = notification.example
+        build.record_error(example.id, [
+          notification.fully_formatted(nil),
+          colorized_rerun_command(example),
+        ].join("\n"))
+      end
+
+      private
+
+      def colorized_rerun_command(example, colorizer=::RSpec::Core::Formatters::ConsoleCodes)
+        colorizer.wrap("rspec #{example.location_rerun_argument}", RSpec.configuration.failure_color) + " " +
+        colorizer.wrap("# #{example.full_description}",   RSpec.configuration.detail_color)
+      end
+
+      def build
+        self.class.build
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.10.1
+  version: 0.11.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-01-10 00:00:00.000000000 Z
+date: 2018-02-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ansi
@@ -173,12 +173,14 @@ files:
 - exe/rspec-queue
 - lib/ci/queue.rb
 - lib/ci/queue/bisect.rb
+- lib/ci/queue/build_record.rb
 - lib/ci/queue/configuration.rb
 - lib/ci/queue/file.rb
 - lib/ci/queue/output_helpers.rb
 - lib/ci/queue/redis.rb
 - lib/ci/queue/redis/acknowledge.lua
 - lib/ci/queue/redis/base.rb
+- lib/ci/queue/redis/build_record.rb
 - lib/ci/queue/redis/requeue.lua
 - lib/ci/queue/redis/reserve.lua
 - lib/ci/queue/redis/reserve_lost.lua
@@ -188,13 +190,17 @@ files:
 - lib/ci/queue/static.rb
 - lib/ci/queue/version.rb
 - lib/minitest/queue.rb
+- lib/minitest/queue/build_status_recorder.rb
+- lib/minitest/queue/build_status_reporter.rb
+- lib/minitest/queue/error_report.rb
+- lib/minitest/queue/failure_formatter.rb
+- lib/minitest/queue/local_requeue_reporter.rb
 - lib/minitest/queue/runner.rb
 - lib/minitest/reporters/bisect_reporter.rb
-- lib/minitest/reporters/failure_formatter.rb
 - lib/minitest/reporters/order_reporter.rb
-- lib/minitest/reporters/queue_reporter.rb
 - lib/minitest/reporters/redis_reporter.rb
 - lib/rspec/queue.rb
+- lib/rspec/queue/build_status_recorder.rb
 homepage: https://github.com/Shopify/ci-queue
 licenses:
 - MIT