ci-queue 0.16.0 → 0.17.0

data/lib/ci/queue/redis/worker.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'ci/queue/static'
 
 module CI
@@ -44,7 +45,7 @@ module CI
 
         def poll
           wait_for_master
-          until shutdown_required? || config.circuit_breaker.open? || exhausted?
+          until shutdown_required? || config.circuit_breakers.any?(&:open?) || exhausted?
             if test = reserve
               yield index.fetch(test), @last_warning
             else

data/lib/ci/queue/static.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module CI
   module Queue
     class Static
@@ -48,7 +50,7 @@ module CI
       end
 
       def poll
-        while !config.circuit_breaker.open? && test = @queue.shift
+        while !config.circuit_breakers.any?(&:open?) && test = @queue.shift
           yield index.fetch(test)
         end
       end

data/lib/ci/queue/version.rb

@@ -1,6 +1,8 @@
+# frozen_string_literal: true
+
 module CI
   module Queue
-    VERSION = '0.16.0'
+    VERSION = '0.17.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue.rb

@@ -1,5 +1,5 @@
+# frozen_string_literal: true
 require 'minitest'
-gem 'minitest-reporters', '~> 1.1'
 require 'minitest/reporters'
 
 require 'minitest/queue/failure_formatter'
@@ -9,6 +9,11 @@ require 'minitest/queue/build_status_recorder'
 require 'minitest/queue/build_status_reporter'
 require 'minitest/queue/order_reporter'
 require 'minitest/queue/junit_reporter'
+require 'minitest/queue/test_data_reporter'
+require 'minitest/queue/grind_recorder'
+require 'minitest/queue/grind_reporter'
+require 'minitest/queue/test_time_recorder'
+require 'minitest/queue/test_time_reporter'
 
 module Minitest
   class Requeue < Skip
@@ -45,7 +50,7 @@ module Minitest
     end
 
     def result_label
-      "Skipped"
+      "Flaked"
     end
 
     def backtrace
@@ -97,6 +102,7 @@ module Minitest
 
   module Queue
     class SingleExample
+
       def initialize(runnable, method_name)
         @runnable = runnable
         @method_name = method_name
@@ -144,8 +150,8 @@ module Minitest
       if queue
         run_from_queue(*args)
 
-        if queue.config.circuit_breaker.open?
-          STDERR.puts "This worker is exiting early because it encountered too many consecutive test failures, probably because of some corrupted state."
+        if queue.config.circuit_breakers.any?(&:open?)
+          STDERR.puts queue.config.circuit_breakers.map(&:message).join(' ').strip
         end
       else
         super

data/lib/minitest/queue/build_status_recorder.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'minitest/reporters'
 
 module Minitest

data/lib/minitest/queue/build_status_reporter.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Minitest
   module Queue
     class BuildStatusReporter < Minitest::Reporters::BaseReporter

data/lib/minitest/queue/error_report.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Minitest
   module Queue
     class ErrorReport
@@ -57,6 +58,18 @@ module Minitest
         @data[:test_and_module_name]
       end
 
+      def test_file
+        @data[:test_file]
+      end
+
+      def test_line
+        @data[:test_line]
+      end
+
+      def to_h
+        @data
+      end
+
       def to_s
         output
       end

data/lib/minitest/queue/failure_formatter.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'delegate'
 require 'ansi'
 
@@ -20,7 +21,10 @@ module Minitest
       end
 
       def to_h
+        test_file, test_line = test.source_location
         {
+          test_file: test_file,
+          test_line: test_line,
           test_and_module_name: "#{test.klass}##{test.name}",
           test_name: test.name,
           output: to_s,

data/lib/minitest/queue/grind_recorder.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+module Minitest
+  module Queue
+    class GrindRecorder < Minitest::Reporters::BaseReporter
+
+      attr_accessor :test_count
+
+      def self.counters
+        @counters ||= {
+          'failures' => 0,
+          'errors' => 0,
+          'skips' => 0,
+          'test_count' => 0
+        }
+      end
+
+      class << self
+        attr_accessor :failure_formatter
+      end
+      self.failure_formatter = FailureFormatter
+
+      def initialize(build:, **options)
+        super(options)
+        @build = build
+      end
+
+      def record(test)
+        increment_counter(test)
+        record_test(test)
+      end
+
+      private
+
+      def record_test(test)
+        stats = self.class.counters
+        if (test.failure || test.error?) && !test.skipped?
+          build.record_error(dump(test), stats: stats)
+        else
+          build.record_success(stats: stats)
+        end
+      end
+
+      def increment_counter(test)
+        if test.skipped?
+          self.class.counters['skips'] += 1
+        elsif test.error?
+          self.class.counters['errors'] += 1
+        elsif test.failure
+          self.class.counters['failures'] += 1
+        end
+        self.class.counters['test_count'] += 1
+
+        key = "count##{test.klass}##{test.name}"
+
+        unless self.class.counters.key?(key)
+          self.class.counters[key] = 0
+        end
+        self.class.counters[key] += 1
+      end
+
+      def dump(test)
+        ErrorReport.new(self.class.failure_formatter.new(test).to_h).dump
+      end
+
+      attr_reader :build
+    end
+  end
+end

data/lib/minitest/queue/grind_reporter.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+require 'minitest/reporters'
+
+module Minitest
+  module Queue
+    class GrindReporter < Minitest::Reporters::BaseReporter
+      include ::CI::Queue::OutputHelpers
+
+      def initialize(build:, **options)
+        @build = build
+        @success = true
+        super(options)
+      end
+
+      def report
+        puts '+++ Results'
+
+        if flaky_tests.empty?
+          puts green('all tests passed every time, grinding did not uncover any flakiness')
+          return
+        end
+        @success = false
+
+        flaky_tests.each do |name, errors|
+          total_runs = fetch_counts(name)
+          flakiness_percentage = (errors.count / total_runs) * 100
+
+          error_messages = errors.map do |message|
+            message.to_s.lines.map { |l| "\t#{l}"}.join
+          end.to_set.to_a.join("\n\n")
+
+          puts <<~EOS
+            #{red(name)}
+            Runs: #{total_runs.to_i}
+            Failures: #{errors.count}
+            Flakiness Percentage: #{flakiness_percentage.to_i}%
+            Errors:
+            #{error_messages}
+          EOS
+        end
+      end
+
+      def success?
+        @success
+      end
+
+      def flaky_tests
+        @flaky_tests ||= begin
+          flaky_tests = {}
+          build.error_reports.each do |error|
+            err = ErrorReport.load(error)
+            name = err.test_and_module_name
+            flaky_tests[name] ||= []
+            flaky_tests[name] << err
+          end
+          flaky_tests
+        end
+      end
+
+      def record(*)
+        raise NotImplementedError
+      end
+
+      def fetch_counts(test)
+        key = "count##{test}"
+        build.fetch_stats([key])[key]
+      end
+
+      private
+
+      attr_reader :build
+    end
+  end
+end

data/lib/minitest/queue/junit_reporter.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'minitest/reporters'
 require 'builder'
 require 'fileutils'
@@ -57,16 +58,16 @@ module Minitest
 
       def xml_message_for(test)
         xml = XmlMarkup.new(indent: 2, margin: 2)
-        error = test.failure
+        failure = test.failure
 
         if test.skipped? && !test.flaked?
-          xml.skipped(type: test.name)
+          xml.skipped(type: failure.error.class.name)
         elsif test.error?
-          xml.error(type: test.name, message: xml.trunc!(error.message)) do
+          xml.error(type: failure.error.class.name, message: xml.trunc!(failure.message)) do
             xml.text!(message_for(test))
           end
-        elsif test.failure
-          xml.failure(type: test.name, message: xml.trunc!(error.message)) do
+        elsif failure
+          xml.failure(type: failure.error.class.name, message: xml.trunc!(failure.message)) do
             xml.text!(message_for(test))
           end
         end

data/lib/minitest/queue/local_requeue_reporter.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'ci/queue/output_helpers'
 require 'minitest/reporters'
 

data/lib/minitest/queue/order_reporter.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'minitest/reporters'
 
 class Minitest::Queue::OrderReporter < Minitest::Reporters::BaseReporter

data/lib/minitest/queue/runner.rb

@@ -1,4 +1,6 @@
+# frozen_string_literal: true
 require 'optparse'
+require 'json'
 require 'minitest/queue'
 require 'ci/queue'
 require 'digest/md5'
@@ -59,6 +61,7 @@ module Minitest
           LocalRequeueReporter.new,
           BuildStatusRecorder.new(build: queue.build),
           JUnitReporter.new,
+          TestDataReporter.new(namespace: queue_config&.namespace),
           OrderReporter.new(path: 'log/test_order.log'),
         ]
         if queue_config.statsd_endpoint
@@ -70,7 +73,7 @@ module Minitest
         trap('INT') { Minitest.queue.shutdown! }
 
         if queue.rescue_connection_errors { queue.exhausted? }
-          puts green("All tests were ran already")
+          puts green('All tests were ran already')
         else
           load_tests
           populate_queue
@@ -78,6 +81,41 @@ module Minitest
         # Let minitest's at_exit hook trigger
       end
 
+      def grind_command
+        invalid_usage!('No list to grind provided') if grind_list.nil?
+        invalid_usage!('No grind count provided') if grind_count.nil?
+
+        set_load_path
+
+        queue_config.build_id = queue_config.build_id + '-grind'
+        queue_config.grind_count = grind_count
+
+        reporter_queue = CI::Queue::Redis::Grind.new(queue_url, queue_config)
+        test_time_record = CI::Queue::Redis::TestTimeRecord.new(queue_url, queue_config)
+
+        Minitest.queue = queue
+        reporters = [
+          GrindRecorder.new(build: reporter_queue.build),
+          TestDataReporter.new(namespace: queue_config&.namespace),
+          TestTimeRecorder.new(build: test_time_record)
+        ]
+        if queue_config.statsd_endpoint
+          reporters << Minitest::Reporters::StatsdReporter.new(statsd_endpoint: queue_config.statsd_endpoint)
+        end
+        Minitest.queue_reporters = reporters
+
+        trap('TERM') { Minitest.queue.shutdown! }
+        trap('INT') { Minitest.queue.shutdown! }
+
+        load_tests
+
+        @queue = CI::Queue::Grind.new(grind_list, queue_config)
+        Minitest.queue = queue
+        populate_queue
+
+        # Let minitest's at_exit hook trigger
+      end
+
       def bisect_command
         invalid_usage! "Missing the FAILING_TEST argument." unless queue_config.failing_test
 
@@ -144,14 +182,44 @@ module Minitest
         end
 
         reporter = BuildStatusReporter.new(build: supervisor.build)
+
+        if queue_config.failure_file
+          failures = reporter.error_reports.map(&:to_h).to_json
+          File.write(queue_config.failure_file, failures)
+        end
+
         reporter.report
         exit! reporter.success? ? 0 : 1
       end
 
+      def report_grind_command
+        queue_config.build_id = queue_config.build_id + '-grind'
+        @queue = CI::Queue::Redis::Grind.new(queue_url, queue_config)
+
+        supervisor = begin
+          queue.supervisor
+        rescue NotImplementedError => error
+          abort! error.message
+        end
+
+        grind_reporter = GrindReporter.new(build: supervisor.build)
+        grind_reporter.report
+
+        test_time_record = CI::Queue::Redis::TestTimeRecord.new(queue_url, queue_config)
+        test_time_reporter = Minitest::Queue::TestTimeReporter.new(
+          build: test_time_record,
+          limit: queue_config.max_test_duration,
+          percentile: queue_config.max_test_duration_percentile,
+        )
+        test_time_reporter.report
+
+        exit! grind_reporter.success? && test_time_reporter.success? ? 0 : 1
+      end
+
       private
 
       attr_reader :queue_config, :options, :command, :argv
-      attr_accessor :queue, :queue_url, :load_paths
+      attr_accessor :queue, :queue_url, :grind_list, :grind_count, :load_paths
 
       def display_warnings(build)
         build.pop_warnings.each do |type, attributes|
@@ -160,7 +228,7 @@ module Minitest
             puts reopen_previous_step
             puts yellow(
               "[WARNING] #{attributes[:test]} was picked up by another worker because it didn't complete in the allocated #{attributes[:timeout]} seconds.\n" \
-              "You may want to either optimize this test of bump ci-queue timeout.\n" \
+              "You may want to either optimize this test or bump ci-queue timeout.\n" \
               "It's also possible that the worker that was processing it was terminated without being able to report back.\n"
             )
           end
@@ -217,31 +285,47 @@ module Minitest
           opts.separator "GLOBAL OPTIONS"
 
 
-          help = split_heredoc(<<-EOS)
+          help = <<~EOS
             URL of the queue, e.g. redis://example.com.
             Defaults to $CI_QUEUE_URL if set.
           EOS
           opts.separator ""
-          opts.on('--queue URL', *help) do |url|
+          opts.on('--queue URL', help) do |url|
             self.queue_url = url
           end
 
-          help = split_heredoc(<<-EOS)
+          help = <<~EOS
+            Path to the file that includes the list of tests to grind.
+          EOS
+          opts.separator ""
+          opts.on('--grind-list PATH', help) do |url|
+            self.grind_list = url
+          end
+
+          help = <<~EOS
+            Count defines how often each test in the grind list is going to be run.
+          EOS
+          opts.separator ""
+          opts.on('--grind-count COUNT', help) do |count|
+            self.grind_count = count.to_i
+          end
+
+          help = <<~EOS
             Unique identifier for the workload. All workers working on the same suite of tests must have the same build identifier.
             If the build is tried again, or another revision is built, this value must be different.
             It's automatically inferred on Buildkite, CircleCI, Heroku CI, and Travis.
           EOS
           opts.separator ""
-          opts.on('--build BUILD_ID', *help) do |build_id|
+          opts.on('--build BUILD_ID', help) do |build_id|
             queue_config.build_id = build_id
           end
 
-          help = split_heredoc(<<-EOS)
+          help = <<~EOS
             Optional. Sets a prefix for the build id in case a single CI build runs multiple independent test suites.
               Example: --namespace integration
           EOS
           opts.separator ""
-          opts.on('--namespace NAMESPACE', *help) do |namespace|
+          opts.on('--namespace NAMESPACE', help) do |namespace|
             queue_config.namespace = namespace
           end
 
@@ -250,68 +334,112 @@ module Minitest
           opts.separator ""
           opts.separator "    run [TEST_FILES...]: Participate in leader election, and then work off the test queue."
 
-          help = split_heredoc(<<-EOS)
+          help = <<~EOS
             Specify a timeout after which if a test haven't completed, it will be picked up by another worker.
             It is very important to set this vlaue higher than the slowest test in the suite, otherwise performance will be impacted.
             Defaults to 30 seconds.
           EOS
           opts.separator ""
-          opts.on('--timeout TIMEOUT', *help) do |timeout|
-            queue_config.timeout = Float(timeout)
+          opts.on('--timeout TIMEOUT', Float, help) do |timeout|
+            queue_config.timeout = timeout
           end
 
-          help = split_heredoc(<<-EOS)
+          help = <<~EOS
             Specify $LOAD_PATH directory, similar to Ruby's -I
           EOS
           opts.separator ""
-          opts.on('-IPATHS', *help) do |paths|
+          opts.on('-IPATHS', help) do |paths|
             self.load_paths = paths
           end
 
-          help = split_heredoc(<<-EOS)
+          help = <<~EOS
             Sepcify a seed used to shuffle the test suite.
             On Buildkite, CircleCI, Heroku CI, and Travis, the commit revision will be used by default.
           EOS
           opts.separator ""
-          opts.on('--seed SEED', *help) do |seed|
+          opts.on('--seed SEED', help) do |seed|
             queue_config.seed = seed
           end
 
-          help = split_heredoc(<<-EOS)
+          help = <<~EOS
             A unique identifier for this worker, It must be consistent to allow retries.
             If not specified, retries won't be available.
             It's automatically inferred on Buildkite, Heroku CI, and CircleCI.
           EOS
           opts.separator ""
-          opts.on('--worker WORKER_ID', *help) do |worker_id|
+          opts.on('--worker WORKER_ID', help) do |worker_id|
             queue_config.worker_id = worker_id
           end
 
-          help = split_heredoc(<<-EOS)
+          help = <<~EOS
             Defines how many time a single test can be requeued.
             Defaults to 0.
           EOS
           opts.separator ""
-          opts.on('--max-requeues MAX', *help) do |max|
-            queue_config.max_requeues = Integer(max)
+          opts.on('--max-requeues MAX', Integer, help) do |max|
+            queue_config.max_requeues = max
+          end
+
+          help = <<~EOS
+            Defines how long ci-queue should maximally run in seconds
+            Defaults to none.
+          EOS
+          opts.separator ""
+          opts.on('--max-duration SECONDS', Integer, help) do |max|
+            queue_config.max_duration = max
           end
 
-          help = split_heredoc(<<-EOS)
+          help = <<~EOS
             Defines how many requeues can happen overall, based on the test suite size. e.g 0.05 for 5%.
             Defaults to 0.
           EOS
           opts.separator ""
-          opts.on('--requeue-tolerance RATIO', *help) do |ratio|
-            queue_config.requeue_tolerance = Float(ratio)
+          opts.on('--requeue-tolerance RATIO', Float, help) do |ratio|
+            queue_config.requeue_tolerance = ratio
+          end
+
+          help = <<~EOS
+            Defines a file where the test failures are written to in the json format.
+            Defaults to disabled.
+          EOS
+          opts.separator ""
+          opts.on('--failure-file FILE', help) do |file|
+            queue_config.failure_file = file
           end
 
-          help = split_heredoc(<<-EOS)
+          help = <<~EOS
             Defines after how many consecutive failures the worker will be considered unhealthy and terminate itself.
             Defaults to disabled.
           EOS
           opts.separator ""
-          opts.on('--max-consecutive-failures MAX', *help) do |max|
-            queue_config.max_consecutive_failures = Integer(max)
+          opts.on('--max-consecutive-failures MAX', Integer, help) do |max|
+            queue_config.max_consecutive_failures = max
+          end
+
+          help = <<~EOS
+            Set the time limit of the execution time from grinds on a given test.
+            For example, when max-test-duration is set to 10 and
+            max-test-duration-percentile is set to 0.5, the test's median execution time during a grind must be
+            lower than 10 milliseconds.
+            The unit is milliseconds and decimal is allowed.
+            Defaults to disabled.
+          EOS
+          opts.on('--max-test-duration LIMIT_IN_MILLISECONDS', Float, help) do |limit|
+            queue_config.max_test_duration = limit
+          end
+
+          help = <<~EOS
+            The percentile for max-test-duration. For example, when max-test-duration is set to 10 and
+            max-test-duration-percentile is set to 0.5, the test's median execution time during a grind must be
+            lower than 10 milliseconds.
+            The percentile must be within the range 0 < percentile <= 1.
+            Defaults to 0.5 (50th percentile).
+          EOS
+          opts.on('--max-test-duration-percentile LIMIT_IN_MILLISECONDS', Float, help) do |percentile|
+            queue_config.max_test_duration_percentile = percentile
+            if queue_config.max_test_duration_percentile <= 0 || queue_config.max_test_duration_percentile > 1
+              raise OptionParser::ParseError.new("must be within range (0, 1]")
+            end
           end
 
           opts.separator ""
@@ -322,7 +450,7 @@ module Minitest
 
           opts.separator ""
           opts.separator "    bisect: bisect a test suite to find global state leaks."
-          help = split_heredoc(<<-EOS)
+          help = <<~EOS
             The identifier of the failing test.
           EOS
           opts.separator ""
@@ -332,10 +460,6 @@ module Minitest
         end
       end
 
-      def split_heredoc(string)
-        string.lines.map(&:strip)
-      end
-
       def ordering_seed
         if queue_config.seed
           Random.new(Digest::MD5.hexdigest(queue_config.seed).to_i(16))