ci-queue 0.64.0 → 0.66.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: af2b3f83ebe968c7f35a010a76a4676f13c6a0e7bf49bfd4f6d6a25ff04270f8
-  data.tar.gz: f647f458fd19dc6117c5ccd5b085b505c0b4f75c68f4ce0e0c0402bd4ca0d909
+  metadata.gz: b78efa3839e0fdc3a3998ef7d664bab01f28d0cdde96c28b790f9c912964d33a
+  data.tar.gz: 4bc014eb8401ce329be509dc76f449a4d80696667563e3236c5f9634514b7df0
 SHA512:
-  metadata.gz: 227d2721922584d7130c7aef4eb88b42404ec62265e43f35e996e96aaf602904b0403d07eee436b2f6e228ed5755de46c994e1c457a975c7567ec53be46db988
-  data.tar.gz: 7b46d8842c805c1b36e26e82d10eab61183e8db562c1a0ac3714cb99006ce657f8b46f8812df029db98bc6b8fb7c4f37d8cda29b4ff8361409dcd2fef633baa4
+  metadata.gz: be5f84376a0a6bc93776eaafa4a725e48061e18ea13d79c407cb599b665201031fc189ae91cd89607f3c2fa062eb388caaa554411405cc7adf6b073610ee078f
+  data.tar.gz: 47057ca6d1b2d35eb0068be06330802a0ce614744f8498d069ef9bc1ffdd499a8a38861a3bfa7aa62e2b10becd0be5c99306026938c7ca704ede42396109367c

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ci-queue (0.64.0)
+    ci-queue (0.66.0)
       logger
 
 GEM

data/lib/ci/queue/configuration.rb

@@ -25,7 +25,14 @@ module CI
 
         def load_flaky_tests(path)
           return [] unless path
-          ::File.readlines(path).map(&:chomp).to_set
+          if ::File.extname(path) == ".xml"
+            require 'rexml/document'
+            REXML::Document.new(::File.read(path)).elements.to_a("//testcase").map do |element|
+              "#{element.attributes['classname']}##{element.attributes['name']}"
+            end.to_set
+          else
+            ::File.readlines(path).map(&:chomp).to_set
+          end
         rescue SystemCallError
           []
         end

data/lib/ci/queue/redis/base.rb

@@ -11,6 +11,8 @@ module CI
           ::SocketError, # https://github.com/redis/redis-rb/pull/631
         ].freeze
 
+        DEFAULT_TIMEOUT = 2
+
         module RedisInstrumentation
           def call(command, redis_config)
             logger = redis_config.custom[:debug_log]
@@ -45,9 +47,10 @@ module CI
               # ci-queue should not contain any sensitive data, so we can just disable the verification.
               ssl_params: { verify_mode: OpenSSL::SSL::VERIFY_NONE },
               custom: custom_config,
+              timeout: DEFAULT_TIMEOUT,
             )
           else
-            @redis = ::Redis.new(url: redis_url)
+            @redis = ::Redis.new(url: redis_url, timeout: DEFAULT_TIMEOUT)
           end
         end
 
@@ -133,11 +136,15 @@ module CI
           redis.zcard(key('running'))
         end
 
-        def to_a
+        def test_ids
           redis.multi do |transaction|
             transaction.lrange(key('queue'), 0, -1)
             transaction.zrange(key('running'), 0, -1)
-          end.flatten.reverse.map { |k| index.fetch(k) }
+          end.flatten
+        end
+
+        def to_a
+          test_ids.reverse.map { |k| index.fetch(k) }
         end
 
         def progress

data/lib/ci/queue/redis/supervisor.rb

@@ -24,29 +24,28 @@ module CI
 
           yield if block_given?
 
-          time_left = config.report_timeout - duration.to_i
-          time_left_with_no_workers = config.inactive_workers_timeout
-          until exhausted? || time_left <= 0 || max_test_failed? || time_left_with_no_workers <= 0
-            time_left -= 1
+          @time_left = config.report_timeout - duration.to_i
+          @time_left_with_no_workers = config.inactive_workers_timeout
+          until exhausted? || @time_left <= 0 || max_test_failed? || @time_left_with_no_workers <= 0
+            @time_left -= 1
             sleep 1
 
             if active_workers?
-              time_left_with_no_workers = config.inactive_workers_timeout
+              @time_left_with_no_workers = config.inactive_workers_timeout
             else
-              time_left_with_no_workers -= 1
+              @time_left_with_no_workers -= 1
             end
 
             yield if block_given?
           end
 
-          puts "Aborting, timed out." if time_left <= 0 && !exhausted?
-          puts "Aborting, it seems all workers died." if time_left_with_no_workers <= 0 && !exhausted?
-
           exhausted?
         rescue CI::Queue::Redis::LostMaster
           false
         end
 
+        attr_reader :time_left, :time_left_with_no_workers
+
         private
 
         def active_workers?

data/lib/ci/queue/redis/worker.rb

@@ -201,7 +201,14 @@ module CI
         def push(tests)
           @total = tests.size
 
-          if @master = redis.setnx(key('master-status'), 'setup')
+          # We set a unique value (worker_id) and read it back to make "SET if Not eXists" idempotent in case of a retry.
+          value = key('setup', worker_id)
+          _, status = redis.pipelined do |pipeline|
+            pipeline.set(key('master-status'), value, nx: true)
+            pipeline.get(key('master-status'))
+          end
+
+          if @master = (value == status)
             puts "Worker electected as leader, pushing #{@total} tests to the queue."
             puts
 

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.64.0'
+    VERSION = '0.66.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue/build_status_reporter.rb

@@ -4,8 +4,91 @@ module Minitest
     class BuildStatusReporter < Minitest::Reporters::BaseReporter
       include ::CI::Queue::OutputHelpers
 
-      def initialize(build:, **options)
-        @build = build
+      class JUnitReporter
+        def initialize(file, error_reports)
+          @file = file
+          @error_reports = error_reports
+        end
+
+        def write
+          File.open(@file, 'w+') do |file|
+            format_document(generate_document(@error_reports), file)
+          end
+        end
+
+        private
+
+        def generate_document(error_reports)
+          suites = error_reports.group_by { |error_report| error_report.test_suite }
+
+          doc = REXML::Document.new(nil, {
+            :prologue_quote => :quote,
+            :attribute_quote => :quote,
+          })
+          doc << REXML::XMLDecl.new('1.1', 'utf-8')
+
+          testsuites = doc.add_element('testsuites')
+          suites.each do |suite, error_reports|
+            add_tests_to(testsuites, suite, error_reports)
+          end
+
+          doc
+        end
+
+        def format_document(doc, io)
+          formatter = REXML::Formatters::Pretty.new
+          formatter.write(doc, io)
+          io << "\n"
+        end
+
+        def add_tests_to(testsuites, suite, error_reports)
+          testsuite = testsuites.add_element(
+            'testsuite',
+            'name' => suite,
+            'filepath' => Minitest::Queue.relative_path(error_reports.first.test_file),
+            'tests' => error_reports.count,
+          )
+
+          error_reports.each do |error_report|
+            attributes = {
+              'name' => error_report.test_name,
+              'classname' => error_report.test_suite,
+            }
+            attributes['lineno'] = error_report.test_line
+
+            testcase = testsuite.add_element('testcase', attributes)
+            add_xml_message_for(testcase, error_report)
+          rescue REXML::ParseException, RuntimeError => error
+            puts error
+          end
+        end
+
+        def add_xml_message_for(testcase, error_report)
+          failure = testcase.add_element('failure', 'type' => error_report.error_class, 'message' => truncate_message(error_report.to_s))
+          failure.add_text(REXML::CData.new(message_for(error_report)))
+        end
+
+        def truncate_message(message)
+          message.lines.first.chomp.gsub(/\e\[[^m]+m/, '')
+        end
+
+        def project_root_path_matcher
+          @project_root_path_matcher ||= %r{(?<=\s)#{Regexp.escape(Minitest::Queue.project_root)}/}
+        end
+
+        def message_for(error_report)
+          suite = error_report.test_suite
+          name = error_report.test_name
+          error = error_report.to_s
+
+          message_with_relative_paths = error.gsub(project_root_path_matcher, '')
+          "\nFailure:\n#{name}(#{suite}) [#{Minitest::Queue.relative_path(error_report.test_file)}]:\n#{message_with_relative_paths}\n"
+        end
+      end
+
+      def initialize(supervisor:, **options)
+        @supervisor = supervisor
+        @build = supervisor.build
         super(options)
       end
 
@@ -26,23 +109,46 @@ module Minitest
       end
 
       def report
+        if requeued_tests.to_a.any?
+          step("Requeued #{requeued_tests.size} tests")
+          requeued_tests.to_a.sort.each do |test_id, count|
+            puts yellow("REQUEUE")
+            puts "#{test_id} (requeued #{count} times)"
+            puts ""
+          end
+        end
+
         puts aggregates
-        errors = error_reports
-        puts errors
 
-        requeued_tests.to_a.sort.each do |test_id, count|
-          puts yellow("REQUEUE")
-          puts "#{test_id} (requeued #{count} times)"
-          puts ""
+        if supervisor.time_left.to_i <= 0
+          puts red("Timed out waiting for tests to be executed.")
+
+          remaining_tests = supervisor.test_ids
+          remaining_tests.first(10).each do |id|
+            puts "  #{id}"
+          end
+
+          if remaining_tests.size > 10
+            puts "  ..."
+          end
+        elsif supervisor.time_left_with_no_workers.to_i <= 0
+          puts red("All workers died.")
+        elsif supervisor.max_test_failed?
+          puts red("Encountered too many failed tests. Test run was ended early.")
         end
 
+        puts
+
+        errors = error_reports
+        puts errors
+
         build.worker_errors.to_a.sort.each do |worker_id, error|
           puts red("Worker #{worker_id } crashed")
           puts error
           puts ""
         end
 
-        errors.empty?
+        success?
       end
 
       def success?
@@ -83,16 +189,22 @@ module Minitest
       end
 
       def write_failure_file(file)
-        File.write(file, error_reports.map(&:to_h).to_json)
+        File.open(file, 'w') do |f|
+          JSON.dump(error_reports.map(&:to_h), f)
+        end
+        xml_file = File.join(File.dirname(file), "#{File.basename(file, File.extname(file))}.xml")
+        JUnitReporter.new(xml_file, error_reports).write
       end
 
       def write_flaky_tests_file(file)
-        File.write(file, flaky_reports.to_json)
+        File.open(file, 'w') do |f|
+          JSON.dump(flaky_reports, f)
+        end
       end
 
       private
 
-      attr_reader :build
+      attr_reader :build, :supervisor
 
       def aggregates
         success = failures.zero? && errors.zero?

data/lib/minitest/queue/error_report.rb

@@ -54,6 +54,10 @@ module Minitest
         @data[:test_name]
       end
 
+      def error_class
+        @data[:error_class]
+      end
+
       def test_and_module_name
         @data[:test_and_module_name]
       end

data/lib/minitest/queue/runner.rb

@@ -257,23 +257,16 @@ module Minitest
           end
 
           unless supervisor.exhausted?
-            reporter = BuildStatusReporter.new(build: supervisor.build)
+            reporter = BuildStatusReporter.new(supervisor: supervisor)
             reporter.report
             reporter.write_failure_file(queue_config.failure_file) if queue_config.failure_file
             reporter.write_flaky_tests_file(queue_config.export_flaky_tests_file) if queue_config.export_flaky_tests_file
 
-            msg = "#{supervisor.size} tests weren't run."
-
-            if supervisor.max_test_failed?
-              puts('Encountered too many failed tests. Test run was ended early.')
-              abort!(msg)
-            else
-              abort!(msg)
-            end
+            abort!("#{supervisor.size} tests weren't run.")
           end
         end
 
-        reporter = BuildStatusReporter.new(build: supervisor.build)
+        reporter = BuildStatusReporter.new(supervisor: supervisor)
         reporter.write_failure_file(queue_config.failure_file) if queue_config.failure_file
         reporter.write_flaky_tests_file(queue_config.export_flaky_tests_file) if queue_config.export_flaky_tests_file
         reporter.report
@@ -330,7 +323,9 @@ module Minitest
         warnings = build.pop_warnings.map do |type, attributes|
           attributes.merge(type: type)
         end.compact
-        File.write(queue_config.warnings_file, warnings.to_json)
+        File.open(queue_config.warnings_file, 'w') do |f|
+          JSON.dump(warnings, f)
+        end
       end
 
       def run_tests_in_fork(queue)

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.64.0
+  version: 0.66.0
 platform: ruby
 authors:
 - Jean Boussier
 bindir: exe
 cert_chain: []
-date: 2025-03-12 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logger
@@ -254,7 +254,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.5
+rubygems_version: 3.6.8
 specification_version: 4
 summary: Distribute tests over many workers using a queue
 test_files: []