ci-queue 0.89.0 → 0.91.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 87b11dfe365c8f57a79ef1d5b65e4f422a1fce2bd94d51b0d653d10bfb95120a
-  data.tar.gz: 1a530c84f9883b85f0784b33e67044f5a325a5815dc06d4b89872b76d1127a74
+  metadata.gz: e5cec1e550b8f99b2e3373bd4d6b2b6de207e2c35dc454b5de50beaf3f2984f9
+  data.tar.gz: b01254ad4c25f5924d2996e57ac38c28e20a07abd4b032d4a19dcfdbb5fb67fe
 SHA512:
-  metadata.gz: 25e7f86d44fc3428e6f1d08430a3940348ae9d2d2561c190894d7a6e1b197d00b22a97fefcb5ade8462d2ccc0e14255ff9d4f10571e5f1f99dd8758b45d8a066
-  data.tar.gz: 0c676475b941684ad92d43088612e31adb597f2f2c22f383e55c8d2dbc2cfa68646e013a58000191f5fecb16aee8151b5ce8f14cd97fe2d9ca2f11c3fd57cf3f
+  metadata.gz: 62db88f6caceddd5d560d1a56a96588e2eff82b20be167741a8eaee0707dfa271a92404911dc4833896346700c14df317ccbf1220dcd61c8bf56ba77edadeb5c
+  data.tar.gz: e35765237703ae8f169f79ce70f334bde33550fd8bba18fac59a72f8c0cb9ac859e02c3618b4ff2847210d5498c37651de847c1af2368186044982a3d8d51b2a

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ci-queue (0.89.0)
+    ci-queue (0.91.0)
       logger
 
 GEM

data/lib/ci/queue/configuration.rb

@@ -6,6 +6,7 @@ module CI
       attr_accessor :requeue_tolerance, :namespace, :failing_test, :statsd_endpoint
       attr_accessor :max_test_duration, :max_test_duration_percentile, :track_test_duration
       attr_accessor :max_test_failed, :redis_ttl, :warnings_file, :debug_log, :max_missed_heartbeat_seconds
+      attr_writer :heartbeat_max_test_duration
       attr_accessor :lazy_load, :lazy_load_stream_batch_size
       attr_writer :lazy_load_streaming_timeout
       attr_accessor :lazy_load_test_helpers
@@ -57,7 +58,7 @@ module CI
         grind_count: nil, max_duration: nil, failure_file: nil, max_test_duration: nil,
         max_test_duration_percentile: 0.5, track_test_duration: false, max_test_failed: nil,
         queue_init_timeout: nil, redis_ttl: 8 * 60 * 60, report_timeout: nil, inactive_workers_timeout: nil,
-        export_flaky_tests_file: nil, warnings_file: nil, debug_log: nil, max_missed_heartbeat_seconds: nil,
+        export_flaky_tests_file: nil, warnings_file: nil, debug_log: nil, max_missed_heartbeat_seconds: nil, heartbeat_max_test_duration: nil,
         lazy_load: false, lazy_load_stream_batch_size: nil, lazy_load_streaming_timeout: nil, lazy_load_test_helpers: nil,
         skip_stale_tests: false)
         @build_id = build_id
@@ -86,6 +87,7 @@ module CI
         @warnings_file = warnings_file
         @debug_log = debug_log
         @max_missed_heartbeat_seconds = max_missed_heartbeat_seconds
+        @heartbeat_max_test_duration = heartbeat_max_test_duration
         @lazy_load = lazy_load
         @lazy_load_stream_batch_size = lazy_load_stream_batch_size || 5_000
         @lazy_load_streaming_timeout = lazy_load_streaming_timeout
@@ -99,6 +101,11 @@ module CI
         @lazy_load_test_helpers.split(',').map(&:strip)
       end
 
+      def retry?
+        ENV.fetch("BUILDKITE_RETRY_COUNT", "0").to_i > 0 ||
+          ENV["SEMAPHORE_PIPELINE_RERUN"] == "true"
+      end
+
       def queue_init_timeout
         @queue_init_timeout || timeout
       end
@@ -148,6 +155,10 @@ module CI
         @inactive_workers_timeout || timeout
       end
 
+      def heartbeat_max_test_duration
+        @heartbeat_max_test_duration || (timeout * 10 if max_missed_heartbeat_seconds)
+      end
+
       def max_consecutive_failures=(max)
         if max
           @circuit_breakers << CircuitBreaker::MaxConsecutiveFailures.new(max_consecutive_failures: max)

data/lib/ci/queue/redis/base.rb

@@ -63,7 +63,7 @@ module CI
         def with_heartbeat(id, lease: nil)
           if heartbeat_enabled?
             ensure_heartbeat_thread_alive!
-            heartbeat_state.set(:tick, id, lease)
+            heartbeat_state.set(:tick, id, lease, Process.clock_gettime(Process::CLOCK_MONOTONIC))
           end
 
           yield
@@ -386,16 +386,32 @@ module CI
           Thread.current.name = "CI::Queue#heartbeat"
           Thread.current.abort_on_exception = true
 
+          capped = false
+
           loop do
             command = heartbeat_state.wait(1) # waits for max 1 second but wakes up immediately if we receive a command
 
             case command&.first
             when :tick
-              # command = [:tick, entry_id, lease_id]
+              next if capped
+
+              max_duration = config.heartbeat_max_test_duration
+              if max_duration
+                # command = [:tick, entry_id, lease_id, started_at]
+                # Use the absolute start time from when with_heartbeat was called so that
+                # the elapsed calculation is not skewed by heartbeat thread startup delay.
+                elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - command[3]
+                if elapsed >= max_duration
+                  capped = true
+                  next
+                end
+              end
+
+              # command = [:tick, entry_id, lease_id, started_at]
               heartbeat_process.tick!(command[1], command[2])
             when :reset
               # Test finished, stop ticking until next test starts
-              nil
+              capped = false
             when :stop
               break
             end

data/lib/ci/queue/redis/supervisor.rb

@@ -39,6 +39,23 @@ module CI
             yield if block_given?
           end
 
+          # On retry runs (BUILDKITE_RETRY_COUNT > 0), the main queue is already
+          # exhausted from the original run. A retry worker may have found unresolved
+          # failures via the error-reports fallback and be running them via the Retry
+          # queue — but those tests are NOT in the Redis running set so active_workers?
+          # returns false and the loop above exits immediately.
+          #
+          # Wait up to inactive_workers_timeout for retry workers to clear error-reports.
+          # This prevents the summary from canceling retry workers before they finish.
+          if exhausted? && config.retry? && !rescue_connection_errors { build.failed_tests }.empty?
+            @time_left_with_no_workers = config.inactive_workers_timeout
+            until rescue_connection_errors { build.failed_tests }.empty? ||
+                @time_left_with_no_workers <= 0
+              sleep 1
+              @time_left_with_no_workers -= 1
+            end
+          end
+
           exhausted?
         rescue CI::Queue::Redis::LostMaster
           false

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.89.0'
+    VERSION = '0.91.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue/build_status_reporter.rb

@@ -267,8 +267,16 @@ module Minitest
       end
 
       def aggregates
-        success = failures.zero? && errors.zero?
-        failures_count = "#{failures} failures, #{errors} errors,"
+        # error-reports is authoritative when workers die before flushing per-test stats.
+        # Floor the displayed count so the summary line is never misleadingly green.
+        known_error_count = error_reports.size
+        effective_total = [failures + errors, known_error_count].max
+        success = effective_total.zero?
+        failures_count = if failures + errors >= known_error_count
+          "#{failures} failures, #{errors} errors,"
+        else
+          "#{effective_total} failures,"
+        end
 
         step([
           'Ran %d tests, %d assertions,' % [progress, assertions],

data/lib/minitest/queue/runner.rb

@@ -736,6 +736,16 @@ module Minitest
             queue_config.max_missed_heartbeat_seconds = time || 30
           end
 
+          help = <<~EOS
+            Maximum duration in seconds that the heartbeat will tick for a single test.
+            If a test runs longer than this, the heartbeat stops and the test entry becomes
+            eligible for reclamation by another worker.
+            Defaults to timeout * 10 when heartbeat is enabled.
+          EOS
+          opts.on("--heartbeat-max-test-duration SECONDS", Float, help) do |seconds|
+            queue_config.heartbeat_max_test_duration = seconds
+          end
+
 
           opts.on("-v", "--verbose", "Verbose. Show progress processing files.") do
             self.verbose = true

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.89.0
+  version: 0.91.0
 platform: ruby
 authors:
 - Jean Boussier
@@ -305,7 +305,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.9
+rubygems_version: 4.0.10
 specification_version: 4
 summary: Distribute tests over many workers using a queue
 test_files: []