ci-queue 0.90.0 → 0.91.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c5233596f5e87ddd8954a44a2b7c2be3d258ade0c567ff808b32d99194d89a7b
-  data.tar.gz: 1026734a1076455a5ac95821ef8a1dbba83cb8ed445720fab8ca93cdd53eb693
+  metadata.gz: e5cec1e550b8f99b2e3373bd4d6b2b6de207e2c35dc454b5de50beaf3f2984f9
+  data.tar.gz: b01254ad4c25f5924d2996e57ac38c28e20a07abd4b032d4a19dcfdbb5fb67fe
 SHA512:
-  metadata.gz: 68b5ce15acc1b90b3c75fb861ad8ea2ff6599c44d096a96ed5d018fa8d4eb4c6c20a9949950e31284436241d9d1df5c2917430eadef673872c9e4a693893ed20
-  data.tar.gz: a1ba50c8998de99d54620efd899bc1a471b0bf037a3cb9a9e056b36d45f551f9430a9bac1b4f46713c5e17426fc7e409be4b307f77071321a9f0789c0eca5ee8
+  metadata.gz: 62db88f6caceddd5d560d1a56a96588e2eff82b20be167741a8eaee0707dfa271a92404911dc4833896346700c14df317ccbf1220dcd61c8bf56ba77edadeb5c
+  data.tar.gz: e35765237703ae8f169f79ce70f334bde33550fd8bba18fac59a72f8c0cb9ac859e02c3618b4ff2847210d5498c37651de847c1af2368186044982a3d8d51b2a

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ci-queue (0.90.0)
+    ci-queue (0.91.0)
       logger
 
 GEM

data/lib/ci/queue/configuration.rb

@@ -6,6 +6,7 @@ module CI
       attr_accessor :requeue_tolerance, :namespace, :failing_test, :statsd_endpoint
       attr_accessor :max_test_duration, :max_test_duration_percentile, :track_test_duration
       attr_accessor :max_test_failed, :redis_ttl, :warnings_file, :debug_log, :max_missed_heartbeat_seconds
+      attr_writer :heartbeat_max_test_duration
       attr_accessor :lazy_load, :lazy_load_stream_batch_size
       attr_writer :lazy_load_streaming_timeout
       attr_accessor :lazy_load_test_helpers
@@ -57,7 +58,7 @@ module CI
         grind_count: nil, max_duration: nil, failure_file: nil, max_test_duration: nil,
         max_test_duration_percentile: 0.5, track_test_duration: false, max_test_failed: nil,
         queue_init_timeout: nil, redis_ttl: 8 * 60 * 60, report_timeout: nil, inactive_workers_timeout: nil,
-        export_flaky_tests_file: nil, warnings_file: nil, debug_log: nil, max_missed_heartbeat_seconds: nil,
+        export_flaky_tests_file: nil, warnings_file: nil, debug_log: nil, max_missed_heartbeat_seconds: nil, heartbeat_max_test_duration: nil,
         lazy_load: false, lazy_load_stream_batch_size: nil, lazy_load_streaming_timeout: nil, lazy_load_test_helpers: nil,
         skip_stale_tests: false)
         @build_id = build_id
@@ -86,6 +87,7 @@ module CI
         @warnings_file = warnings_file
         @debug_log = debug_log
         @max_missed_heartbeat_seconds = max_missed_heartbeat_seconds
+        @heartbeat_max_test_duration = heartbeat_max_test_duration
         @lazy_load = lazy_load
         @lazy_load_stream_batch_size = lazy_load_stream_batch_size || 5_000
         @lazy_load_streaming_timeout = lazy_load_streaming_timeout
@@ -153,6 +155,10 @@ module CI
         @inactive_workers_timeout || timeout
       end
 
+      def heartbeat_max_test_duration
+        @heartbeat_max_test_duration || (timeout * 10 if max_missed_heartbeat_seconds)
+      end
+
       def max_consecutive_failures=(max)
         if max
           @circuit_breakers << CircuitBreaker::MaxConsecutiveFailures.new(max_consecutive_failures: max)

data/lib/ci/queue/redis/base.rb

@@ -63,7 +63,7 @@ module CI
         def with_heartbeat(id, lease: nil)
           if heartbeat_enabled?
             ensure_heartbeat_thread_alive!
-            heartbeat_state.set(:tick, id, lease)
+            heartbeat_state.set(:tick, id, lease, Process.clock_gettime(Process::CLOCK_MONOTONIC))
           end
 
           yield
@@ -386,16 +386,32 @@ module CI
           Thread.current.name = "CI::Queue#heartbeat"
           Thread.current.abort_on_exception = true
 
+          capped = false
+
           loop do
             command = heartbeat_state.wait(1) # waits for max 1 second but wakes up immediately if we receive a command
 
             case command&.first
             when :tick
-              # command = [:tick, entry_id, lease_id]
+              next if capped
+
+              max_duration = config.heartbeat_max_test_duration
+              if max_duration
+                # command = [:tick, entry_id, lease_id, started_at]
+                # Use the absolute start time from when with_heartbeat was called so that
+                # the elapsed calculation is not skewed by heartbeat thread startup delay.
+                elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - command[3]
+                if elapsed >= max_duration
+                  capped = true
+                  next
+                end
+              end
+
+              # command = [:tick, entry_id, lease_id, started_at]
               heartbeat_process.tick!(command[1], command[2])
             when :reset
               # Test finished, stop ticking until next test starts
-              nil
+              capped = false
             when :stop
               break
             end

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.90.0'
+    VERSION = '0.91.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue/runner.rb

@@ -736,6 +736,16 @@ module Minitest
             queue_config.max_missed_heartbeat_seconds = time || 30
           end
 
+          help = <<~EOS
+            Maximum duration in seconds that the heartbeat will tick for a single test.
+            If a test runs longer than this, the heartbeat stops and the test entry becomes
+            eligible for reclamation by another worker.
+            Defaults to timeout * 10 when heartbeat is enabled.
+          EOS
+          opts.on("--heartbeat-max-test-duration SECONDS", Float, help) do |seconds|
+            queue_config.heartbeat_max_test_duration = seconds
+          end
+
 
           opts.on("-v", "--verbose", "Verbose. Show progress processing files.") do
             self.verbose = true

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.90.0
+  version: 0.91.0
 platform: ruby
 authors:
 - Jean Boussier
@@ -305,7 +305,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.9
+rubygems_version: 4.0.10
 specification_version: 4
 summary: Distribute tests over many workers using a queue
 test_files: []