ci-queue 0.90.0 → 0.92.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c5233596f5e87ddd8954a44a2b7c2be3d258ade0c567ff808b32d99194d89a7b
-  data.tar.gz: 1026734a1076455a5ac95821ef8a1dbba83cb8ed445720fab8ca93cdd53eb693
+  metadata.gz: add1590f9cc361b896a70a61c456eb18f3bdcf9b6fd58aaf1432aa276c2cef48
+  data.tar.gz: 5a6ce1545cefa80a46b499286fb7de228a2c7c35c507af1e10728389f1951aac
 SHA512:
-  metadata.gz: 68b5ce15acc1b90b3c75fb861ad8ea2ff6599c44d096a96ed5d018fa8d4eb4c6c20a9949950e31284436241d9d1df5c2917430eadef673872c9e4a693893ed20
-  data.tar.gz: a1ba50c8998de99d54620efd899bc1a471b0bf037a3cb9a9e056b36d45f551f9430a9bac1b4f46713c5e17426fc7e409be4b307f77071321a9f0789c0eca5ee8
+  metadata.gz: 90a0e14eb954c3fba2f54eb680a523072a518d92aa2fa56584a93568f812f60f2ba7d6a656eba0370a5be8ff5da42c03727307b854edced06ad7e15e31f2eab3
+  data.tar.gz: '0716598054eb0b32c0453debb67a47656b266d35c14bf85a42e66503216c1617868d9dd081fc738852f74a31f66d622825523b451c0e6ab3fc3fcacd287aa162'

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ci-queue (0.90.0)
+    ci-queue (0.92.0)
       logger
 
 GEM

data/lib/ci/queue/configuration.rb

@@ -6,6 +6,7 @@ module CI
       attr_accessor :requeue_tolerance, :namespace, :failing_test, :statsd_endpoint
       attr_accessor :max_test_duration, :max_test_duration_percentile, :track_test_duration
       attr_accessor :max_test_failed, :redis_ttl, :warnings_file, :debug_log, :max_missed_heartbeat_seconds
+      attr_writer :heartbeat_max_test_duration
       attr_accessor :lazy_load, :lazy_load_stream_batch_size
       attr_writer :lazy_load_streaming_timeout
       attr_accessor :lazy_load_test_helpers
@@ -57,7 +58,7 @@ module CI
         grind_count: nil, max_duration: nil, failure_file: nil, max_test_duration: nil,
         max_test_duration_percentile: 0.5, track_test_duration: false, max_test_failed: nil,
         queue_init_timeout: nil, redis_ttl: 8 * 60 * 60, report_timeout: nil, inactive_workers_timeout: nil,
-        export_flaky_tests_file: nil, warnings_file: nil, debug_log: nil, max_missed_heartbeat_seconds: nil,
+        export_flaky_tests_file: nil, warnings_file: nil, debug_log: nil, max_missed_heartbeat_seconds: nil, heartbeat_max_test_duration: nil,
         lazy_load: false, lazy_load_stream_batch_size: nil, lazy_load_streaming_timeout: nil, lazy_load_test_helpers: nil,
         skip_stale_tests: false)
         @build_id = build_id
@@ -86,6 +87,7 @@ module CI
         @warnings_file = warnings_file
         @debug_log = debug_log
         @max_missed_heartbeat_seconds = max_missed_heartbeat_seconds
+        @heartbeat_max_test_duration = heartbeat_max_test_duration
         @lazy_load = lazy_load
         @lazy_load_stream_batch_size = lazy_load_stream_batch_size || 5_000
         @lazy_load_streaming_timeout = lazy_load_streaming_timeout
@@ -153,6 +155,10 @@ module CI
         @inactive_workers_timeout || timeout
       end
 
+      def heartbeat_max_test_duration
+        @heartbeat_max_test_duration || (timeout * 10 if max_missed_heartbeat_seconds)
+      end
+
       def max_consecutive_failures=(max)
         if max
           @circuit_breakers << CircuitBreaker::MaxConsecutiveFailures.new(max_consecutive_failures: max)

data/lib/ci/queue/redis/base.rb

@@ -63,7 +63,7 @@ module CI
         def with_heartbeat(id, lease: nil)
           if heartbeat_enabled?
             ensure_heartbeat_thread_alive!
-            heartbeat_state.set(:tick, id, lease)
+            heartbeat_state.set(:tick, id, lease, Process.clock_gettime(Process::CLOCK_MONOTONIC))
           end
 
           yield
@@ -266,6 +266,11 @@ module CI
         class HeartbeatProcess
           MAX_RESTART_ATTEMPTS = 3
 
+          # Cached command marker. Passing this frozen String instead of the
+          # :tick! Symbol avoids allocating a 'tick!' String (from Symbol#to_s)
+          # on every heartbeat, which fires once per running test per worker.
+          TICK_COMMAND = 'tick!'.freeze
+
           def initialize(redis_url, zset_key, owners_key, leases_key)
             @redis_url = redis_url
             @zset_key = zset_key
@@ -314,7 +319,7 @@ module CI
           end
 
           def tick!(id, lease)
-            send_message(:tick!, id: id, lease: lease.to_s)
+            send_message(TICK_COMMAND, id: id, lease: lease.to_s)
             @restart_attempts = 0
           rescue IOError, Errno::EPIPE => error
             @restart_attempts = (@restart_attempts || 0) + 1
@@ -386,16 +391,32 @@ module CI
           Thread.current.name = "CI::Queue#heartbeat"
           Thread.current.abort_on_exception = true
 
+          capped = false
+
           loop do
             command = heartbeat_state.wait(1) # waits for max 1 second but wakes up immediately if we receive a command
 
             case command&.first
             when :tick
-              # command = [:tick, entry_id, lease_id]
+              next if capped
+
+              max_duration = config.heartbeat_max_test_duration
+              if max_duration
+                # command = [:tick, entry_id, lease_id, started_at]
+                # Use the absolute start time from when with_heartbeat was called so that
+                # the elapsed calculation is not skewed by heartbeat thread startup delay.
+                elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - command[3]
+                if elapsed >= max_duration
+                  capped = true
+                  next
+                end
+              end
+
+              # command = [:tick, entry_id, lease_id, started_at]
               heartbeat_process.tick!(command[1], command[2])
             when :reset
               # Test finished, stop ticking until next test starts
-              nil
+              capped = false
             when :stop
               break
             end

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.90.0'
+    VERSION = '0.92.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue/runner.rb

@@ -736,6 +736,16 @@ module Minitest
             queue_config.max_missed_heartbeat_seconds = time || 30
           end
 
+          help = <<~EOS
+            Maximum duration in seconds that the heartbeat will tick for a single test.
+            If a test runs longer than this, the heartbeat stops and the test entry becomes
+            eligible for reclamation by another worker.
+            Defaults to timeout * 10 when heartbeat is enabled.
+          EOS
+          opts.on("--heartbeat-max-test-duration SECONDS", Float, help) do |seconds|
+            queue_config.heartbeat_max_test_duration = seconds
+          end
+
 
           opts.on("-v", "--verbose", "Verbose. Show progress processing files.") do
             self.verbose = true

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.90.0
+  version: 0.92.0
 platform: ruby
 authors:
 - Jean Boussier
@@ -305,7 +305,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.9
+rubygems_version: 4.0.10
 specification_version: 4
 summary: Distribute tests over many workers using a queue
 test_files: []