ci-queue 0.42.0 → 0.43.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fb0b6d2c1537200e0a3540652972d25456cda0fe2115a65a884c8f5feeb17ff7
-  data.tar.gz: 7ac240806fdf6f16edfa314c59dccb22b379dbc2795d23d0c4ad19f4a46956d1
+  metadata.gz: 9fafefb68f5e00faba6e1c19944c0289bd0e7c5ca4d67090605fea402f2b04f3
+  data.tar.gz: fe85dc8a004f45a54203eac3e4294e5348bca9d8920b70ad2427fed9f5f26776
 SHA512:
-  metadata.gz: feaea129a7c672e57372e6475dc0fdc049c1e8b7815994b9ece3b4db0cbf6970ba19c222ed6bf9a652c7e68c7781e46edbd45e0eaa0da2b4e4d31b3856980aed
-  data.tar.gz: e2b9112e41b223c85b0acc20a7129d2ec517a816e73a42bea381ce696af8821dbaee1c5151d5002784630b1c0104b6c342d91b6273dd12de01c063d5bef5b1fe
+  metadata.gz: a190ef07194b9cbb9c880de74abd9e54ede4d3e52480e26ba20c16a80720fafb66ce5933bc21df4213e7cefc9abb5f3893b75e46fb7e8327ee6e11b8a5e26de4
+  data.tar.gz: 5b3a127d41fde0f8094878e8cfcd088eb2d1a32fc86798b98adafbf06caa19e201631899fa4aec9c0afa8d8a9174215d529dd1dfdf637e4e9122dfb93f99d741

data/lib/ci/queue/configuration.rb

@@ -5,7 +5,7 @@ module CI
       attr_accessor :timeout, :worker_id, :max_requeues, :grind_count, :failure_file, :export_flaky_tests_file
       attr_accessor :requeue_tolerance, :namespace, :failing_test, :statsd_endpoint
       attr_accessor :max_test_duration, :max_test_duration_percentile, :track_test_duration
-      attr_accessor :max_test_failed, :redis_ttl, :warnings_file, :debug_log
+      attr_accessor :max_test_failed, :redis_ttl, :warnings_file, :debug_log, :max_missed_heartbeat_seconds
       attr_reader :circuit_breakers
       attr_writer :seed, :build_id
       attr_writer :queue_init_timeout, :report_timeout, :inactive_workers_timeout
@@ -37,7 +37,7 @@ module CI
         grind_count: nil, max_duration: nil, failure_file: nil, max_test_duration: nil,
         max_test_duration_percentile: 0.5, track_test_duration: false, max_test_failed: nil,
         queue_init_timeout: nil, redis_ttl: 8 * 60 * 60, report_timeout: nil, inactive_workers_timeout: nil,
-        export_flaky_tests_file: nil, warnings_file: nil, debug_log: nil)
+        export_flaky_tests_file: nil, warnings_file: nil, debug_log: nil, max_missed_heartbeat_seconds: nil)
         @build_id = build_id
         @circuit_breakers = [CircuitBreaker::Disabled]
         @failure_file = failure_file
@@ -63,6 +63,7 @@ module CI
         @export_flaky_tests_file = export_flaky_tests_file
         @warnings_file = warnings_file
         @debug_log = debug_log
+        @max_missed_heartbeat_seconds = max_missed_heartbeat_seconds
       end
 
       def queue_init_timeout

data/lib/ci/queue/redis/base.rb

@@ -35,7 +35,7 @@ module CI
               url: redis_url,
               # Booting a CI worker is costly, so in case of a Redis blip,
               # it makes sense to retry for a while before giving up.
-              reconnect_attempts: [0, 0, 0.1, 0.5, 1, 3, 5],
+              reconnect_attempts: reconnect_attempts,
               middlewares: custom_middlewares,
               custom: custom_config,
             )
@@ -44,6 +44,43 @@ module CI
           end
         end
 
+        def reconnect_attempts
+          return [] if ENV["CI_QUEUE_DISABLE_RECONNECT_ATTEMPTS"]
+
+          [0, 0, 0.1, 0.5, 1, 3, 5]
+        end
+
+        def with_heartbeat(id)
+          if heartbeat_enabled?
+            ensure_heartbeat_thread_alive!
+            heartbeat_state.set(:tick, id)
+          end
+
+          yield
+        ensure
+          heartbeat_state.set(:reset) if heartbeat_enabled?
+        end
+
+        def ensure_heartbeat_thread_alive!
+          return unless heartbeat_enabled?
+          return if @heartbeat_thread&.alive?
+
+          @heartbeat_thread = Thread.start { heartbeat }
+        end
+
+        def boot_heartbeat_process!
+          return unless heartbeat_enabled?
+
+          heartbeat_process.boot!
+        end
+
+        def stop_heartbeat!
+          return unless heartbeat_enabled?
+
+          heartbeat_state.set(:stop)
+          heartbeat_process.shutdown!
+        end
+
         def custom_config
           return unless config.debug_log
 
@@ -163,6 +200,131 @@ module CI
         rescue SystemCallError
           ::File.read(::File.join(CI::Queue::RELEASE_SCRIPTS_ROOT, "#{name}.lua"))
         end
+
+        class HeartbeatProcess
+          def initialize(redis_url, zset_key, processed_key, owners_key, worker_queue_key)
+            @redis_url = redis_url
+            @zset_key = zset_key
+            @processed_key = processed_key
+            @owners_key = owners_key
+            @worker_queue_key = worker_queue_key
+          end
+
+          def boot!
+            child_read, @pipe = IO.pipe
+            ready_pipe, child_write = IO.pipe
+            @pipe.binmode
+            @pid = Process.spawn(
+              RbConfig.ruby,
+              ::File.join(__dir__, "monitor.rb"),
+              @redis_url,
+              @zset_key,
+              @processed_key,
+              @owners_key,
+              @worker_queue_key,
+              in: child_read,
+              out: child_write,
+            )
+            child_read.close
+            child_write.close
+
+            # Check the process is alive.
+            if ready_pipe.wait_readable(10)
+              ready_pipe.gets
+              ready_pipe.close
+              Process.kill(0, @pid)
+            else
+              Process.kill(0, @pid)
+              Process.wait(@pid)
+              raise "Monitor child wasn't ready after 10 seconds"
+            end
+            @pipe
+          end
+
+          def shutdown!
+            @pipe.close
+            begin
+              _, status = Process.waitpid2(@pid)
+              status
+            rescue Errno::ECHILD
+              nil
+            end
+          end
+
+          def tick!(id)
+            send_message(:tick!, id: id)
+          end
+
+          private
+
+          def send_message(*message)
+            payload = message.to_json
+            @pipe.write([payload.bytesize].pack("L").b, payload)
+          end
+        end
+
+        class State
+          def initialize
+            @state = nil
+            @mutex = Mutex.new
+            @cond = ConditionVariable.new
+          end
+
+          def set(*state)
+            @state = state
+            @mutex.synchronize do
+              @cond.broadcast
+            end
+          end
+
+          def wait(timeout)
+            @mutex.synchronize do
+              @cond.wait(@mutex, timeout)
+            end
+            @state
+          end
+        end
+
+        def heartbeat_state
+          @heartbeat_state ||= State.new
+        end
+
+        def heartbeat_process
+          @heartbeat_process ||= HeartbeatProcess.new(
+            @redis_url,
+            key('running'),
+            key('processed'),
+            key('owners'),
+            key('worker', worker_id, 'queue'),
+          )
+        end
+
+        def heartbeat_enabled?
+          config.max_missed_heartbeat_seconds
+        end
+
+        def heartbeat
+          Thread.current.name = "CI::Queue#heartbeat"
+          Thread.current.abort_on_exception = true
+
+          timeout = config.timeout.to_i
+          loop do
+            command = nil
+            command = heartbeat_state.wait(1) # waits for max 1 second but wakes up immediately if we receive a command
+
+            case command&.first
+            when :tick
+              if timeout > 0
+                heartbeat_process.tick!(command.last)
+                timeout -= 1
+              end
+            when :reset
+              timeout = config.timeout.to_i
+            when :stop
+              break
+            end
+          end
+        end
       end
     end
   end

data/lib/ci/queue/redis/heartbeat.lua

@@ -0,0 +1,18 @@
+-- AUTOGENERATED FILE DO NOT EDIT DIRECTLY
+local zset_key = KEYS[1]
+local processed_key = KEYS[2]
+local owners_key = KEYS[3]
+local worker_queue_key = KEYS[4]
+
+local current_time = ARGV[1]
+local test = ARGV[2]
+
+-- already processed, we do not need to bump the timestamp
+if redis.call('sismember', processed_key, test) == 1 then
+  return false
+end
+
+-- we're still the owner of the test, we can bump the timestamp
+if redis.call('hget', owners_key, test) == worker_queue_key then
+  return redis.call('zadd', zset_key, current_time, test)
+end

data/lib/ci/queue/redis/monitor.rb

@@ -0,0 +1,153 @@
+#!/usr/bin/env -S ruby --disable-gems
+# typed: false
+# frozen_string_literal: true
+
+require 'logger'
+require 'redis'
+require 'json'
+
+module CI
+  module Queue
+    module Redis
+      class Monitor
+        DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../../redis', __FILE__)
+        RELEASE_SCRIPTS_ROOT = ::File.expand_path('../../redis', __FILE__)
+
+        def initialize(pipe, logger, redis_url, zset_key, processed_key, owners_key, worker_queue_key)
+          @zset_key = zset_key
+          @processed_key = processed_key
+          @owners_key = owners_key
+          @worker_queue_key = worker_queue_key
+          @logger = logger
+          @redis = ::Redis.new(url: redis_url, reconnect_attempts: [0, 0, 0.1, 0.5, 1, 3, 5])
+          @shutdown = false
+          @pipe = pipe
+          @self_pipe_reader, @self_pipe_writer = IO.pipe
+          @self_pipe_writer.sync = true
+          @queue = []
+          @deadlines = {}
+          %i[TERM INT USR1].each do |sig|
+            Signal.trap(sig) { soft_signal(sig) }
+          end
+        end
+
+        def soft_signal(sig)
+          @queue << sig
+          @self_pipe_writer << '.'
+        end
+
+        def process_tick!(id:)
+          eval_script(
+            :heartbeat,
+            keys: [@zset_key, @processed_key, @owners_key, @worker_queue_key],
+            argv: [Time.now.to_f, id]
+          )
+        rescue => error
+          @logger.info(error)
+        end
+
+        def eval_script(script, *args)
+          @redis.evalsha(load_script(script), *args)
+        end
+
+        def load_script(script)
+          @scripts_cache ||= {}
+          @scripts_cache[script] ||= @redis.script(:load, read_script(script))
+        end
+
+        def read_script(name)
+          ::File.read(::File.join(DEV_SCRIPTS_ROOT, "#{name}.lua"))
+        rescue SystemCallError
+          ::File.read(::File.join(RELEASE_SCRIPTS_ROOT, "#{name}.lua"))
+        end
+
+        HEADER = 'L'
+        HEADER_SIZE = [0].pack(HEADER).bytesize
+        def read_message(io)
+          case header = io.read_nonblock(HEADER_SIZE, exception: false)
+          when :wait_readable
+            nil
+          when nil
+            @logger.debug('Broken pipe, exiting')
+            @shutdown = 0
+            false
+          else
+            JSON.parse(io.read(header.unpack1(HEADER)))
+          end
+        end
+
+        def process_messages(io)
+          while (message = read_message(io))
+            type, kwargs = message
+            kwargs.transform_keys!(&:to_sym)
+            public_send("process_#{type}", **kwargs)
+          end
+        end
+
+        def wait_for_events(ios)
+          return if @shutdown
+
+          return unless (ready = IO.select(ios, nil, nil, 10))
+
+          ready[0].each do |io|
+            case io
+            when @self_pipe_reader
+              io.read_nonblock(512, exception: false) # Just flush the pipe, the information is in the @queue
+            when @pipe
+              process_messages(@pipe)
+            else
+              @logger.debug("Unknown reader: #{io.inspect}")
+              raise "Unknown reader: #{io.inspect}"
+            end
+          end
+        end
+
+        def monitor
+          @logger.debug("Starting monitor")
+          ios = [@self_pipe_reader, @pipe]
+
+          until @shutdown
+            while (sig = @queue.shift)
+              case sig
+              when :INT, :TERM
+                @logger.debug("Received #{sig}, exiting")
+                @shutdown = 0
+                break
+              else
+                raise "Unknown signal: #{sig.inspect}"
+              end
+            end
+
+            wait_for_events(ios)
+          end
+
+          @logger.debug('Done')
+          @shutdown
+        end
+      end
+    end
+  end
+end
+
+logger = Logger.new($stderr)
+if ARGV.include?('-v')
+  logger.level = Logger::DEBUG
+else
+  logger.level = Logger::INFO
+  logger.formatter = ->(_severity, _timestamp, _progname, msg) { "[CI Queue Monitor] #{msg}\n" }
+end
+
+redis_url = ARGV[0]
+zset_key = ARGV[1]
+processed_key = ARGV[2]
+owners_key = ARGV[3]
+worker_queue_key = ARGV[4]
+
+logger.debug("Starting monitor: #{redis_url} #{zset_key} #{processed_key}")
+manager = CI::Queue::Redis::Monitor.new($stdin, logger, redis_url, zset_key, processed_key, owners_key, worker_queue_key)
+
+# Notify the parent we're ready
+$stdout.puts(".")
+$stdout.close
+
+exit(manager.monitor)

data/lib/ci/queue/redis/worker.rb

@@ -144,10 +144,6 @@ module CI
           config.worker_id
         end
 
-        def timeout
-          config.timeout
-        end
-
         def raise_on_mismatching_test(test)
           if @reserved_test == test
             @reserved_test = nil
@@ -180,6 +176,8 @@ module CI
         end
 
         def try_to_reserve_lost_test
+          timeout = config.max_missed_heartbeat_seconds ? config.max_missed_heartbeat_seconds : config.timeout
+
           lost_test = eval_script(
             :reserve_lost,
             keys: [
@@ -192,7 +190,7 @@ module CI
           )
 
           if lost_test
-            build.record_warning(Warnings::RESERVED_LOST_TEST, test: lost_test, timeout: timeout)
+            build.record_warning(Warnings::RESERVED_LOST_TEST, test: lost_test, timeout: config.timeout)
           end
 
           lost_test

data/lib/ci/queue/static.rb

@@ -48,6 +48,16 @@ module CI
         self
       end
 
+      def with_heartbeat(id)
+        yield
+      end
+
+      def ensure_heartbeat_thread_alive!; end
+
+      def boot_heartbeat_process!; end
+
+      def stop_heartbeat!; end
+
       def created_at=(timestamp)
         @created_at ||= timestamp
       end

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.42.0'
+    VERSION = '0.43.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/ci/queue.rb

@@ -2,6 +2,7 @@
 
 require 'uri'
 require 'cgi'
+require 'json'
 
 require 'ci/queue/version'
 require 'ci/queue/output_helpers'

data/lib/minitest/queue/runner.rb

@@ -64,6 +64,7 @@ module Minitest
         end
 
         queue.rescue_connection_errors { queue.created_at = CI::Queue.time_now.to_f }
+        queue.boot_heartbeat_process!
 
         set_load_path
         Minitest.queue = queue
@@ -582,6 +583,16 @@ module Minitest
             queue.config.redis_ttl = time
           end
 
+          help = <<~EOS
+            If heartbeat is enabled, a background process will periodically signal it's still processing
+            the current test. If the heartbeat stops for the specified amount of seconds,
+            the test will be requeued to another worker.
+          EOS
+          opts.on("--heartbeat [SECONDS]", Integer, help) do |time|
+            queue_config.max_missed_heartbeat_seconds = time || 30
+          end
+
+
           opts.on("-v", "--verbose", "Verbose. Show progress processing files.") do
             self.verbose = true
           end

data/lib/minitest/queue.rb

@@ -226,7 +226,10 @@ module Minitest
 
     def run_from_queue(reporter, *)
       queue.poll do |example|
-        result = example.run
+        result = queue.with_heartbeat(example.id) do
+          example.run
+        end
+
         failed = !(result.passed? || result.skipped?)
 
         if example.flaky?
@@ -256,6 +259,7 @@ module Minitest
           reporter.record(result)
         end
       end
+      queue.stop_heartbeat!
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.42.0
+  version: 0.43.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-01-15 00:00:00.000000000 Z
+date: 2024-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -188,6 +188,8 @@ files:
 - lib/ci/queue/redis/grind.rb
 - lib/ci/queue/redis/grind_record.rb
 - lib/ci/queue/redis/grind_supervisor.rb
+- lib/ci/queue/redis/heartbeat.lua
+- lib/ci/queue/redis/monitor.rb
 - lib/ci/queue/redis/release.lua
 - lib/ci/queue/redis/requeue.lua
 - lib/ci/queue/redis/reserve.lua
@@ -239,7 +241,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.4
+rubygems_version: 3.5.5
 signing_key: 
 specification_version: 4
 summary: Distribute tests over many workers using a queue