ci-queue 0.41.0 → 0.43.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 23effff409f076f80b7af3728d46a4122a59ca1a5a4c3bd1de5162250d60e72f
-  data.tar.gz: ae3c402de5003a6acb3ad002305b85c092ae76d4850634892cab935e03d4479b
+  metadata.gz: 9fafefb68f5e00faba6e1c19944c0289bd0e7c5ca4d67090605fea402f2b04f3
+  data.tar.gz: fe85dc8a004f45a54203eac3e4294e5348bca9d8920b70ad2427fed9f5f26776
 SHA512:
-  metadata.gz: 0601ef053acbb457931feec6428c627cdc1ef3d8d3e76cdd54a98f02ef6f8f6f531a38be13dc3c40d2800ca0e9efe00c91c21a569470887978e8ba997880933a
-  data.tar.gz: c95d7058762b67a059c4868b1d69a8fd99d5f6098d70fa08d71ec7533f7674b8c0886d1090c41db0ba200273b0cbb1c5fbb0fc12ecadddab04f2fb6a9ec950f3
+  metadata.gz: a190ef07194b9cbb9c880de74abd9e54ede4d3e52480e26ba20c16a80720fafb66ce5933bc21df4213e7cefc9abb5f3893b75e46fb7e8327ee6e11b8a5e26de4
+  data.tar.gz: 5b3a127d41fde0f8094878e8cfcd088eb2d1a32fc86798b98adafbf06caa19e201631899fa4aec9c0afa8d8a9174215d529dd1dfdf637e4e9122dfb93f99d741

data/lib/ci/queue/circuit_breaker.rb

@@ -46,7 +46,7 @@ module CI
         private
 
         def current_timestamp
-          Time.now.to_i
+          CI::Queue.time_now.to_i
         end
       end
 

data/lib/ci/queue/configuration.rb

@@ -5,7 +5,7 @@ module CI
       attr_accessor :timeout, :worker_id, :max_requeues, :grind_count, :failure_file, :export_flaky_tests_file
       attr_accessor :requeue_tolerance, :namespace, :failing_test, :statsd_endpoint
       attr_accessor :max_test_duration, :max_test_duration_percentile, :track_test_duration
-      attr_accessor :max_test_failed, :redis_ttl, :warnings_file
+      attr_accessor :max_test_failed, :redis_ttl, :warnings_file, :debug_log, :max_missed_heartbeat_seconds
       attr_reader :circuit_breakers
       attr_writer :seed, :build_id
       attr_writer :queue_init_timeout, :report_timeout, :inactive_workers_timeout
@@ -19,6 +19,7 @@ module CI
             flaky_tests: load_flaky_tests(env['CI_QUEUE_FLAKY_TESTS']),
             statsd_endpoint: env['CI_QUEUE_STATSD_ADDR'],
             redis_ttl: env['CI_QUEUE_REDIS_TTL']&.to_i ||  8 * 60 * 60,
+            debug_log: env['CI_QUEUE_DEBUG_LOG'],
           )
         end
 
@@ -36,8 +37,7 @@ module CI
         grind_count: nil, max_duration: nil, failure_file: nil, max_test_duration: nil,
         max_test_duration_percentile: 0.5, track_test_duration: false, max_test_failed: nil,
         queue_init_timeout: nil, redis_ttl: 8 * 60 * 60, report_timeout: nil, inactive_workers_timeout: nil,
-        export_flaky_tests_file: nil, warnings_file: nil
-      )
+        export_flaky_tests_file: nil, warnings_file: nil, debug_log: nil, max_missed_heartbeat_seconds: nil)
         @build_id = build_id
         @circuit_breakers = [CircuitBreaker::Disabled]
         @failure_file = failure_file
@@ -62,6 +62,8 @@ module CI
         @inactive_workers_timeout = inactive_workers_timeout
         @export_flaky_tests_file = export_flaky_tests_file
         @warnings_file = warnings_file
+        @debug_log = debug_log
+        @max_missed_heartbeat_seconds = max_missed_heartbeat_seconds
       end
 
       def queue_init_timeout

data/lib/ci/queue/redis/base.rb

@@ -11,19 +11,87 @@ module CI
           ::SocketError, # https://github.com/redis/redis-rb/pull/631
         ].freeze
 
+        module RedisInstrumentation
+          def call(command, redis_config)
+            result = super
+            logger = redis_config.custom[:debug_log]
+            logger.info("#{command}: #{result}")
+            result
+          end
+
+          def call_pipelined(commands, redis_config)
+            result = super
+            logger = redis_config.custom[:debug_log]
+            logger.info("#{commands}: #{result}")
+            result
+          end
+        end
+
         def initialize(redis_url, config)
           @redis_url = redis_url
+          @config = config
           if ::Redis::VERSION > "5.0.0"
             @redis = ::Redis.new(
               url: redis_url,
               # Booting a CI worker is costly, so in case of a Redis blip,
               # it makes sense to retry for a while before giving up.
-              reconnect_attempts: [0, 0, 0.1, 0.5, 1, 3, 5],
+              reconnect_attempts: reconnect_attempts,
+              middlewares: custom_middlewares,
+              custom: custom_config,
             )
           else
             @redis = ::Redis.new(url: redis_url)
           end
-          @config = config
+        end
+
+        def reconnect_attempts
+          return [] if ENV["CI_QUEUE_DISABLE_RECONNECT_ATTEMPTS"]
+
+          [0, 0, 0.1, 0.5, 1, 3, 5]
+        end
+
+        def with_heartbeat(id)
+          if heartbeat_enabled?
+            ensure_heartbeat_thread_alive!
+            heartbeat_state.set(:tick, id)
+          end
+
+          yield
+        ensure
+          heartbeat_state.set(:reset) if heartbeat_enabled?
+        end
+
+        def ensure_heartbeat_thread_alive!
+          return unless heartbeat_enabled?
+          return if @heartbeat_thread&.alive?
+
+          @heartbeat_thread = Thread.start { heartbeat }
+        end
+
+        def boot_heartbeat_process!
+          return unless heartbeat_enabled?
+
+          heartbeat_process.boot!
+        end
+
+        def stop_heartbeat!
+          return unless heartbeat_enabled?
+
+          heartbeat_state.set(:stop)
+          heartbeat_process.shutdown!
+        end
+
+        def custom_config
+          return unless config.debug_log
+
+          require 'logger'
+          { debug_log: Logger.new(config.debug_log) }
+        end
+
+        def custom_middlewares
+          return unless config.debug_log
+
+          [RedisInstrumentation]
         end
 
         def exhausted?
@@ -32,7 +100,7 @@ module CI
 
         def expired?
           if (created_at = redis.get(key('created-at')))
-            (created_at.to_f + config.redis_ttl + TEN_MINUTES) < Time.now.to_f
+            (created_at.to_f + config.redis_ttl + TEN_MINUTES) < CI::Queue.time_now.to_f
           else
             # if there is no created at set anymore we assume queue is expired
             true
@@ -132,6 +200,131 @@ module CI
         rescue SystemCallError
           ::File.read(::File.join(CI::Queue::RELEASE_SCRIPTS_ROOT, "#{name}.lua"))
         end
+
+        class HeartbeatProcess
+          def initialize(redis_url, zset_key, processed_key, owners_key, worker_queue_key)
+            @redis_url = redis_url
+            @zset_key = zset_key
+            @processed_key = processed_key
+            @owners_key = owners_key
+            @worker_queue_key = worker_queue_key
+          end
+
+          def boot!
+            child_read, @pipe = IO.pipe
+            ready_pipe, child_write = IO.pipe
+            @pipe.binmode
+            @pid = Process.spawn(
+              RbConfig.ruby,
+              ::File.join(__dir__, "monitor.rb"),
+              @redis_url,
+              @zset_key,
+              @processed_key,
+              @owners_key,
+              @worker_queue_key,
+              in: child_read,
+              out: child_write,
+            )
+            child_read.close
+            child_write.close
+
+            # Check the process is alive.
+            if ready_pipe.wait_readable(10)
+              ready_pipe.gets
+              ready_pipe.close
+              Process.kill(0, @pid)
+            else
+              Process.kill(0, @pid)
+              Process.wait(@pid)
+              raise "Monitor child wasn't ready after 10 seconds"
+            end
+            @pipe
+          end
+
+          def shutdown!
+            @pipe.close
+            begin
+              _, status = Process.waitpid2(@pid)
+              status
+            rescue Errno::ECHILD
+              nil
+            end
+          end
+
+          def tick!(id)
+            send_message(:tick!, id: id)
+          end
+
+          private
+
+          def send_message(*message)
+            payload = message.to_json
+            @pipe.write([payload.bytesize].pack("L").b, payload)
+          end
+        end
+
+        class State
+          def initialize
+            @state = nil
+            @mutex = Mutex.new
+            @cond = ConditionVariable.new
+          end
+
+          def set(*state)
+            @state = state
+            @mutex.synchronize do
+              @cond.broadcast
+            end
+          end
+
+          def wait(timeout)
+            @mutex.synchronize do
+              @cond.wait(@mutex, timeout)
+            end
+            @state
+          end
+        end
+
+        def heartbeat_state
+          @heartbeat_state ||= State.new
+        end
+
+        def heartbeat_process
+          @heartbeat_process ||= HeartbeatProcess.new(
+            @redis_url,
+            key('running'),
+            key('processed'),
+            key('owners'),
+            key('worker', worker_id, 'queue'),
+          )
+        end
+
+        def heartbeat_enabled?
+          config.max_missed_heartbeat_seconds
+        end
+
+        def heartbeat
+          Thread.current.name = "CI::Queue#heartbeat"
+          Thread.current.abort_on_exception = true
+
+          timeout = config.timeout.to_i
+          loop do
+            command = nil
+            command = heartbeat_state.wait(1) # waits for max 1 second but wakes up immediately if we receive a command
+
+            case command&.first
+            when :tick
+              if timeout > 0
+                heartbeat_process.tick!(command.last)
+                timeout -= 1
+              end
+            when :reset
+              timeout = config.timeout.to_i
+            when :stop
+              break
+            end
+          end
+        end
       end
     end
   end

data/lib/ci/queue/redis/heartbeat.lua

@@ -0,0 +1,18 @@
+-- AUTOGENERATED FILE DO NOT EDIT DIRECTLY
+local zset_key = KEYS[1]
+local processed_key = KEYS[2]
+local owners_key = KEYS[3]
+local worker_queue_key = KEYS[4]
+
+local current_time = ARGV[1]
+local test = ARGV[2]
+
+-- already processed, we do not need to bump the timestamp
+if redis.call('sismember', processed_key, test) == 1 then
+  return false
+end
+
+-- we're still the owner of the test, we can bump the timestamp
+if redis.call('hget', owners_key, test) == worker_queue_key then
+  return redis.call('zadd', zset_key, current_time, test)
+end

data/lib/ci/queue/redis/monitor.rb

@@ -0,0 +1,153 @@
+#!/usr/bin/env -S ruby --disable-gems
+# typed: false
+# frozen_string_literal: true
+
+require 'logger'
+require 'redis'
+require 'json'
+
+module CI
+  module Queue
+    module Redis
+      class Monitor
+        DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../../redis', __FILE__)
+        RELEASE_SCRIPTS_ROOT = ::File.expand_path('../../redis', __FILE__)
+
+        def initialize(pipe, logger, redis_url, zset_key, processed_key, owners_key, worker_queue_key)
+          @zset_key = zset_key
+          @processed_key = processed_key
+          @owners_key = owners_key
+          @worker_queue_key = worker_queue_key
+          @logger = logger
+          @redis = ::Redis.new(url: redis_url, reconnect_attempts: [0, 0, 0.1, 0.5, 1, 3, 5])
+          @shutdown = false
+          @pipe = pipe
+          @self_pipe_reader, @self_pipe_writer = IO.pipe
+          @self_pipe_writer.sync = true
+          @queue = []
+          @deadlines = {}
+          %i[TERM INT USR1].each do |sig|
+            Signal.trap(sig) { soft_signal(sig) }
+          end
+        end
+
+        def soft_signal(sig)
+          @queue << sig
+          @self_pipe_writer << '.'
+        end
+
+        def process_tick!(id:)
+          eval_script(
+            :heartbeat,
+            keys: [@zset_key, @processed_key, @owners_key, @worker_queue_key],
+            argv: [Time.now.to_f, id]
+          )
+        rescue => error
+          @logger.info(error)
+        end
+
+        def eval_script(script, *args)
+          @redis.evalsha(load_script(script), *args)
+        end
+
+        def load_script(script)
+          @scripts_cache ||= {}
+          @scripts_cache[script] ||= @redis.script(:load, read_script(script))
+        end
+
+        def read_script(name)
+          ::File.read(::File.join(DEV_SCRIPTS_ROOT, "#{name}.lua"))
+        rescue SystemCallError
+          ::File.read(::File.join(RELEASE_SCRIPTS_ROOT, "#{name}.lua"))
+        end
+
+        HEADER = 'L'
+        HEADER_SIZE = [0].pack(HEADER).bytesize
+        def read_message(io)
+          case header = io.read_nonblock(HEADER_SIZE, exception: false)
+          when :wait_readable
+            nil
+          when nil
+            @logger.debug('Broken pipe, exiting')
+            @shutdown = 0
+            false
+          else
+            JSON.parse(io.read(header.unpack1(HEADER)))
+          end
+        end
+
+        def process_messages(io)
+          while (message = read_message(io))
+            type, kwargs = message
+            kwargs.transform_keys!(&:to_sym)
+            public_send("process_#{type}", **kwargs)
+          end
+        end
+
+        def wait_for_events(ios)
+          return if @shutdown
+
+          return unless (ready = IO.select(ios, nil, nil, 10))
+
+          ready[0].each do |io|
+            case io
+            when @self_pipe_reader
+              io.read_nonblock(512, exception: false) # Just flush the pipe, the information is in the @queue
+            when @pipe
+              process_messages(@pipe)
+            else
+              @logger.debug("Unknown reader: #{io.inspect}")
+              raise "Unknown reader: #{io.inspect}"
+            end
+          end
+        end
+
+        def monitor
+          @logger.debug("Starting monitor")
+          ios = [@self_pipe_reader, @pipe]
+
+          until @shutdown
+            while (sig = @queue.shift)
+              case sig
+              when :INT, :TERM
+                @logger.debug("Received #{sig}, exiting")
+                @shutdown = 0
+                break
+              else
+                raise "Unknown signal: #{sig.inspect}"
+              end
+            end
+
+            wait_for_events(ios)
+          end
+
+          @logger.debug('Done')
+          @shutdown
+        end
+      end
+    end
+  end
+end
+
+logger = Logger.new($stderr)
+if ARGV.include?('-v')
+  logger.level = Logger::DEBUG
+else
+  logger.level = Logger::INFO
+  logger.formatter = ->(_severity, _timestamp, _progname, msg) { "[CI Queue Monitor] #{msg}\n" }
+end
+
+redis_url = ARGV[0]
+zset_key = ARGV[1]
+processed_key = ARGV[2]
+owners_key = ARGV[3]
+worker_queue_key = ARGV[4]
+
+logger.debug("Starting monitor: #{redis_url} #{zset_key} #{processed_key}")
+manager = CI::Queue::Redis::Monitor.new($stdin, logger, redis_url, zset_key, processed_key, owners_key, worker_queue_key)
+
+# Notify the parent we're ready
+$stdout.puts(".")
+$stdout.close
+
+exit(manager.monitor)

data/lib/ci/queue/redis/supervisor.rb

@@ -46,7 +46,7 @@ module CI
 
         def active_workers?
           # if there are running jobs we assume there are still agents active
-          redis.zrangebyscore(key('running'), Time.now.to_f - config.timeout, "+inf", limit: [0,1]).count > 0
+          redis.zrangebyscore(key('running'), CI::Queue.time_now.to_f - config.timeout, "+inf", limit: [0,1]).count > 0
         end
       end
     end

data/lib/ci/queue/redis/worker.rb

@@ -144,10 +144,6 @@ module CI
           config.worker_id
         end
 
-        def timeout
-          config.timeout
-        end
-
         def raise_on_mismatching_test(test)
           if @reserved_test == test
             @reserved_test = nil
@@ -175,11 +171,13 @@ module CI
               key('worker', worker_id, 'queue'),
               key('owners'),
             ],
-            argv: [Time.now.to_f],
+            argv: [CI::Queue.time_now.to_f],
           )
         end
 
         def try_to_reserve_lost_test
+          timeout = config.max_missed_heartbeat_seconds ? config.max_missed_heartbeat_seconds : config.timeout
+
           lost_test = eval_script(
             :reserve_lost,
             keys: [
@@ -188,11 +186,11 @@ module CI
               key('worker', worker_id, 'queue'),
               key('owners'),
             ],
-            argv: [Time.now.to_f, timeout],
+            argv: [CI::Queue.time_now.to_f, timeout],
           )
 
           if lost_test
-            build.record_warning(Warnings::RESERVED_LOST_TEST, test: lost_test, timeout: timeout)
+            build.record_warning(Warnings::RESERVED_LOST_TEST, test: lost_test, timeout: config.timeout)
           end
 
           lost_test

data/lib/ci/queue/static.rb

@@ -48,12 +48,22 @@ module CI
         self
       end
 
+      def with_heartbeat(id)
+        yield
+      end
+
+      def ensure_heartbeat_thread_alive!; end
+
+      def boot_heartbeat_process!; end
+
+      def stop_heartbeat!; end
+
       def created_at=(timestamp)
         @created_at ||= timestamp
       end
 
       def expired?
-        (@created_at.to_f TEN_MINUTES) < Time.now.to_f
+        (@created_at.to_f TEN_MINUTES) < CI::Queue.time_now.to_f
       end
 
       def populated?

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.41.0'
+    VERSION = '0.43.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/ci/queue.rb

@@ -2,6 +2,7 @@
 
 require 'uri'
 require 'cgi'
+require 'json'
 
 require 'ci/queue/version'
 require 'ci/queue/output_helpers'
@@ -24,6 +25,15 @@ module CI
       RESERVED_LOST_TEST = :RESERVED_LOST_TEST
     end
 
+    GET_NOW = ::Time.method(:now)
+    private_constant :GET_NOW
+    def time_now
+      # Mocks like freeze_time should be cleaned when ci-queue runs, however,
+      # we experienced cases when tests were enqueued with wrong timestamps, so we
+      # safeguard Time.now here.
+      GET_NOW.call
+    end
+
     def requeueable?(test_result)
       requeueable.nil? || requeueable.call(test_result)
     end

data/lib/minitest/queue/runner.rb

@@ -63,7 +63,8 @@ module Minitest
           end
         end
 
-        queue.rescue_connection_errors { queue.created_at = Time.now.to_f }
+        queue.rescue_connection_errors { queue.created_at = CI::Queue.time_now.to_f }
+        queue.boot_heartbeat_process!
 
         set_load_path
         Minitest.queue = queue
@@ -582,10 +583,24 @@ module Minitest
             queue.config.redis_ttl = time
           end
 
+          help = <<~EOS
+            If heartbeat is enabled, a background process will periodically signal it's still processing
+            the current test. If the heartbeat stops for the specified amount of seconds,
+            the test will be requeued to another worker.
+          EOS
+          opts.on("--heartbeat [SECONDS]", Integer, help) do |time|
+            queue_config.max_missed_heartbeat_seconds = time || 30
+          end
+
+
           opts.on("-v", "--verbose", "Verbose. Show progress processing files.") do
             self.verbose = true
           end
 
+          opts.on("--debug-log FILE", "Path to debug log file for e.g. Redis instrumentation") do |path|
+            queue_config.debug_log = path
+          end
+
           opts.separator ""
           opts.separator "    retry: Replays a previous run in the same order."
 

data/lib/minitest/queue.rb

@@ -187,7 +187,7 @@ module Minitest
       private
 
       def current_timestamp
-        Time.now.to_i
+        CI::Queue.time_now.to_i
       end
     end
 
@@ -226,7 +226,10 @@ module Minitest
 
     def run_from_queue(reporter, *)
       queue.poll do |example|
-        result = example.run
+        result = queue.with_heartbeat(example.id) do
+          example.run
+        end
+
         failed = !(result.passed? || result.skipped?)
 
         if example.flaky?
@@ -256,6 +259,7 @@ module Minitest
           reporter.record(result)
         end
       end
+      queue.stop_heartbeat!
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.41.0
+  version: 0.43.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-01-04 00:00:00.000000000 Z
+date: 2024-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -188,6 +188,8 @@ files:
 - lib/ci/queue/redis/grind.rb
 - lib/ci/queue/redis/grind_record.rb
 - lib/ci/queue/redis/grind_supervisor.rb
+- lib/ci/queue/redis/heartbeat.lua
+- lib/ci/queue/redis/monitor.rb
 - lib/ci/queue/redis/release.lua
 - lib/ci/queue/redis/requeue.lua
 - lib/ci/queue/redis/reserve.lua
@@ -239,7 +241,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.4
+rubygems_version: 3.5.5
 signing_key: 
 specification_version: 4
 summary: Distribute tests over many workers using a queue