ci-queue 0.40.0 → 0.42.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: be4adb99c549118b56b3b989e226acd24276f2105cd407ebe7c60685b439712f
-  data.tar.gz: 53ad2297734965e9eb942710e6c6e0dcff81b2597d864f1c0336cbf88c5b792d
+  metadata.gz: fb0b6d2c1537200e0a3540652972d25456cda0fe2115a65a884c8f5feeb17ff7
+  data.tar.gz: 7ac240806fdf6f16edfa314c59dccb22b379dbc2795d23d0c4ad19f4a46956d1
 SHA512:
-  metadata.gz: c5748d3f7cb1b9a9dd9cd386394c6e812f900fe7883686e920b8d2375fbb4fbfaf6c1a1a78703866f1bc3a64bf704cb21188c9fe3b1ddf762490aa1a36344770
-  data.tar.gz: d8d146b0427253a80d1ad3739d976bef95bf43fbee151b59fb3810d75450cfb3f3dbddd2a0825c9aea966b44d2c3226874fb182940a473e27873f526f2d7564b
+  metadata.gz: feaea129a7c672e57372e6475dc0fdc049c1e8b7815994b9ece3b4db0cbf6970ba19c222ed6bf9a652c7e68c7781e46edbd45e0eaa0da2b4e4d31b3856980aed
+  data.tar.gz: e2b9112e41b223c85b0acc20a7129d2ec517a816e73a42bea381ce696af8821dbaee1c5151d5002784630b1c0104b6c342d91b6273dd12de01c063d5bef5b1fe

data/dev.yml

@@ -3,7 +3,7 @@
 name: ci-queue
 
 up:
-- ruby: 3.2.0
+- ruby: 3.3.0
 - bundler
 - redis
 

data/lib/ci/queue/circuit_breaker.rb

@@ -46,7 +46,7 @@ module CI
         private
 
         def current_timestamp
-          Time.now.to_i
+          CI::Queue.time_now.to_i
         end
       end
 

data/lib/ci/queue/configuration.rb

@@ -5,7 +5,7 @@ module CI
       attr_accessor :timeout, :worker_id, :max_requeues, :grind_count, :failure_file, :export_flaky_tests_file
       attr_accessor :requeue_tolerance, :namespace, :failing_test, :statsd_endpoint
       attr_accessor :max_test_duration, :max_test_duration_percentile, :track_test_duration
-      attr_accessor :max_test_failed, :redis_ttl, :warnings_file
+      attr_accessor :max_test_failed, :redis_ttl, :warnings_file, :debug_log
       attr_reader :circuit_breakers
       attr_writer :seed, :build_id
       attr_writer :queue_init_timeout, :report_timeout, :inactive_workers_timeout
@@ -19,6 +19,7 @@ module CI
             flaky_tests: load_flaky_tests(env['CI_QUEUE_FLAKY_TESTS']),
             statsd_endpoint: env['CI_QUEUE_STATSD_ADDR'],
             redis_ttl: env['CI_QUEUE_REDIS_TTL']&.to_i ||  8 * 60 * 60,
+            debug_log: env['CI_QUEUE_DEBUG_LOG'],
           )
         end
 
@@ -36,8 +37,7 @@ module CI
         grind_count: nil, max_duration: nil, failure_file: nil, max_test_duration: nil,
         max_test_duration_percentile: 0.5, track_test_duration: false, max_test_failed: nil,
         queue_init_timeout: nil, redis_ttl: 8 * 60 * 60, report_timeout: nil, inactive_workers_timeout: nil,
-        export_flaky_tests_file: nil, warnings_file: nil
-      )
+        export_flaky_tests_file: nil, warnings_file: nil, debug_log: nil)
         @build_id = build_id
         @circuit_breakers = [CircuitBreaker::Disabled]
         @failure_file = failure_file
@@ -62,6 +62,7 @@ module CI
         @inactive_workers_timeout = inactive_workers_timeout
         @export_flaky_tests_file = export_flaky_tests_file
         @warnings_file = warnings_file
+        @debug_log = debug_log
       end
 
       def queue_init_timeout

data/lib/ci/queue/redis/base.rb

@@ -11,15 +11,50 @@ module CI
           ::SocketError, # https://github.com/redis/redis-rb/pull/631
         ].freeze
 
+        module RedisInstrumentation
+          def call(command, redis_config)
+            result = super
+            logger = redis_config.custom[:debug_log]
+            logger.info("#{command}: #{result}")
+            result
+          end
+
+          def call_pipelined(commands, redis_config)
+            result = super
+            logger = redis_config.custom[:debug_log]
+            logger.info("#{commands}: #{result}")
+            result
+          end
+        end
+
         def initialize(redis_url, config)
           @redis_url = redis_url
-          @redis = ::Redis.new(
-            url: redis_url,
-            # Booting a CI worker is costly, so in case of a Redis blip,
-            # it makes sense to retry for a while before giving up.
-            reconnect_attempts: [0, 0, 0.1, 0.5, 1, 3, 5],
-          )
           @config = config
+          if ::Redis::VERSION > "5.0.0"
+            @redis = ::Redis.new(
+              url: redis_url,
+              # Booting a CI worker is costly, so in case of a Redis blip,
+              # it makes sense to retry for a while before giving up.
+              reconnect_attempts: [0, 0, 0.1, 0.5, 1, 3, 5],
+              middlewares: custom_middlewares,
+              custom: custom_config,
+            )
+          else
+            @redis = ::Redis.new(url: redis_url)
+          end
+        end
+
+        def custom_config
+          return unless config.debug_log
+
+          require 'logger'
+          { debug_log: Logger.new(config.debug_log) }
+        end
+
+        def custom_middlewares
+          return unless config.debug_log
+
+          [RedisInstrumentation]
         end
 
         def exhausted?
@@ -28,7 +63,7 @@ module CI
 
         def expired?
           if (created_at = redis.get(key('created-at')))
-            (created_at.to_f + config.redis_ttl + TEN_MINUTES) < Time.now.to_f
+            (created_at.to_f + config.redis_ttl + TEN_MINUTES) < CI::Queue.time_now.to_f
           else
             # if there is no created at set anymore we assume queue is expired
             true

data/lib/ci/queue/redis/supervisor.rb

@@ -46,7 +46,7 @@ module CI
 
         def active_workers?
           # if there are running jobs we assume there are still agents active
-          redis.zrangebyscore(key('running'), Time.now.to_f - config.timeout, "+inf", limit: [0,1]).count > 0
+          redis.zrangebyscore(key('running'), CI::Queue.time_now.to_f - config.timeout, "+inf", limit: [0,1]).count > 0
         end
       end
     end

data/lib/ci/queue/redis/worker.rb

@@ -175,7 +175,7 @@ module CI
               key('worker', worker_id, 'queue'),
               key('owners'),
             ],
-            argv: [Time.now.to_f],
+            argv: [CI::Queue.time_now.to_f],
           )
         end
 
@@ -188,7 +188,7 @@ module CI
               key('worker', worker_id, 'queue'),
               key('owners'),
             ],
-            argv: [Time.now.to_f, timeout],
+            argv: [CI::Queue.time_now.to_f, timeout],
           )
 
           if lost_test

data/lib/ci/queue/redis.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-gem 'redis', '>= 5'
 require 'redis'
 require 'ci/queue/redis/build_record'
 require 'ci/queue/redis/base'

data/lib/ci/queue/static.rb

@@ -53,7 +53,7 @@ module CI
       end
 
       def expired?
-        (@created_at.to_f TEN_MINUTES) < Time.now.to_f
+        (@created_at.to_f TEN_MINUTES) < CI::Queue.time_now.to_f
       end
 
       def populated?

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.40.0'
+    VERSION = '0.42.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/ci/queue.rb

@@ -24,6 +24,15 @@ module CI
       RESERVED_LOST_TEST = :RESERVED_LOST_TEST
     end
 
+    GET_NOW = ::Time.method(:now)
+    private_constant :GET_NOW
+    def time_now
+      # Mocks like freeze_time should be cleaned when ci-queue runs, however,
+      # we experienced cases when tests were enqueued with wrong timestamps, so we
+      # safeguard Time.now here.
+      GET_NOW.call
+    end
+
     def requeueable?(test_result)
       requeueable.nil? || requeueable.call(test_result)
     end

data/lib/minitest/queue/runner.rb

@@ -63,7 +63,7 @@ module Minitest
           end
         end
 
-        queue.rescue_connection_errors { queue.created_at = Time.now.to_f }
+        queue.rescue_connection_errors { queue.created_at = CI::Queue.time_now.to_f }
 
         set_load_path
         Minitest.queue = queue
@@ -227,11 +227,11 @@ module Minitest
           end
 
           unless supervisor.exhausted?
+            reporter = BuildStatusReporter.new(build: supervisor.build)
+            reporter.report
             msg = "#{supervisor.size} tests weren't run."
             if supervisor.max_test_failed?
               puts('Encountered too many failed tests. Test run was ended early.')
-              reporter = BuildStatusReporter.new(build: supervisor.build)
-              reporter.report
               abort!(msg)
             else
               abort!(msg)
@@ -301,14 +301,9 @@ module Minitest
         return unless queue_config.warnings_file
 
         warnings = build.pop_warnings.map do |type, attributes|
-          case type
-          when CI::Queue::Warnings::RESERVED_LOST_TEST
-            "[WARNING] #{attributes[:test]} was picked up by another worker because it didn't complete in the allocated #{attributes[:timeout]} seconds.\n" \
-            "You may want to either optimize this test or bump ci-queue timeout.\n" \
-            "It's also possible that the worker that was processing it was terminated without being able to report back.\n"
-          end
+          attributes.merge(type: type)
         end.compact
-        File.write(queue_config.warnings_file, warnings.join("\n"))
+        File.write(queue_config.warnings_file, warnings.to_json)
       end
 
       def run_tests_in_fork(queue)
@@ -591,6 +586,10 @@ module Minitest
             self.verbose = true
           end
 
+          opts.on("--debug-log FILE", "Path to debug log file for e.g. Redis instrumentation") do |path|
+            queue_config.debug_log = path
+          end
+
           opts.separator ""
           opts.separator "    retry: Replays a previous run in the same order."
 

data/lib/minitest/queue.rb

@@ -187,7 +187,7 @@ module Minitest
       private
 
       def current_timestamp
-        Time.now.to_i
+        CI::Queue.time_now.to_i
       end
     end
 
@@ -244,20 +244,17 @@ module Minitest
           queue.report_success!
         end
 
-        requeued = false
         if failed && CI::Queue.requeueable?(result) && queue.requeue(example)
-          requeued = true
           result.requeue!
           reporter.record(result)
-        elsif queue.acknowledge(example) || !failed
+        elsif queue.acknowledge(example)
+          reporter.record(result)
+          queue.increment_test_failed if failed
+        elsif !failed
           # If the test was already acknowledged by another worker (we timed out)
           # Then we only record it if it is successful.
           reporter.record(result)
         end
-
-        if !requeued && failed
-          queue.increment_test_failed
-        end
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.40.0
+  version: 0.42.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-12-21 00:00:00.000000000 Z
+date: 2024-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -239,7 +239,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.22
+rubygems_version: 3.5.4
 signing_key: 
 specification_version: 4
 summary: Distribute tests over many workers using a queue