ci-queue 0.23.1 → 0.24.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aaa8a7cb585452d354e884d29772f29e0ea6dda29e894a4e69088ab302cec1b7
-  data.tar.gz: 8dc28379b0488fce6b951a0bf5ac434cce71fa5751c453f807ee6412edc86ef4
+  metadata.gz: 19eb1f45380007118047d2c25939122868e7cd69413d5f7b398fc400a697475e
+  data.tar.gz: 36062c5ce17f5a547a255fa86f6c089a0a5e0cbc3acf3784beec3e4d423c11a6
 SHA512:
-  metadata.gz: b2fe2f4bfd08b4165576b2f881fdad537b22566fe8c940ee87a03e1ecb13488398b3a7e86147f83f284603c007cffbd606f8576b5f5a17a7d7c034f7dbacdf41
-  data.tar.gz: d12edaffb47e719b0c9e437da5261c4808dac54f2fa1e080d857c36225550a3f3a6d20517fe54e0da3ddaf1cb6d965fa6e666866049564ce741f6573aa187ff5
+  metadata.gz: 6107331dffffba8ac435081865a3b2cea6de604d61168e9735c419c28fabd433c62b5d05ed6b7bd5fc645901668906e0c98fc3acc57613599d91236f90c148dd
+  data.tar.gz: 7d79100c0de4c37a3f1dbe0d2ac8a264f2ef3f1d52852c64f04bb7a4e24691c150c5d3b967023b39a6ef5be91bd0314c68fed3f09fec141a55b944e439e7fdef

data/dev.yml

@@ -5,7 +5,7 @@ name: ci-queue
 up:
 - ruby: 2.6.5
 - bundler
-- railgun
+- isogun
 
 commands:
   test: REDIS_HOST=ci-queue.railgun bundle exec rake test

data/{railgun.yml → isogun.yml}

@@ -3,11 +3,8 @@
 name: ci-queue
 
 vm:
-  image: /opt/dev/misc/railgun-images/default
   ip_address: 192.168.64.245
   memory: 1G
   cores: 2
-volumes:
-  root: 512M
 services:
    - redis

data/lib/ci/queue/configuration.rb

@@ -8,7 +8,7 @@ module CI
       attr_accessor :max_test_failed, :redis_ttl
       attr_reader :circuit_breakers
       attr_writer :seed, :build_id
-      attr_writer :queue_init_timeout
+      attr_writer :queue_init_timeout, :report_timeout, :inactive_workers_timeout
 
       class << self
         def from_env(env)
@@ -35,7 +35,7 @@ module CI
         namespace: nil, seed: nil, flaky_tests: [], statsd_endpoint: nil, max_consecutive_failures: nil,
         grind_count: nil, max_duration: nil, failure_file: nil, max_test_duration: nil,
         max_test_duration_percentile: 0.5, track_test_duration: false, max_test_failed: nil,
-        queue_init_timeout: nil, redis_ttl: 8 * 60 * 60
+        queue_init_timeout: nil, redis_ttl: 8 * 60 * 60, report_timeout: nil, inactive_workers_timeout: nil
       )
         @build_id = build_id
         @circuit_breakers = [CircuitBreaker::Disabled]
@@ -57,12 +57,22 @@ module CI
         self.max_consecutive_failures = max_consecutive_failures
         self.max_duration = max_duration
         @redis_ttl = redis_ttl
+        @report_timeout = report_timeout
+        @inactive_workers_timeout = inactive_workers_timeout
       end
 
       def queue_init_timeout
         @queue_init_timeout || timeout
       end
 
+      def report_timeout
+        @report_timeout || timeout
+      end
+
+      def inactive_workers_timeout
+        @inactive_workers_timeout || timeout
+      end
+
       def max_consecutive_failures=(max)
         if max
           @circuit_breakers << CircuitBreaker::MaxConsecutiveFailures.new(max_consecutive_failures: max)

data/lib/ci/queue/redis/base.rb

@@ -5,6 +5,7 @@ module CI
       class Base
         include Common
 
+        TEN_MINUTES = 60 * 10
         CONNECTION_ERRORS = [
           ::Redis::BaseConnectionError,
           ::SocketError, # https://github.com/redis/redis-rb/pull/631
@@ -20,6 +21,19 @@ module CI
           queue_initialized? && size == 0
         end
 
+        def expired?
+          if (created_at = redis.get(key('created-at')))
+            (created_at.to_f + config.redis_ttl + TEN_MINUTES) < Time.now.to_f
+          else
+            # if there is no created at set anymore we assume queue is expired
+            true
+          end
+        end
+
+        def created_at=(timestamp)
+          redis.setnx(key('created-at'), timestamp)
+        end
+
         def size
           redis.multi do |transaction|
             transaction.llen(key('queue'))
@@ -61,6 +75,10 @@ module CI
           end
         end
 
+        def queue_initializing?
+          master_status == 'setup'
+        end
+
         def increment_test_failed
           redis.incr(key('test_failed_count'))
         end

data/lib/ci/queue/redis/supervisor.rb

@@ -21,17 +21,33 @@ module CI
 
           yield if block_given?
 
-          time_left = config.timeout
-          until exhausted? || time_left <= 0 || max_test_failed?
-            sleep 1
+          time_left = config.report_timeout
+          time_left_with_no_workers = config.inactive_workers_timeout
+          until exhausted? || time_left <= 0 || max_test_failed? || time_left_with_no_workers <= 0
             time_left -= 1
+            sleep 1
+
+            if active_workers?
+              time_left_with_no_workers = config.inactive_workers_timeout
+            else
+              time_left_with_no_workers -= 1
+            end
 
             yield if block_given?
           end
+
+          puts "Aborting, it seems all workers died." if time_left_with_no_workers <= 0
           exhausted?
         rescue CI::Queue::Redis::LostMaster
           false
         end
+
+        private
+
+        def active_workers?
+          # if there are running jobs we assume there are still agents active
+          redis.zrangebyscore(key('running'), Time.now.to_f - config.timeout, "+inf", limit: [0,1]).count > 0
+        end
       end
     end
   end

data/lib/ci/queue/redis/worker.rb

@@ -215,7 +215,7 @@ module CI
         end
 
         def register
-          redis.sadd(key('workers'), worker_id)
+          redis.sadd(key('workers'), [worker_id])
         end
       end
     end

data/lib/ci/queue/static.rb

@@ -11,6 +11,8 @@ module CI
         end
       end
 
+      TEN_MINUTES = 60 * 10
+
       attr_reader :progress, :total
 
       def initialize(tests, config)
@@ -37,6 +39,14 @@ module CI
         self
       end
 
+      def created_at=(timestamp)
+        @created_at ||= timestamp
+      end
+
+      def expired?
+        (@created_at.to_f TEN_MINUTES) < Time.now.to_f
+      end
+
       def populated?
         !!defined?(@index)
       end

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.23.1'
+    VERSION = '0.24.2'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue/runner.rb

@@ -47,7 +47,10 @@ module Minitest
       end
 
       def run_command
-        if queue.retrying?
+        if queue.retrying? || retry?
+          if queue.expired?
+            abort! "The test run is too old and can't be retried"
+          end
           reset_counters
           retry_queue = queue.retry_queue
           if retry_queue.exhausted?
@@ -58,6 +61,8 @@ module Minitest
           end
         end
 
+        queue.rescue_connection_errors { queue.created_at = Time.now.to_f }
+
         set_load_path
         Minitest.queue = queue
         reporters = [
@@ -367,6 +372,24 @@ module Minitest
             queue_config.timeout = timeout
           end
 
+          help = <<~EOS
+            Specify a timeout after which the report command will fail if not all tests have been processed.
+            Defaults to the value set for --timeout.
+          EOS
+          opts.separator ""
+          opts.on('--report-timeout TIMEOUT', Float, help) do |timeout|
+            queue_config.report_timeout = timeout
+          end
+
+          help = <<~EOS
+            Specify a timeout after the report will fail if all workers are inactive (e.g. died).
+            Defaults to the value set for --timeout.
+          EOS
+          opts.separator ""
+          opts.on('--inactive-workers-timeout TIMEOUT', Float, help) do |timeout|
+            queue_config.inactive_workers_timeout = timeout
+          end
+
           help = <<~EOS
             Specify a timeout to elect the leader and populate the queue.
             Defaults to the value set for --timeout.
@@ -546,6 +569,11 @@ module Minitest
         puts red(message)
         exit! 1 # exit! is required to avoid minitest at_exit callback
       end
+
+      def retry?
+        ENV["BUILDKITE_RETRY_COUNT"].to_i > 0 ||
+          ENV["SEMAPHORE_PIPELINE_RERUN"] == "true"
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.23.1
+  version: 0.24.2
 platform: ruby
 authors:
 - Jean Boussier
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-06-17 00:00:00.000000000 Z
+date: 2022-09-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -171,6 +171,7 @@ files:
 - dev.yml
 - exe/minitest-queue
 - exe/rspec-queue
+- isogun.yml
 - lib/ci/queue.rb
 - lib/ci/queue/bisect.rb
 - lib/ci/queue/build_record.rb
@@ -218,7 +219,6 @@ files:
 - lib/rspec/queue.rb
 - lib/rspec/queue/build_status_recorder.rb
 - lib/rspec/queue/order_recorder.rb
-- railgun.yml
 homepage: https://github.com/Shopify/ci-queue
 licenses:
 - MIT