ci-queue 0.23.1 → 0.24.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aaa8a7cb585452d354e884d29772f29e0ea6dda29e894a4e69088ab302cec1b7
-  data.tar.gz: 8dc28379b0488fce6b951a0bf5ac434cce71fa5751c453f807ee6412edc86ef4
+  metadata.gz: 5f51b9eaf62e347a724e6e1fc533c26d504cca7922a4296b750ae2599c7cfa6e
+  data.tar.gz: 0b9833d46d95e90d67663a2ff5bea2a8ad17c781b812a9f8b4ec7485bf0789d0
 SHA512:
-  metadata.gz: b2fe2f4bfd08b4165576b2f881fdad537b22566fe8c940ee87a03e1ecb13488398b3a7e86147f83f284603c007cffbd606f8576b5f5a17a7d7c034f7dbacdf41
-  data.tar.gz: d12edaffb47e719b0c9e437da5261c4808dac54f2fa1e080d857c36225550a3f3a6d20517fe54e0da3ddaf1cb6d965fa6e666866049564ce741f6573aa187ff5
+  metadata.gz: b627008782774ad37ae5dbd5a2db0ae24df1f3fcda1dae9e0c0064f7c8df7890b926734855fd0f35d440e5e03d25441b6b9dc60c48bcaa19b8c5537c5d33c98b
+  data.tar.gz: e3d1c41abfe02341deb545fda1e6eebb94826801ca104788d2aa48fab1886e6093acd402ef097f58c6b2bc4f403568386e45d1b8032f1323d4f7cfbf6f7f381b

data/dev.yml

@@ -5,7 +5,7 @@ name: ci-queue
 up:
 - ruby: 2.6.5
 - bundler
-- railgun
+- isogun
 
 commands:
   test: REDIS_HOST=ci-queue.railgun bundle exec rake test

data/{railgun.yml → isogun.yml}

@@ -3,11 +3,8 @@
 name: ci-queue
 
 vm:
-  image: /opt/dev/misc/railgun-images/default
   ip_address: 192.168.64.245
   memory: 1G
   cores: 2
-volumes:
-  root: 512M
 services:
    - redis

data/lib/ci/queue/configuration.rb

@@ -8,7 +8,7 @@ module CI
       attr_accessor :max_test_failed, :redis_ttl
       attr_reader :circuit_breakers
       attr_writer :seed, :build_id
-      attr_writer :queue_init_timeout
+      attr_writer :queue_init_timeout, :report_timeout, :inactive_workers_timeout
 
       class << self
         def from_env(env)
@@ -35,7 +35,7 @@ module CI
         namespace: nil, seed: nil, flaky_tests: [], statsd_endpoint: nil, max_consecutive_failures: nil,
         grind_count: nil, max_duration: nil, failure_file: nil, max_test_duration: nil,
         max_test_duration_percentile: 0.5, track_test_duration: false, max_test_failed: nil,
-        queue_init_timeout: nil, redis_ttl: 8 * 60 * 60
+        queue_init_timeout: nil, redis_ttl: 8 * 60 * 60, report_timeout: nil, inactive_workers_timeout: nil
       )
         @build_id = build_id
         @circuit_breakers = [CircuitBreaker::Disabled]
@@ -57,12 +57,22 @@ module CI
         self.max_consecutive_failures = max_consecutive_failures
         self.max_duration = max_duration
         @redis_ttl = redis_ttl
+        @report_timeout = report_timeout
+        @inactive_workers_timeout = inactive_workers_timeout
       end
 
       def queue_init_timeout
         @queue_init_timeout || timeout
       end
 
+      def report_timeout
+        @report_timeout || timeout
+      end
+
+      def inactive_workers_timeout
+        @inactive_workers_timeout || timeout
+      end
+
       def max_consecutive_failures=(max)
         if max
           @circuit_breakers << CircuitBreaker::MaxConsecutiveFailures.new(max_consecutive_failures: max)

data/lib/ci/queue/redis/base.rb

@@ -5,6 +5,7 @@ module CI
       class Base
         include Common
 
+        TEN_MINUTES = 60 * 10
         CONNECTION_ERRORS = [
           ::Redis::BaseConnectionError,
           ::SocketError, # https://github.com/redis/redis-rb/pull/631
@@ -20,6 +21,15 @@ module CI
           queue_initialized? && size == 0
         end
 
+        def expired?
+          if (created_at = redis.get(key('master-created-at')))
+            (created_at.to_f + config.redis_ttl + TEN_MINUTES) < Time.now.to_f
+          else
+            # if there is no created at set anymore we assume queue is expired
+            true
+          end
+        end
+
         def size
           redis.multi do |transaction|
             transaction.llen(key('queue'))

data/lib/ci/queue/redis/supervisor.rb

@@ -21,17 +21,33 @@ module CI
 
           yield if block_given?
 
-          time_left = config.timeout
-          until exhausted? || time_left <= 0 || max_test_failed?
-            sleep 1
+          time_left = config.report_timeout
+          time_left_with_no_workers = config.inactive_workers_timeout
+          until exhausted? || time_left <= 0 || max_test_failed? || time_left_with_no_workers <= 0
             time_left -= 1
+            sleep 1
+
+            if active_workers?
+              time_left_with_no_workers = config.inactive_workers_timeout
+            else
+              time_left_with_no_workers -= 1
+            end
 
             yield if block_given?
           end
+
+          puts "Aborting, it seems all workers died." if time_left_with_no_workers <= 0
           exhausted?
         rescue CI::Queue::Redis::LostMaster
           false
         end
+
+        private
+
+        def active_workers?
+          # if there are running jobs we assume there are still agents active
+          redis.zrangebyscore(key('running'), Time.now.to_f - config.timeout, "+inf", limit: [0,1]).count > 0
+        end
       end
     end
   end

data/lib/ci/queue/redis/worker.rb

@@ -202,10 +202,12 @@ module CI
               transaction.lpush(key('queue'), tests) unless tests.empty?
               transaction.set(key('total'), @total)
               transaction.set(key('master-status'), 'ready')
+              transaction.set(key('master-created-at'), Time.now.to_f)
 
               transaction.expire(key('queue'), config.redis_ttl)
               transaction.expire(key('total'), config.redis_ttl)
               transaction.expire(key('master-status'), config.redis_ttl)
+              transaction.expire(key('master-created-at'), config.redis_ttl)
             end
           end
           register

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.23.1'
+    VERSION = '0.24.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue/runner.rb

@@ -47,7 +47,10 @@ module Minitest
       end
 
       def run_command
-        if queue.retrying?
+        if queue.retrying? || retry?
+          if queue.expired?
+            abort! "The test run is too old and can't be retried"
+          end
           reset_counters
           retry_queue = queue.retry_queue
           if retry_queue.exhausted?
@@ -367,6 +370,24 @@ module Minitest
             queue_config.timeout = timeout
           end
 
+          help = <<~EOS
+            Specify a timeout after which the report command will fail if not all tests have been processed.
+            Defaults to the value set for --timeout.
+          EOS
+          opts.separator ""
+          opts.on('--report-timeout TIMEOUT', Float, help) do |timeout|
+            queue_config.report_timeout = timeout
+          end
+
+          help = <<~EOS
+            Specify a timeout after the report will fail if all workers are inactive (e.g. died).
+            Defaults to the value set for --timeout.
+          EOS
+          opts.separator ""
+          opts.on('--inactive-workers-timeout TIMEOUT', Float, help) do |timeout|
+            queue_config.inactive_workers_timeout = timeout
+          end
+
           help = <<~EOS
             Specify a timeout to elect the leader and populate the queue.
             Defaults to the value set for --timeout.
@@ -546,6 +567,11 @@ module Minitest
         puts red(message)
         exit! 1 # exit! is required to avoid minitest at_exit callback
       end
+
+      def retry?
+        ENV["BUILDKITE_RETRY_COUNT"].to_i > 0 ||
+          ENV["SEMAPHORE_PIPELINE_RERUN"] == "true"
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.23.1
+  version: 0.24.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-06-17 00:00:00.000000000 Z
+date: 2022-07-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -171,6 +171,7 @@ files:
 - dev.yml
 - exe/minitest-queue
 - exe/rspec-queue
+- isogun.yml
 - lib/ci/queue.rb
 - lib/ci/queue/bisect.rb
 - lib/ci/queue/build_record.rb
@@ -218,7 +219,6 @@ files:
 - lib/rspec/queue.rb
 - lib/rspec/queue/build_status_recorder.rb
 - lib/rspec/queue/order_recorder.rb
-- railgun.yml
 homepage: https://github.com/Shopify/ci-queue
 licenses:
 - MIT