ci-queue 0.23.0 → 0.24.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 652049f77362ba8fa3fd01b79a0260c5ff17ba678751ee2424de463afe035d38
-  data.tar.gz: 745b2c493350d2315a10f36147d5d3b4cc74df74d5f38e82d2250f10be219d4a
+  metadata.gz: 2dddff029008b5dbe7ff19995f824f6c1d54a19e699377742a6e4599642f4b55
+  data.tar.gz: 9ff1268a7be473c4f35cb907193b24e04ae22189df0c116be9cc09db50112925
 SHA512:
-  metadata.gz: d47070eca76cfd6191cc003191af153ba2b40596dca4c3408325389a110ae1b798e99dcdb7e27b33c23d5bf457d6eab003dd608602a5df9de38741408d7bb48a
-  data.tar.gz: 0d2f4f69af4fc3714dbd6d5660d0e3106890a7bd5cef5782bae41847aa1552b35a8c9af6e77b7d8d18264b2100463062cb205f7ed45cef7646aeb2ff4ffe3bd3
+  metadata.gz: 755d03be6402d76d3d225c5c4dae6dafdf60f604fb6377c48403f98898b6ee6bc44634ad8d5dfdafe15f9ca2e3b8e304244850984f75aed67ae872064e9caedf
+  data.tar.gz: 157c12e846214542d78116fa168e380eba0b9266bcd2f3194d28c55951cc9c11cbc015582654ad6dfe09fac852e1123f3b9bbd8cd5b936ecdbf4075ce6a06784

data/dev.yml

@@ -5,7 +5,7 @@ name: ci-queue
 up:
 - ruby: 2.6.5
 - bundler
-- railgun
+- isogun
 
 commands:
   test: REDIS_HOST=ci-queue.railgun bundle exec rake test

data/{railgun.yml → isogun.yml}

@@ -3,11 +3,8 @@
 name: ci-queue
 
 vm:
-  image: /opt/dev/misc/railgun-images/default
   ip_address: 192.168.64.245
   memory: 1G
   cores: 2
-volumes:
-  root: 512M
 services:
    - redis

data/lib/ci/queue/configuration.rb

@@ -8,7 +8,7 @@ module CI
       attr_accessor :max_test_failed, :redis_ttl
       attr_reader :circuit_breakers
       attr_writer :seed, :build_id
-      attr_writer :queue_init_timeout
+      attr_writer :queue_init_timeout, :report_timeout, :inactive_workers_timeout
 
       class << self
         def from_env(env)
@@ -35,7 +35,7 @@ module CI
         namespace: nil, seed: nil, flaky_tests: [], statsd_endpoint: nil, max_consecutive_failures: nil,
         grind_count: nil, max_duration: nil, failure_file: nil, max_test_duration: nil,
         max_test_duration_percentile: 0.5, track_test_duration: false, max_test_failed: nil,
-        queue_init_timeout: nil, redis_ttl: 8 * 60 * 60
+        queue_init_timeout: nil, redis_ttl: 8 * 60 * 60, report_timeout: nil, inactive_workers_timeout: nil
       )
         @build_id = build_id
         @circuit_breakers = [CircuitBreaker::Disabled]
@@ -57,12 +57,22 @@ module CI
         self.max_consecutive_failures = max_consecutive_failures
         self.max_duration = max_duration
         @redis_ttl = redis_ttl
+        @report_timeout = report_timeout
+        @inactive_workers_timeout = inactive_workers_timeout
       end
 
       def queue_init_timeout
         @queue_init_timeout || timeout
       end
 
+      def report_timeout
+        @report_timeout || timeout
+      end
+
+      def inactive_workers_timeout
+        @inactive_workers_timeout || timeout
+      end
+
       def max_consecutive_failures=(max)
         if max
           @circuit_breakers << CircuitBreaker::MaxConsecutiveFailures.new(max_consecutive_failures: max)

data/lib/ci/queue/redis/base.rb

@@ -5,6 +5,7 @@ module CI
       class Base
         include Common
 
+        TEN_MINUTES = 60 * 10
         CONNECTION_ERRORS = [
           ::Redis::BaseConnectionError,
           ::SocketError, # https://github.com/redis/redis-rb/pull/631
@@ -20,6 +21,19 @@ module CI
           queue_initialized? && size == 0
         end
 
+        def expired?
+          if (created_at = redis.get(key('created-at')))
+            (created_at.to_f + config.redis_ttl + TEN_MINUTES) < Time.now.to_f
+          else
+            # if there is no created at set anymore we assume queue is expired
+            true
+          end
+        end
+
+        def created_at=(timestamp)
+          redis.setnx(key('created-at'), timestamp)
+        end
+
         def size
           redis.multi do |transaction|
             transaction.llen(key('queue'))
@@ -61,6 +75,10 @@ module CI
           end
         end
 
+        def queue_initializing?
+          master_status == 'setup'
+        end
+
         def increment_test_failed
           redis.incr(key('test_failed_count'))
         end

data/lib/ci/queue/redis/supervisor.rb

@@ -21,17 +21,33 @@ module CI
 
           yield if block_given?
 
-          time_left = config.timeout
-          until exhausted? || time_left <= 0 || max_test_failed?
-            sleep 1
+          time_left = config.report_timeout
+          time_left_with_no_workers = config.inactive_workers_timeout
+          until exhausted? || time_left <= 0 || max_test_failed? || time_left_with_no_workers <= 0
             time_left -= 1
+            sleep 1
+
+            if active_workers?
+              time_left_with_no_workers = config.inactive_workers_timeout
+            else
+              time_left_with_no_workers -= 1
+            end
 
             yield if block_given?
           end
+
+          puts "Aborting, it seems all workers died." if time_left_with_no_workers <= 0
           exhausted?
         rescue CI::Queue::Redis::LostMaster
           false
         end
+
+        private
+
+        def active_workers?
+          # if there are running jobs we assume there are still agents active
+          redis.zrangebyscore(key('running'), Time.now.to_f - config.timeout, "+inf", limit: [0,1]).count > 0
+        end
       end
     end
   end

data/lib/ci/queue/static.rb

@@ -11,6 +11,8 @@ module CI
         end
       end
 
+      TEN_MINUTES = 60 * 10
+
       attr_reader :progress, :total
 
       def initialize(tests, config)
@@ -37,6 +39,14 @@ module CI
         self
       end
 
+      def created_at=(timestamp)
+        @created_at ||= timestamp
+      end
+
+      def expired?
+        (@created_at.to_f TEN_MINUTES) < Time.now.to_f
+      end
+
       def populated?
         !!defined?(@index)
       end

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.23.0'
+    VERSION = '0.24.1'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue/runner.rb

@@ -47,7 +47,10 @@ module Minitest
       end
 
       def run_command
-        if queue.retrying?
+        if queue.retrying? || retry?
+          if queue.expired?
+            abort! "The test run is too old and can't be retried"
+          end
           reset_counters
           retry_queue = queue.retry_queue
           if retry_queue.exhausted?
@@ -58,6 +61,8 @@ module Minitest
           end
         end
 
+        queue.rescue_connection_errors { queue.created_at = Time.now.to_f }
+
         set_load_path
         Minitest.queue = queue
         reporters = [
@@ -367,6 +372,24 @@ module Minitest
             queue_config.timeout = timeout
           end
 
+          help = <<~EOS
+            Specify a timeout after which the report command will fail if not all tests have been processed.
+            Defaults to the value set for --timeout.
+          EOS
+          opts.separator ""
+          opts.on('--report-timeout TIMEOUT', Float, help) do |timeout|
+            queue_config.report_timeout = timeout
+          end
+
+          help = <<~EOS
+            Specify a timeout after the report will fail if all workers are inactive (e.g. died).
+            Defaults to the value set for --timeout.
+          EOS
+          opts.separator ""
+          opts.on('--inactive-workers-timeout TIMEOUT', Float, help) do |timeout|
+            queue_config.inactive_workers_timeout = timeout
+          end
+
           help = <<~EOS
             Specify a timeout to elect the leader and populate the queue.
             Defaults to the value set for --timeout.
@@ -546,6 +569,11 @@ module Minitest
         puts red(message)
         exit! 1 # exit! is required to avoid minitest at_exit callback
       end
+
+      def retry?
+        ENV["BUILDKITE_RETRY_COUNT"].to_i > 0 ||
+          ENV["SEMAPHORE_PIPELINE_RERUN"] == "true"
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.23.0
+  version: 0.24.1
 platform: ruby
 authors:
 - Jean Boussier
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-06-17 00:00:00.000000000 Z
+date: 2022-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -171,6 +171,7 @@ files:
 - dev.yml
 - exe/minitest-queue
 - exe/rspec-queue
+- isogun.yml
 - lib/ci/queue.rb
 - lib/ci/queue/bisect.rb
 - lib/ci/queue/build_record.rb
@@ -218,7 +219,6 @@ files:
 - lib/rspec/queue.rb
 - lib/rspec/queue/build_status_recorder.rb
 - lib/rspec/queue/order_recorder.rb
-- railgun.yml
 homepage: https://github.com/Shopify/ci-queue
 licenses:
 - MIT