ci-queue 0.22.1 → 0.24.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2847d2cc56c64030f51a65ef6b9262d04b647b15b6d6ef846cf76adb90715e44
-  data.tar.gz: bf4f65911c9f39c2e6049abc6006f853f8de1aa444dfe8d5c7237cd698d0a59f
+  metadata.gz: 5f51b9eaf62e347a724e6e1fc533c26d504cca7922a4296b750ae2599c7cfa6e
+  data.tar.gz: 0b9833d46d95e90d67663a2ff5bea2a8ad17c781b812a9f8b4ec7485bf0789d0
 SHA512:
-  metadata.gz: b7c62dd52f53069dd2aad2b51724334e7ea2e8fc2cdc8f19f022b6dfd926bb7323437c3f6d1c4ffe7071316c07cde6b924671be07a2e7b402e2d528becd16acf
-  data.tar.gz: 056b3a585d95b88479f79448dd204eef72e16635c0c2df5c619ad78be2caf3b97a6e52c15edf09f2a2dcccf80afa3ea0018cfe9361114f6d03914383ab080efb
+  metadata.gz: b627008782774ad37ae5dbd5a2db0ae24df1f3fcda1dae9e0c0064f7c8df7890b926734855fd0f35d440e5e03d25441b6b9dc60c48bcaa19b8c5537c5d33c98b
+  data.tar.gz: e3d1c41abfe02341deb545fda1e6eebb94826801ca104788d2aa48fab1886e6093acd402ef097f58c6b2bc4f403568386e45d1b8032f1323d4f7cfbf6f7f381b

data/README.md

@@ -70,3 +70,9 @@ rspec-queue --queue redis://example.com --timeout 600 --report
 #### Limitations
 
 Because of how `ci-queue` executes the examples, `before(:all)` and `after(:all)` hooks are not supported. `rspec-queue` will explicitly reject them.
+
+## Custom Redis Expiry
+
+`ci-queue` expects the Redis server to have an [eviction policy](https://redis.io/docs/manual/eviction/#eviction-policies) of `allkeys-lru`.
+
+You can also use `--redis-ttl` to set a custom expiration time for all CI Queue keys, this defaults to 8 hours (28,800 seconds)

data/dev.yml

@@ -5,7 +5,7 @@ name: ci-queue
 up:
 - ruby: 2.6.5
 - bundler
-- railgun
+- isogun
 
 commands:
   test: REDIS_HOST=ci-queue.railgun bundle exec rake test

data/{railgun.yml → isogun.yml}

@@ -3,11 +3,8 @@
 name: ci-queue
 
 vm:
-  image: /opt/dev/misc/railgun-images/default
   ip_address: 192.168.64.245
   memory: 1G
   cores: 2
-volumes:
-  root: 512M
 services:
    - redis

data/lib/ci/queue/configuration.rb

@@ -5,10 +5,10 @@ module CI
       attr_accessor :timeout, :worker_id, :max_requeues, :grind_count, :failure_file
       attr_accessor :requeue_tolerance, :namespace, :failing_test, :statsd_endpoint
       attr_accessor :max_test_duration, :max_test_duration_percentile, :track_test_duration
-      attr_accessor :max_test_failed
+      attr_accessor :max_test_failed, :redis_ttl
       attr_reader :circuit_breakers
       attr_writer :seed, :build_id
-      attr_writer :queue_init_timeout
+      attr_writer :queue_init_timeout, :report_timeout, :inactive_workers_timeout
 
       class << self
         def from_env(env)
@@ -18,6 +18,7 @@ module CI
             seed: env['CIRCLE_SHA1'] || env['BUILDKITE_COMMIT'] || env['TRAVIS_COMMIT'] || env['HEROKU_TEST_RUN_COMMIT_VERSION'] || env['SEMAPHORE_GIT_SHA'],
             flaky_tests: load_flaky_tests(env['CI_QUEUE_FLAKY_TESTS']),
             statsd_endpoint: env['CI_QUEUE_STATSD_ADDR'],
+            redis_ttl: env['CI_QUEUE_REDIS_TTL']&.to_i ||  8 * 60 * 60,
           )
         end
 
@@ -34,7 +35,7 @@ module CI
         namespace: nil, seed: nil, flaky_tests: [], statsd_endpoint: nil, max_consecutive_failures: nil,
         grind_count: nil, max_duration: nil, failure_file: nil, max_test_duration: nil,
         max_test_duration_percentile: 0.5, track_test_duration: false, max_test_failed: nil,
-        queue_init_timeout: nil
+        queue_init_timeout: nil, redis_ttl: 8 * 60 * 60, report_timeout: nil, inactive_workers_timeout: nil
       )
         @build_id = build_id
         @circuit_breakers = [CircuitBreaker::Disabled]
@@ -55,12 +56,23 @@ module CI
         @worker_id = worker_id
         self.max_consecutive_failures = max_consecutive_failures
         self.max_duration = max_duration
+        @redis_ttl = redis_ttl
+        @report_timeout = report_timeout
+        @inactive_workers_timeout = inactive_workers_timeout
       end
 
       def queue_init_timeout
         @queue_init_timeout || timeout
       end
 
+      def report_timeout
+        @report_timeout || timeout
+      end
+
+      def inactive_workers_timeout
+        @inactive_workers_timeout || timeout
+      end
+
       def max_consecutive_failures=(max)
         if max
           @circuit_breakers << CircuitBreaker::MaxConsecutiveFailures.new(max_consecutive_failures: max)

data/lib/ci/queue/redis/base.rb

@@ -5,6 +5,7 @@ module CI
       class Base
         include Common
 
+        TEN_MINUTES = 60 * 10
         CONNECTION_ERRORS = [
           ::Redis::BaseConnectionError,
           ::SocketError, # https://github.com/redis/redis-rb/pull/631
@@ -20,6 +21,15 @@ module CI
           queue_initialized? && size == 0
         end
 
+        def expired?
+          if (created_at = redis.get(key('master-created-at')))
+            (created_at.to_f + config.redis_ttl + TEN_MINUTES) < Time.now.to_f
+          else
+            # if there is no created at set anymore we assume queue is expired
+            true
+          end
+        end
+
         def size
           redis.multi do |transaction|
             transaction.llen(key('queue'))

data/lib/ci/queue/redis/build_record.rb

@@ -41,6 +41,7 @@ module CI
               id.dup.force_encoding(Encoding::BINARY),
               payload.dup.force_encoding(Encoding::BINARY),
             )
+            pipeline.expire(key('error-reports'), config.redis_ttl)
             record_stats(stats, pipeline: pipeline)
           end
           nil
@@ -90,6 +91,7 @@ module CI
           return unless stats
           stats.each do |stat_name, stat_value|
             pipeline.hset(key(stat_name), config.worker_id, stat_value)
+            pipeline.expire(key(stat_name), config.redis_ttl)
           end
         end
 

data/lib/ci/queue/redis/grind_record.rb

@@ -16,6 +16,7 @@ module CI
               key('error-reports'),
               payload.force_encoding(Encoding::BINARY),
             )
+            pipeline.expire(key('error-reports'), config.redis_ttl)
             record_stats(stats, pipeline: pipeline)
           end
           nil
@@ -58,6 +59,7 @@ module CI
           return unless stats
           stats.each do |stat_name, stat_value|
             pipeline.hset(key(stat_name), config.worker_id, stat_value)
+            pipeline.expire(key(stat_name), config.redis_ttl)
           end
         end
       end

data/lib/ci/queue/redis/supervisor.rb

@@ -21,17 +21,33 @@ module CI
 
           yield if block_given?
 
-          time_left = config.timeout
-          until exhausted? || time_left <= 0 || max_test_failed?
-            sleep 1
+          time_left = config.report_timeout
+          time_left_with_no_workers = config.inactive_workers_timeout
+          until exhausted? || time_left <= 0 || max_test_failed? || time_left_with_no_workers <= 0
             time_left -= 1
+            sleep 1
+
+            if active_workers?
+              time_left_with_no_workers = config.inactive_workers_timeout
+            else
+              time_left_with_no_workers -= 1
+            end
 
             yield if block_given?
           end
+
+          puts "Aborting, it seems all workers died." if time_left_with_no_workers <= 0
           exhausted?
         rescue CI::Queue::Redis::LostMaster
           false
         end
+
+        private
+
+        def active_workers?
+          # if there are running jobs we assume there are still agents active
+          redis.zrangebyscore(key('running'), Time.now.to_f - config.timeout, "+inf", limit: [0,1]).count > 0
+        end
       end
     end
   end

data/lib/ci/queue/redis/test_time_record.rb

@@ -24,6 +24,7 @@ module CI
               test_time_key(test_name),
               duration.to_s.force_encoding(Encoding::BINARY),
             )
+            pipeline.expire(test_time_key(test_name), config.redis_ttl)
           end
           nil
         end
@@ -34,6 +35,7 @@ module CI
               all_test_names_key,
               test_name.dup.force_encoding(Encoding::BINARY),
             )
+            pipeline.expire(all_test_names_key, config.redis_ttl)
           end
           nil
         end

data/lib/ci/queue/redis/worker.rb

@@ -53,6 +53,10 @@ module CI
               sleep 0.05
             end
           end
+          redis.pipelined do |pipeline|
+            pipeline.expire(key('worker', worker_id, 'queue'), config.redis_ttl)
+            pipeline.expire(key('processed'), config.redis_ttl)
+          end
         rescue *CONNECTION_ERRORS
         end
 
@@ -198,9 +202,16 @@ module CI
               transaction.lpush(key('queue'), tests) unless tests.empty?
               transaction.set(key('total'), @total)
               transaction.set(key('master-status'), 'ready')
+              transaction.set(key('master-created-at'), Time.now.to_f)
+
+              transaction.expire(key('queue'), config.redis_ttl)
+              transaction.expire(key('total'), config.redis_ttl)
+              transaction.expire(key('master-status'), config.redis_ttl)
+              transaction.expire(key('master-created-at'), config.redis_ttl)
             end
           end
           register
+          redis.expire(key('workers'), config.redis_ttl)
         rescue *CONNECTION_ERRORS
           raise if @master
         end

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.22.1'
+    VERSION = '0.24.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue/runner.rb

@@ -22,6 +22,9 @@ module Minitest
       def initialize(argv)
         @queue_config = CI::Queue::Configuration.from_env(ENV)
         @command, @argv = parse(argv)
+        if Minitest.respond_to?(:seed=)
+          Minitest.seed = @queue_config.seed.to_i
+        end
       end
 
       def run!
@@ -44,7 +47,10 @@ module Minitest
       end
 
       def run_command
-        if queue.retrying?
+        if queue.retrying? || retry?
+          if queue.expired?
+            abort! "The test run is too old and can't be retried"
+          end
           reset_counters
           retry_queue = queue.retry_queue
           if retry_queue.exhausted?
@@ -364,6 +370,24 @@ module Minitest
             queue_config.timeout = timeout
           end
 
+          help = <<~EOS
+            Specify a timeout after which the report command will fail if not all tests have been processed.
+            Defaults to the value set for --timeout.
+          EOS
+          opts.separator ""
+          opts.on('--report-timeout TIMEOUT', Float, help) do |timeout|
+            queue_config.report_timeout = timeout
+          end
+
+          help = <<~EOS
+            Specify a timeout after the report will fail if all workers are inactive (e.g. died).
+            Defaults to the value set for --timeout.
+          EOS
+          opts.separator ""
+          opts.on('--inactive-workers-timeout TIMEOUT', Float, help) do |timeout|
+            queue_config.inactive_workers_timeout = timeout
+          end
+
           help = <<~EOS
             Specify a timeout to elect the leader and populate the queue.
             Defaults to the value set for --timeout.
@@ -487,6 +511,14 @@ module Minitest
             end
           end
 
+          help = <<~EOS
+            Defines how long the test report remain after the test run, in seconds.
+            Defaults to 28,800 (8 hours)
+          EOS
+          opts.on("--redis-ttl SECONDS", Integer, help) do |time|
+            queue.config.redis_ttl = time
+          end
+
           opts.separator ""
           opts.separator "    retry: Replays a previous run in the same order."
 
@@ -535,6 +567,11 @@ module Minitest
         puts red(message)
         exit! 1 # exit! is required to avoid minitest at_exit callback
       end
+
+      def retry?
+        ENV["BUILDKITE_RETRY_COUNT"].to_i > 0 ||
+          ENV["SEMAPHORE_PIPELINE_RERUN"] == "true"
+      end
     end
   end
 end

data/lib/rspec/queue.rb

@@ -157,6 +157,15 @@ module RSpec
           queue_config.max_consecutive_failures = Integer(max)
         end
 
+        help = <<~EOS
+          Defines how long the test report remain after the test run, in seconds.
+          Defaults to 28,800 (8 hours)
+        EOS
+        parser.separator ""
+        parser.on("--redis-ttl SECONDS", Integer, help) do |time|
+          queue.config.redis_ttl = time
+        end
+
         parser
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.22.1
+  version: 0.24.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-01-25 00:00:00.000000000 Z
+date: 2022-07-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -171,6 +171,7 @@ files:
 - dev.yml
 - exe/minitest-queue
 - exe/rspec-queue
+- isogun.yml
 - lib/ci/queue.rb
 - lib/ci/queue/bisect.rb
 - lib/ci/queue/build_record.rb
@@ -218,7 +219,6 @@ files:
 - lib/rspec/queue.rb
 - lib/rspec/queue/build_status_recorder.rb
 - lib/rspec/queue/order_recorder.rb
-- railgun.yml
 homepage: https://github.com/Shopify/ci-queue
 licenses:
 - MIT
@@ -239,7 +239,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.20
+rubygems_version: 3.3.3
 signing_key: 
 specification_version: 4
 summary: Distribute tests over many workers using a queue