ci-queue 0.22.0 → 0.23.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a727d6a5f5c01a4b5837d1f7ed629ce3f9c971fed502d6b850a43b384e72c59a
-  data.tar.gz: 3ab46419fd8c1932d2e94cd40bbfb087df927d9013f9d0e998f3ed60839440e6
+  metadata.gz: aaa8a7cb585452d354e884d29772f29e0ea6dda29e894a4e69088ab302cec1b7
+  data.tar.gz: 8dc28379b0488fce6b951a0bf5ac434cce71fa5751c453f807ee6412edc86ef4
 SHA512:
-  metadata.gz: f6fd6c5f71fc10b0b5085b0d5259aa25eb1b1cc969f7b182e45c975deff56e72b1bef6e133adb812db87809feda33044ad7e137f36d88c5866c34a469af4b0fd
-  data.tar.gz: 0415f91c5aa6c1bdfd185842ae88ee5d73ed3c553945eeacd2a095288156336f4bb00d2d49ab302f7758d7fcbb78c7fcd2b500d6265c12aa3b33dfbc2d5d7829
+  metadata.gz: b2fe2f4bfd08b4165576b2f881fdad537b22566fe8c940ee87a03e1ecb13488398b3a7e86147f83f284603c007cffbd606f8576b5f5a17a7d7c034f7dbacdf41
+  data.tar.gz: d12edaffb47e719b0c9e437da5261c4808dac54f2fa1e080d857c36225550a3f3a6d20517fe54e0da3ddaf1cb6d965fa6e666866049564ce741f6573aa187ff5

data/README.md

@@ -70,3 +70,9 @@ rspec-queue --queue redis://example.com --timeout 600 --report
 #### Limitations
 
 Because of how `ci-queue` executes the examples, `before(:all)` and `after(:all)` hooks are not supported. `rspec-queue` will explicitly reject them.
+
+## Custom Redis Expiry
+
+`ci-queue` expects the Redis server to have an [eviction policy](https://redis.io/docs/manual/eviction/#eviction-policies) of `allkeys-lru`.
+
+You can also use `--redis-ttl` to set a custom expiration time for all CI Queue keys, this defaults to 8 hours (28,800 seconds)

data/ci-queue.gemspec

@@ -32,7 +32,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'minitest', ENV.fetch('MINITEST_VERSION', '~> 5.11')
-  spec.add_development_dependency 'rspec', '~> 3.7.0'
+  spec.add_development_dependency 'rspec', '~> 3.10'
   spec.add_development_dependency 'redis'
   spec.add_development_dependency 'simplecov', '~> 0.12'
   spec.add_development_dependency 'minitest-reporters', '~> 1.1'

data/lib/ci/queue/configuration.rb

@@ -5,7 +5,7 @@ module CI
       attr_accessor :timeout, :worker_id, :max_requeues, :grind_count, :failure_file
       attr_accessor :requeue_tolerance, :namespace, :failing_test, :statsd_endpoint
       attr_accessor :max_test_duration, :max_test_duration_percentile, :track_test_duration
-      attr_accessor :max_test_failed
+      attr_accessor :max_test_failed, :redis_ttl
       attr_reader :circuit_breakers
       attr_writer :seed, :build_id
       attr_writer :queue_init_timeout
@@ -18,6 +18,7 @@ module CI
             seed: env['CIRCLE_SHA1'] || env['BUILDKITE_COMMIT'] || env['TRAVIS_COMMIT'] || env['HEROKU_TEST_RUN_COMMIT_VERSION'] || env['SEMAPHORE_GIT_SHA'],
             flaky_tests: load_flaky_tests(env['CI_QUEUE_FLAKY_TESTS']),
             statsd_endpoint: env['CI_QUEUE_STATSD_ADDR'],
+            redis_ttl: env['CI_QUEUE_REDIS_TTL']&.to_i ||  8 * 60 * 60,
           )
         end
 
@@ -34,7 +35,7 @@ module CI
         namespace: nil, seed: nil, flaky_tests: [], statsd_endpoint: nil, max_consecutive_failures: nil,
         grind_count: nil, max_duration: nil, failure_file: nil, max_test_duration: nil,
         max_test_duration_percentile: 0.5, track_test_duration: false, max_test_failed: nil,
-        queue_init_timeout: nil
+        queue_init_timeout: nil, redis_ttl: 8 * 60 * 60
       )
         @build_id = build_id
         @circuit_breakers = [CircuitBreaker::Disabled]
@@ -55,6 +56,7 @@ module CI
         @worker_id = worker_id
         self.max_consecutive_failures = max_consecutive_failures
         self.max_duration = max_duration
+        @redis_ttl = redis_ttl
       end
 
       def queue_init_timeout

data/lib/ci/queue/redis/base.rb

@@ -21,16 +21,16 @@ module CI
         end
 
         def size
-          redis.multi do
-            redis.llen(key('queue'))
-            redis.zcard(key('running'))
+          redis.multi do |transaction|
+            transaction.llen(key('queue'))
+            transaction.zcard(key('running'))
           end.inject(:+)
         end
 
         def to_a
-          redis.multi do
-            redis.lrange(key('queue'), 0, -1)
-            redis.zrange(key('running'), 0, -1)
+          redis.multi do |transaction|
+            transaction.lrange(key('queue'), 0, -1)
+            transaction.zrange(key('running'), 0, -1)
           end.flatten.reverse.map { |k| index.fetch(k) }
         end
 

data/lib/ci/queue/redis/build_record.rb

@@ -22,9 +22,9 @@ module CI
         end
 
         def pop_warnings
-          warnings = redis.multi do
-            redis.lrange(key('warnings'), 0, -1)
-            redis.del(key('warnings'))
+          warnings = redis.multi do |transaction|
+            transaction.lrange(key('warnings'), 0, -1)
+            transaction.del(key('warnings'))
           end.first
 
           warnings.map { |p| Marshal.load(p) }
@@ -35,21 +35,22 @@ module CI
         end
 
         def record_error(id, payload, stats: nil)
-          redis.pipelined do
-            redis.hset(
+          redis.pipelined do |pipeline|
+            pipeline.hset(
               key('error-reports'),
               id.dup.force_encoding(Encoding::BINARY),
               payload.dup.force_encoding(Encoding::BINARY),
             )
-            record_stats(stats)
+            pipeline.expire(key('error-reports'), config.redis_ttl)
+            record_stats(stats, pipeline: pipeline)
           end
           nil
         end
 
         def record_success(id, stats: nil)
-          redis.pipelined do
-            redis.hdel(key('error-reports'), id.dup.force_encoding(Encoding::BINARY))
-            record_stats(stats)
+          redis.pipelined do |pipeline|
+            pipeline.hdel(key('error-reports'), id.dup.force_encoding(Encoding::BINARY))
+            record_stats(stats, pipeline: pipeline)
           end
           nil
         end
@@ -65,8 +66,8 @@ module CI
         end
 
         def fetch_stats(stat_names)
-          counts = redis.pipelined do
-            stat_names.each { |c| redis.hvals(key(c)) }
+          counts = redis.pipelined do |pipeline|
+            stat_names.each { |c| pipeline.hvals(key(c)) }
           end
           sum_counts = counts.map do |values|
             values.map(&:to_f).inject(:+).to_f
@@ -75,9 +76,9 @@ module CI
         end
 
         def reset_stats(stat_names)
-          redis.pipelined do
+          redis.pipelined do |pipeline|
             stat_names.each do |stat_name|
-              redis.hdel(key(stat_name), config.worker_id)
+              pipeline.hdel(key(stat_name), config.worker_id)
             end
           end
         end
@@ -86,10 +87,11 @@ module CI
 
         attr_reader :config, :redis
 
-        def record_stats(stats)
+        def record_stats(stats, pipeline: redis)
           return unless stats
           stats.each do |stat_name, stat_value|
-            redis.hset(key(stat_name), config.worker_id, stat_value)
+            pipeline.hset(key(stat_name), config.worker_id, stat_value)
+            pipeline.expire(key(stat_name), config.redis_ttl)
           end
         end
 

data/lib/ci/queue/redis/grind_record.rb

@@ -11,12 +11,13 @@ module CI
         end
 
         def record_error(payload, stats: nil)
-          redis.pipelined do
-            redis.lpush(
+          redis.pipelined do |pipeline|
+            pipeline.lpush(
               key('error-reports'),
               payload.force_encoding(Encoding::BINARY),
             )
-            record_stats(stats)
+            pipeline.expire(key('error-reports'), config.redis_ttl)
+            record_stats(stats, pipeline: pipeline)
           end
           nil
         end
@@ -34,8 +35,8 @@ module CI
         end
 
         def fetch_stats(stat_names)
-          counts = redis.pipelined do
-            stat_names.each { |c| redis.hvals(key(c)) }
+          counts = redis.pipelined do |pipeline|
+            stat_names.each { |c| pipeline.hvals(key(c)) }
           end
           stat_names.zip(counts.map { |values| values.map(&:to_f).inject(:+).to_f }).to_h
         end
@@ -54,10 +55,11 @@ module CI
           ['build', config.build_id, *args].join(':')
         end
 
-        def record_stats(stats)
+        def record_stats(stats, pipeline: redis)
           return unless stats
           stats.each do |stat_name, stat_value|
-            redis.hset(key(stat_name), config.worker_id, stat_value)
+            pipeline.hset(key(stat_name), config.worker_id, stat_value)
+            pipeline.expire(key(stat_name), config.redis_ttl)
           end
         end
       end

data/lib/ci/queue/redis/test_time_record.rb

@@ -19,38 +19,37 @@ module CI
         attr_reader :redis
 
         def record_test_time(test_name, duration)
-          redis.pipelined do
-            redis.lpush(
+          redis.pipelined do |pipeline|
+            pipeline.lpush(
               test_time_key(test_name),
               duration.to_s.force_encoding(Encoding::BINARY),
             )
+            pipeline.expire(test_time_key(test_name), config.redis_ttl)
           end
           nil
         end
 
         def record_test_name(test_name)
-          redis.pipelined do
-            redis.lpush(
+          redis.pipelined do |pipeline|
+            pipeline.lpush(
               all_test_names_key,
               test_name.dup.force_encoding(Encoding::BINARY),
             )
+            pipeline.expire(all_test_names_key, config.redis_ttl)
           end
           nil
         end
 
         def fetch_all_test_names
-          values = redis.pipelined do
-            redis.lrange(all_test_names_key, 0, -1)
+          values = redis.pipelined do |pipeline|
+            pipeline.lrange(all_test_names_key, 0, -1)
           end
           values.flatten.map(&:to_s)
         end
 
         def fetch_test_time(test_name)
-          values = redis.pipelined do
-            key = test_time_key(test_name)
-            redis.lrange(key, 0, -1)
-          end
-          values.flatten.map(&:to_f)
+          key = test_time_key(test_name)
+          redis.lrange(key, 0, -1).map(&:to_f)
         end
 
         def all_test_names_key

data/lib/ci/queue/redis/worker.rb

@@ -53,6 +53,10 @@ module CI
               sleep 0.05
             end
           end
+          redis.pipelined do |pipeline|
+            pipeline.expire(key('worker', worker_id, 'queue'), config.redis_ttl)
+            pipeline.expire(key('processed'), config.redis_ttl)
+          end
         rescue *CONNECTION_ERRORS
         end
 
@@ -194,13 +198,18 @@ module CI
           @total = tests.size
 
           if @master = redis.setnx(key('master-status'), 'setup')
-            redis.multi do
-              redis.lpush(key('queue'), tests) unless tests.empty?
-              redis.set(key('total'), @total)
-              redis.set(key('master-status'), 'ready')
+            redis.multi do |transaction|
+              transaction.lpush(key('queue'), tests) unless tests.empty?
+              transaction.set(key('total'), @total)
+              transaction.set(key('master-status'), 'ready')
+
+              transaction.expire(key('queue'), config.redis_ttl)
+              transaction.expire(key('total'), config.redis_ttl)
+              transaction.expire(key('master-status'), config.redis_ttl)
             end
           end
           register
+          redis.expire(key('workers'), config.redis_ttl)
         rescue *CONNECTION_ERRORS
           raise if @master
         end

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.22.0'
+    VERSION = '0.23.1'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue/runner.rb

@@ -22,6 +22,9 @@ module Minitest
       def initialize(argv)
         @queue_config = CI::Queue::Configuration.from_env(ENV)
         @command, @argv = parse(argv)
+        if Minitest.respond_to?(:seed=)
+          Minitest.seed = @queue_config.seed.to_i
+        end
       end
 
       def run!
@@ -487,6 +490,14 @@ module Minitest
             end
           end
 
+          help = <<~EOS
+            Defines how long the test report remain after the test run, in seconds.
+            Defaults to 28,800 (8 hours)
+          EOS
+          opts.on("--redis-ttl SECONDS", Integer, help) do |time|
+            queue.config.redis_ttl = time
+          end
+
           opts.separator ""
           opts.separator "    retry: Replays a previous run in the same order."
 

data/lib/rspec/queue.rb

@@ -157,6 +157,15 @@ module RSpec
           queue_config.max_consecutive_failures = Integer(max)
         end
 
+        help = <<~EOS
+          Defines how long the test report remain after the test run, in seconds.
+          Defaults to 28,800 (8 hours)
+        EOS
+        parser.separator ""
+        parser.on("--redis-ttl SECONDS", Integer, help) do |time|
+          queue.config.redis_ttl = time
+        end
+
         parser
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.22.0
+  version: 0.23.1
 platform: ruby
 authors:
 - Jean Boussier
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-12-14 00:00:00.000000000 Z
+date: 2022-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.7.0
+        version: '3.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.7.0
+        version: '3.10'
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
@@ -239,7 +239,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.20
+rubygems_version: 3.3.3
 signing_key: 
 specification_version: 4
 summary: Distribute tests over many workers using a queue