ci-queue 0.20.9 → 0.22.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ce180525dfe099a326292c007d5654789406e1a4af4ed74455ea036891fd5e3b
-  data.tar.gz: 570f697ebc2d320e3a556b0b7323d8a2cc9920e3263e07ebef8fa2fc3517500e
+  metadata.gz: 2847d2cc56c64030f51a65ef6b9262d04b647b15b6d6ef846cf76adb90715e44
+  data.tar.gz: bf4f65911c9f39c2e6049abc6006f853f8de1aa444dfe8d5c7237cd698d0a59f
 SHA512:
-  metadata.gz: abe9a5431c9e61435f8f93ede21eae622c000b27d2359d24171e3e0bc10562d6b9c87f10138ccbd3d12e149aec09601ba2007510e1e629b1564df43a05675ce2
-  data.tar.gz: d504aa4e467eba720a2aaa35bb448dbbf13d52502d6e16a491afcfbfa8413face59eefc5b260c5412889b3d7dc0e65c194569e676e6f656147a163e67f622394
+  metadata.gz: b7c62dd52f53069dd2aad2b51724334e7ea2e8fc2cdc8f19f022b6dfd926bb7323437c3f6d1c4ffe7071316c07cde6b924671be07a2e7b402e2d528becd16acf
+  data.tar.gz: 056b3a585d95b88479f79448dd204eef72e16635c0c2df5c619ad78be2caf3b97a6e52c15edf09f2a2dcccf80afa3ea0018cfe9361114f6d03914383ab080efb

data/ci-queue.gemspec

@@ -32,7 +32,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'minitest', ENV.fetch('MINITEST_VERSION', '~> 5.11')
-  spec.add_development_dependency 'rspec', '~> 3.7.0'
+  spec.add_development_dependency 'rspec', '~> 3.10'
   spec.add_development_dependency 'redis'
   spec.add_development_dependency 'simplecov', '~> 0.12'
   spec.add_development_dependency 'minitest-reporters', '~> 1.1'

data/lib/ci/queue/bisect.rb

@@ -31,6 +31,10 @@ module CI
         Static.new(first_half + [config.failing_test], config).populate(@all_tests)
       end
 
+      def release!
+        # noop
+      end
+
       def failed!
         @tests = first_half
       end

data/lib/ci/queue/common.rb

@@ -11,6 +11,10 @@ module CI
         false
       end
 
+      def release!
+        # noop
+      end
+
       def flaky?(test)
         @config.flaky?(test)
       end

data/lib/ci/queue/redis/acknowledge.lua

@@ -1,8 +1,10 @@
 -- AUTOGENERATED FILE DO NOT EDIT DIRECTLY
 local zset_key = KEYS[1]
 local processed_key = KEYS[2]
+local owners_key = KEYS[3]
 
 local test = ARGV[1]
 
 redis.call('zrem', zset_key, test)
+redis.call('hdel', owners_key, test)  -- Doesn't matter if it was reclaimed by another workers
 return redis.call('sadd', processed_key, test)

data/lib/ci/queue/redis/base.rb

@@ -21,16 +21,16 @@ module CI
         end
 
         def size
-          redis.multi do
-            redis.llen(key('queue'))
-            redis.zcard(key('running'))
+          redis.multi do |transaction|
+            transaction.llen(key('queue'))
+            transaction.zcard(key('running'))
           end.inject(:+)
         end
 
         def to_a
-          redis.multi do
-            redis.lrange(key('queue'), 0, -1)
-            redis.zrange(key('running'), 0, -1)
+          redis.multi do |transaction|
+            transaction.lrange(key('queue'), 0, -1)
+            transaction.zrange(key('running'), 0, -1)
           end.flatten.reverse.map { |k| index.fetch(k) }
         end
 

data/lib/ci/queue/redis/build_record.rb

@@ -22,9 +22,9 @@ module CI
         end
 
         def pop_warnings
-          warnings = redis.multi do
-            redis.lrange(key('warnings'), 0, -1)
-            redis.del(key('warnings'))
+          warnings = redis.multi do |transaction|
+            transaction.lrange(key('warnings'), 0, -1)
+            transaction.del(key('warnings'))
           end.first
 
           warnings.map { |p| Marshal.load(p) }
@@ -35,21 +35,21 @@ module CI
         end
 
         def record_error(id, payload, stats: nil)
-          redis.pipelined do
-            redis.hset(
+          redis.pipelined do |pipeline|
+            pipeline.hset(
               key('error-reports'),
               id.dup.force_encoding(Encoding::BINARY),
               payload.dup.force_encoding(Encoding::BINARY),
             )
-            record_stats(stats)
+            record_stats(stats, pipeline: pipeline)
           end
           nil
         end
 
         def record_success(id, stats: nil)
-          redis.pipelined do
-            redis.hdel(key('error-reports'), id.dup.force_encoding(Encoding::BINARY))
-            record_stats(stats)
+          redis.pipelined do |pipeline|
+            pipeline.hdel(key('error-reports'), id.dup.force_encoding(Encoding::BINARY))
+            record_stats(stats, pipeline: pipeline)
           end
           nil
         end
@@ -65,8 +65,8 @@ module CI
         end
 
         def fetch_stats(stat_names)
-          counts = redis.pipelined do
-            stat_names.each { |c| redis.hvals(key(c)) }
+          counts = redis.pipelined do |pipeline|
+            stat_names.each { |c| pipeline.hvals(key(c)) }
           end
           sum_counts = counts.map do |values|
             values.map(&:to_f).inject(:+).to_f
@@ -75,9 +75,9 @@ module CI
         end
 
         def reset_stats(stat_names)
-          redis.pipelined do
+          redis.pipelined do |pipeline|
             stat_names.each do |stat_name|
-              redis.hdel(key(stat_name), config.worker_id)
+              pipeline.hdel(key(stat_name), config.worker_id)
             end
           end
         end
@@ -86,10 +86,10 @@ module CI
 
         attr_reader :config, :redis
 
-        def record_stats(stats)
+        def record_stats(stats, pipeline: redis)
           return unless stats
           stats.each do |stat_name, stat_value|
-            redis.hset(key(stat_name), config.worker_id, stat_value)
+            pipeline.hset(key(stat_name), config.worker_id, stat_value)
           end
         end
 

data/lib/ci/queue/redis/grind_record.rb

@@ -11,12 +11,12 @@ module CI
         end
 
         def record_error(payload, stats: nil)
-          redis.pipelined do
-            redis.lpush(
+          redis.pipelined do |pipeline|
+            pipeline.lpush(
               key('error-reports'),
               payload.force_encoding(Encoding::BINARY),
             )
-            record_stats(stats)
+            record_stats(stats, pipeline: pipeline)
           end
           nil
         end
@@ -34,8 +34,8 @@ module CI
         end
 
         def fetch_stats(stat_names)
-          counts = redis.pipelined do
-            stat_names.each { |c| redis.hvals(key(c)) }
+          counts = redis.pipelined do |pipeline|
+            stat_names.each { |c| pipeline.hvals(key(c)) }
           end
           stat_names.zip(counts.map { |values| values.map(&:to_f).inject(:+).to_f }).to_h
         end
@@ -54,10 +54,10 @@ module CI
           ['build', config.build_id, *args].join(':')
         end
 
-        def record_stats(stats)
+        def record_stats(stats, pipeline: redis)
           return unless stats
           stats.each do |stat_name, stat_value|
-            redis.hset(key(stat_name), config.worker_id, stat_value)
+            pipeline.hset(key(stat_name), config.worker_id, stat_value)
           end
         end
       end

data/lib/ci/queue/redis/release.lua

@@ -0,0 +1,16 @@
+-- AUTOGENERATED FILE DO NOT EDIT DIRECTLY
+local zset_key = KEYS[1]
+local worker_queue_key = KEYS[2]
+local owners_key = KEYS[3]
+
+-- owned_tests = {"SomeTest", "worker:1", "SomeOtherTest", "worker:2", ...}
+local owned_tests = redis.call('hgetall', owners_key)
+for index, owner_or_test in ipairs(owned_tests) do
+  if owner_or_test == worker_queue_key then -- If we owned a test
+    local test = owned_tests[index - 1]
+    redis.call('zadd', zset_key, "0", test) -- We expire the lease immediately
+    return nil
+  end
+end
+
+return nil

data/lib/ci/queue/redis/requeue.lua

@@ -3,12 +3,18 @@ local processed_key = KEYS[1]
 local requeues_count_key = KEYS[2]
 local queue_key = KEYS[3]
 local zset_key = KEYS[4]
+local worker_queue_key = KEYS[5]
+local owners_key = KEYS[6]
 
 local max_requeues = tonumber(ARGV[1])
 local global_max_requeues = tonumber(ARGV[2])
 local test = ARGV[3]
 local offset = ARGV[4]
 
+if redis.call('hget', owners_key, test) == worker_queue_key then
+   redis.call('hdel', owners_key, test)
+end
+
 if redis.call('sismember', processed_key, test) == 1 then
   return false
 end

data/lib/ci/queue/redis/reserve.lua

@@ -3,6 +3,7 @@ local queue_key = KEYS[1]
 local zset_key = KEYS[2]
 local processed_key = KEYS[3]
 local worker_queue_key = KEYS[4]
+local owners_key = KEYS[5]
 
 local current_time = ARGV[1]
 
@@ -10,6 +11,7 @@ local test = redis.call('rpop', queue_key)
 if test then
   redis.call('zadd', zset_key, current_time, test)
   redis.call('lpush', worker_queue_key, test)
+  redis.call('hset', owners_key, test, worker_queue_key)
   return test
 else
   return nil

data/lib/ci/queue/redis/reserve_lost.lua

@@ -2,6 +2,7 @@
 local zset_key = KEYS[1]
 local processed_key = KEYS[2]
 local worker_queue_key = KEYS[3]
+local owners_key = KEYS[4]
 
 local current_time = ARGV[1]
 local timeout = ARGV[2]
@@ -11,6 +12,7 @@ for _, test in ipairs(lost_tests) do
   if redis.call('sismember', processed_key, test) == 0 then
     redis.call('zadd', zset_key, current_time, test)
     redis.call('lpush', worker_queue_key, test)
+    redis.call('hset', owners_key, test, worker_queue_key) -- Take ownership
     return test
   end
 end

data/lib/ci/queue/redis/test_time_record.rb

@@ -19,8 +19,8 @@ module CI
         attr_reader :redis
 
         def record_test_time(test_name, duration)
-          redis.pipelined do
-            redis.lpush(
+          redis.pipelined do |pipeline|
+            pipeline.lpush(
               test_time_key(test_name),
               duration.to_s.force_encoding(Encoding::BINARY),
             )
@@ -29,8 +29,8 @@ module CI
         end
 
         def record_test_name(test_name)
-          redis.pipelined do
-            redis.lpush(
+          redis.pipelined do |pipeline|
+            pipeline.lpush(
               all_test_names_key,
               test_name.dup.force_encoding(Encoding::BINARY),
             )
@@ -39,18 +39,15 @@ module CI
         end
 
         def fetch_all_test_names
-          values = redis.pipelined do
-            redis.lrange(all_test_names_key, 0, -1)
+          values = redis.pipelined do |pipeline|
+            pipeline.lrange(all_test_names_key, 0, -1)
           end
           values.flatten.map(&:to_s)
         end
 
         def fetch_test_time(test_name)
-          values = redis.pipelined do
-            key = test_time_key(test_name)
-            redis.lrange(key, 0, -1)
-          end
-          values.flatten.map(&:to_f)
+          key = test_time_key(test_name)
+          redis.lrange(key, 0, -1).map(&:to_f)
         end
 
         def all_test_names_key

data/lib/ci/queue/redis/worker.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 require 'ci/queue/static'
+require 'set'
 
 module CI
   module Queue
@@ -15,7 +16,6 @@ module CI
         attr_reader :total
 
         def initialize(redis, config)
-          @last_warning = nil
           @reserved_test = nil
           @shutdown_required = false
           super(redis, config)
@@ -48,7 +48,7 @@ module CI
           wait_for_master
           until shutdown_required? || config.circuit_breakers.any?(&:open?) || exhausted? || max_test_failed?
             if test = reserve
-              yield index.fetch(test), @last_warning
+              yield index.fetch(test)
             else
               sleep 0.05
             end
@@ -92,7 +92,7 @@ module CI
           raise_on_mismatching_test(test_key)
           eval_script(
             :acknowledge,
-            keys: [key('running'), key('processed')],
+            keys: [key('running'), key('processed'), key('owners')],
             argv: [test_key],
           ) == 1
         end
@@ -104,7 +104,14 @@ module CI
 
           requeued = config.max_requeues > 0 && global_max_requeues > 0 && eval_script(
             :requeue,
-            keys: [key('processed'), key('requeues-count'), key('queue'), key('running')],
+            keys: [
+              key('processed'),
+              key('requeues-count'),
+              key('queue'),
+              key('running'),
+              key('worker', worker_id, 'queue'),
+              key('owners'),
+            ],
             argv: [config.max_requeues, global_max_requeues, test_key, offset],
           ) == 1
 
@@ -112,6 +119,15 @@ module CI
           requeued
         end
 
+        def release!
+          eval_script(
+            :release,
+            keys: [key('running'), key('worker', worker_id, 'queue'), key('owners')],
+            argv: [],
+          )
+          nil
+        end
+
         private
 
         attr_reader :index
@@ -144,7 +160,13 @@ module CI
         def try_to_reserve_test
           eval_script(
             :reserve,
-            keys: [key('queue'), key('running'), key('processed'), key('worker', worker_id, 'queue')],
+            keys: [
+              key('queue'),
+              key('running'),
+              key('processed'),
+              key('worker', worker_id, 'queue'),
+              key('owners'),
+            ],
             argv: [Time.now.to_f],
           )
         end
@@ -152,7 +174,12 @@ module CI
         def try_to_reserve_lost_test
           lost_test = eval_script(
             :reserve_lost,
-            keys: [key('running'), key('completed'), key('worker', worker_id, 'queue')],
+            keys: [
+              key('running'),
+              key('completed'),
+              key('worker', worker_id, 'queue'),
+              key('owners'),
+            ],
             argv: [Time.now.to_f, timeout],
           )
 
@@ -167,10 +194,10 @@ module CI
           @total = tests.size
 
           if @master = redis.setnx(key('master-status'), 'setup')
-            redis.multi do
-              redis.lpush(key('queue'), tests) unless tests.empty?
-              redis.set(key('total'), @total)
-              redis.set(key('master-status'), 'ready')
+            redis.multi do |transaction|
+              transaction.lpush(key('queue'), tests) unless tests.empty?
+              transaction.set(key('total'), @total)
+              transaction.set(key('master-status'), 'ready')
             end
           end
           register

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.20.9'
+    VERSION = '0.22.1'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue/runner.rb

@@ -81,6 +81,10 @@ module Minitest
         # Let minitest's at_exit hook trigger
       end
 
+      def release_command
+        queue.release!
+      end
+
       def grind_command
         invalid_usage!('No list to grind provided') if grind_list.nil?
         invalid_usage!('No grind count provided') if grind_count.nil?

data/lib/minitest/queue/test_data.rb

@@ -52,6 +52,14 @@ module Minitest
         @test.time
       end
 
+      def test_start_timestamp
+        @test.start_timestamp
+      end
+
+      def test_finish_timestamp
+        @test.finish_timestamp
+      end
+
       def test_file_path
         path = @test.source_location.first
         begin
@@ -107,6 +115,8 @@ module Minitest
           test_retried: test_retried,
           test_assertions: test_assertions,
           test_duration: test_duration,
+          test_start_timestamp: test_start_timestamp,
+          test_finish_timestamp: test_finish_timestamp,
           test_file_path: test_file_path,
           test_file_line_number: test_file_line_number,
           error_class: error_class,

data/lib/minitest/queue.rb

@@ -102,6 +102,10 @@ module Minitest
     end
   end
 
+  module WithTimestamps
+    attr_accessor :start_timestamp, :finish_timestamp
+  end
+
   module Queue
     attr_writer :run_command_formatter, :project_root
 
@@ -159,21 +163,36 @@ module Minitest
         id <=> other.id
       end
 
+      def with_timestamps
+        start_timestamp = current_timestamp
+        result = yield
+        result
+      ensure
+        if result
+          result.start_timestamp = start_timestamp
+          result.finish_timestamp = current_timestamp
+        end
+      end
+
       def run
-        Minitest.run_one_method(@runnable, @method_name)
+        with_timestamps do
+          Minitest.run_one_method(@runnable, @method_name)
+        end
       end
 
       def flaky?
         Minitest.queue.flaky?(self)
       end
-    end
 
-    attr_reader :queue
+      private
 
-    def queue=(queue)
-      @queue = queue
+      def current_timestamp
+        Time.now.to_i
+      end
     end
 
+    attr_accessor :queue
+
     def queue_reporters=(reporters)
       @queue_reporters ||= []
       Reporters.use!(((Reporters.reporters || []) - @queue_reporters) + reporters)
@@ -244,9 +263,11 @@ MiniTest.singleton_class.prepend(MiniTest::Queue)
 if defined? MiniTest::Result
   MiniTest::Result.prepend(MiniTest::Requeueing)
   MiniTest::Result.prepend(MiniTest::Flakiness)
+  MiniTest::Result.prepend(MiniTest::WithTimestamps)
 else
   MiniTest::Test.prepend(MiniTest::Requeueing)
   MiniTest::Test.prepend(MiniTest::Flakiness)
+  MiniTest::Test.prepend(MiniTest::WithTimestamps)
 
   module MinitestBackwardCompatibility
     def source_location

data/lib/rspec/queue.rb

@@ -112,7 +112,7 @@ module RSpec
 
         help = <<~EOS
           Specify a timeout after which if a test haven't completed, it will be picked up by another worker.
-          It is very important to set this vlaue higher than the slowest test in the suite, otherwise performance will be impacted.
+          It is very important to set this value higher than the slowest test in the suite, otherwise performance will be impacted.
           Defaults to 30 seconds.
         EOS
         parser.separator ""

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.20.9
+  version: 0.22.1
 platform: ruby
 authors:
 - Jean Boussier
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-09-14 00:00:00.000000000 Z
+date: 2022-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.7.0
+        version: '3.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.7.0
+        version: '3.10'
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
@@ -187,6 +187,7 @@ files:
 - lib/ci/queue/redis/grind.rb
 - lib/ci/queue/redis/grind_record.rb
 - lib/ci/queue/redis/grind_supervisor.rb
+- lib/ci/queue/redis/release.lua
 - lib/ci/queue/redis/requeue.lua
 - lib/ci/queue/redis/reserve.lua
 - lib/ci/queue/redis/reserve_lost.lua