ci-queue 0.21.1 → 0.22.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bb09b44ca70cde6218873faed3b7067295088a115b2b33ddc93b3853ad318042
-  data.tar.gz: d0eb4a637fb89aa8c2858e5f1c363b86518cb310729d417fc6e7847c7bd261b8
+  metadata.gz: a727d6a5f5c01a4b5837d1f7ed629ce3f9c971fed502d6b850a43b384e72c59a
+  data.tar.gz: 3ab46419fd8c1932d2e94cd40bbfb087df927d9013f9d0e998f3ed60839440e6
 SHA512:
-  metadata.gz: e3b35848df951df70cddc2bab778f5160dc0293f09cc66218b31567ce8b2fe7e7a020f5aeec9fb2b6890e53a175b70ada164b404306ee5582cc1a40601515643
-  data.tar.gz: 8f5ffdaaad075ddcc724117f3b2414f7a15b49509d0b0df5b4415827b6452efc251974398a58b2203c776fdb70819ab53c8042ce4d3add056b245ff8cafed179
+  metadata.gz: f6fd6c5f71fc10b0b5085b0d5259aa25eb1b1cc969f7b182e45c975deff56e72b1bef6e133adb812db87809feda33044ad7e137f36d88c5866c34a469af4b0fd
+  data.tar.gz: 0415f91c5aa6c1bdfd185842ae88ee5d73ed3c553945eeacd2a095288156336f4bb00d2d49ab302f7758d7fcbb78c7fcd2b500d6265c12aa3b33dfbc2d5d7829

data/lib/ci/queue/bisect.rb

@@ -31,6 +31,10 @@ module CI
         Static.new(first_half + [config.failing_test], config).populate(@all_tests)
       end
 
+      def release!
+        # noop
+      end
+
       def failed!
         @tests = first_half
       end

data/lib/ci/queue/common.rb

@@ -11,6 +11,10 @@ module CI
         false
       end
 
+      def release!
+        # noop
+      end
+
       def flaky?(test)
         @config.flaky?(test)
       end

data/lib/ci/queue/redis/acknowledge.lua

@@ -1,8 +1,10 @@
 -- AUTOGENERATED FILE DO NOT EDIT DIRECTLY
 local zset_key = KEYS[1]
 local processed_key = KEYS[2]
+local owners_key = KEYS[3]
 
 local test = ARGV[1]
 
 redis.call('zrem', zset_key, test)
+redis.call('hdel', owners_key, test)  -- Doesn't matter if it was reclaimed by another workers
 return redis.call('sadd', processed_key, test)

data/lib/ci/queue/redis/release.lua

@@ -0,0 +1,16 @@
+-- AUTOGENERATED FILE DO NOT EDIT DIRECTLY
+local zset_key = KEYS[1]
+local worker_queue_key = KEYS[2]
+local owners_key = KEYS[3]
+
+-- owned_tests = {"SomeTest", "worker:1", "SomeOtherTest", "worker:2", ...}
+local owned_tests = redis.call('hgetall', owners_key)
+for index, owner_or_test in ipairs(owned_tests) do
+  if owner_or_test == worker_queue_key then -- If we owned a test
+    local test = owned_tests[index - 1]
+    redis.call('zadd', zset_key, "0", test) -- We expire the lease immediately
+    return nil
+  end
+end
+
+return nil

data/lib/ci/queue/redis/requeue.lua

@@ -3,12 +3,18 @@ local processed_key = KEYS[1]
 local requeues_count_key = KEYS[2]
 local queue_key = KEYS[3]
 local zset_key = KEYS[4]
+local worker_queue_key = KEYS[5]
+local owners_key = KEYS[6]
 
 local max_requeues = tonumber(ARGV[1])
 local global_max_requeues = tonumber(ARGV[2])
 local test = ARGV[3]
 local offset = ARGV[4]
 
+if redis.call('hget', owners_key, test) == worker_queue_key then
+   redis.call('hdel', owners_key, test)
+end
+
 if redis.call('sismember', processed_key, test) == 1 then
   return false
 end

data/lib/ci/queue/redis/reserve.lua

@@ -3,6 +3,7 @@ local queue_key = KEYS[1]
 local zset_key = KEYS[2]
 local processed_key = KEYS[3]
 local worker_queue_key = KEYS[4]
+local owners_key = KEYS[5]
 
 local current_time = ARGV[1]
 
@@ -10,6 +11,7 @@ local test = redis.call('rpop', queue_key)
 if test then
   redis.call('zadd', zset_key, current_time, test)
   redis.call('lpush', worker_queue_key, test)
+  redis.call('hset', owners_key, test, worker_queue_key)
   return test
 else
   return nil

data/lib/ci/queue/redis/reserve_lost.lua

@@ -2,6 +2,7 @@
 local zset_key = KEYS[1]
 local processed_key = KEYS[2]
 local worker_queue_key = KEYS[3]
+local owners_key = KEYS[4]
 
 local current_time = ARGV[1]
 local timeout = ARGV[2]
@@ -11,6 +12,7 @@ for _, test in ipairs(lost_tests) do
   if redis.call('sismember', processed_key, test) == 0 then
     redis.call('zadd', zset_key, current_time, test)
     redis.call('lpush', worker_queue_key, test)
+    redis.call('hset', owners_key, test, worker_queue_key) -- Take ownership
     return test
   end
 end

data/lib/ci/queue/redis/worker.rb

@@ -92,7 +92,7 @@ module CI
           raise_on_mismatching_test(test_key)
           eval_script(
             :acknowledge,
-            keys: [key('running'), key('processed')],
+            keys: [key('running'), key('processed'), key('owners')],
             argv: [test_key],
           ) == 1
         end
@@ -104,7 +104,14 @@ module CI
 
           requeued = config.max_requeues > 0 && global_max_requeues > 0 && eval_script(
             :requeue,
-            keys: [key('processed'), key('requeues-count'), key('queue'), key('running')],
+            keys: [
+              key('processed'),
+              key('requeues-count'),
+              key('queue'),
+              key('running'),
+              key('worker', worker_id, 'queue'),
+              key('owners'),
+            ],
             argv: [config.max_requeues, global_max_requeues, test_key, offset],
           ) == 1
 
@@ -112,6 +119,15 @@ module CI
           requeued
         end
 
+        def release!
+          eval_script(
+            :release,
+            keys: [key('running'), key('worker', worker_id, 'queue'), key('owners')],
+            argv: [],
+          )
+          nil
+        end
+
         private
 
         attr_reader :index
@@ -144,7 +160,13 @@ module CI
         def try_to_reserve_test
           eval_script(
             :reserve,
-            keys: [key('queue'), key('running'), key('processed'), key('worker', worker_id, 'queue')],
+            keys: [
+              key('queue'),
+              key('running'),
+              key('processed'),
+              key('worker', worker_id, 'queue'),
+              key('owners'),
+            ],
             argv: [Time.now.to_f],
           )
         end
@@ -152,7 +174,12 @@ module CI
         def try_to_reserve_lost_test
           lost_test = eval_script(
             :reserve_lost,
-            keys: [key('running'), key('completed'), key('worker', worker_id, 'queue')],
+            keys: [
+              key('running'),
+              key('completed'),
+              key('worker', worker_id, 'queue'),
+              key('owners'),
+            ],
             argv: [Time.now.to_f, timeout],
           )
 

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.21.1'
+    VERSION = '0.22.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue/runner.rb

@@ -81,6 +81,10 @@ module Minitest
         # Let minitest's at_exit hook trigger
       end
 
+      def release_command
+        queue.release!
+      end
+
       def grind_command
         invalid_usage!('No list to grind provided') if grind_list.nil?
         invalid_usage!('No grind count provided') if grind_count.nil?

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.21.1
+  version: 0.22.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-10-20 00:00:00.000000000 Z
+date: 2021-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -187,6 +187,7 @@ files:
 - lib/ci/queue/redis/grind.rb
 - lib/ci/queue/redis/grind_record.rb
 - lib/ci/queue/redis/grind_supervisor.rb
+- lib/ci/queue/redis/release.lua
 - lib/ci/queue/redis/requeue.lua
 - lib/ci/queue/redis/reserve.lua
 - lib/ci/queue/redis/reserve_lost.lua