ci-queue 0.100.0 → 0.101.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 64510328261d26c0b3299aa0e1bb4bbbb2fdd4f784293a304cec028527d25e8b
-  data.tar.gz: 8cf5e99fcf74935b71a2ee6814cfc7565b03b1b314af8380f999f4228400417c
+  metadata.gz: 612d0f2f3077fdd5b7e74b343649a162101c2961c8b4cda8329ca10fbd831e6b
+  data.tar.gz: bdd01c95971bcbfab5999b3754f8d019be35a7202e5f766d5df19191462e5563
 SHA512:
-  metadata.gz: c044a9cafe05602f22debf2ea91b5cc63c42dabc67f8689b88430c7c34ece68da7bf7ad03b62036648073c8225e3588382bce588f486359b339a65aa62e71951
-  data.tar.gz: 5d9f475df1260d6cc6cd04d1f1ceec0ca28e7b12e9d5d0b0931ac7b18a2e864a5a2386f102f878a8392722dbb535e260b8661e5ccc5722097ee8d8e866d977dc
+  metadata.gz: 69a03983e12372fe0ab0258ec08262b2672624d49d2d63a0c35d0a89b4c8c85b3fd89ece2ce3c71de112652cd327868ee0fa7633ae283facb3b0bf57035d5724
+  data.tar.gz: 44ead0bb47b690b19652982496f8a48d698f5bb78ea4f88c01e56b7774d4c79f6b6015b50e91cb70e241fc00a13bf842a72828792c7f716644cacf12bfb8bb2d

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ci-queue (0.100.0)
+    ci-queue (0.101.0)
       logger
 
 GEM

data/lib/ci/queue/redis/requeue.lua

@@ -12,10 +12,30 @@ local leases_key = KEYS[9]
 local max_requeues = tonumber(ARGV[1])
 local global_max_requeues = tonumber(ARGV[2])
 local entry = ARGV[3]
-local offset = ARGV[4]
+local offset = tonumber(ARGV[4]) or 0
 local ttl = tonumber(ARGV[5])
 local lease_id = ARGV[6]
 
+-- Inserts `entry` behind the `offset` + 1 entries nearest the tail, the end RPOP reserves
+-- from: where `LINSERT BEFORE <entry at index -(offset + 1)>` would put it. LINSERT finds its
+-- pivot by scanning from the head, which is O(queue length) and blocks Redis on large queues;
+-- this only touches the tail, so it is O(offset). Queues of at most `offset` + 1 entries, and
+-- offsets of zero or less, push to the head instead.
+-- Keep in sync with reserve.lua: the Python client does not resolve `-- @include`.
+local function insert_with_offset(queue_key, entry, offset)
+  if offset <= 0 or redis.call('llen', queue_key) <= offset + 1 then
+    redis.call('lpush', queue_key, entry)
+    return
+  end
+
+  local ahead = redis.call('lrange', queue_key, -1 - offset, -1)
+  redis.call('ltrim', queue_key, 0, -2 - offset)
+  redis.call('rpush', queue_key, entry)
+  for _, ahead_entry in ipairs(ahead) do
+    redis.call('rpush', queue_key, ahead_entry)
+  end
+end
+
 -- Only the current lease holder can requeue a test.
 -- If the lease was transferred (e.g. via reserve_lost), reject the stale
 -- worker's requeue so the running entry stays intact for the new holder.
@@ -42,12 +62,7 @@ redis.call('hincrby', requeues_count_key, entry, 1)
 
 redis.call('hdel', error_reports_key, entry)
 
-local pivot = redis.call('lrange', queue_key, -1 - offset, 0 - offset)[1]
-if pivot then
-  redis.call('linsert', queue_key, 'BEFORE', pivot, entry)
-else
-  redis.call('lpush', queue_key, entry)
-end
+insert_with_offset(queue_key, entry, offset)
 
 redis.call('hset', requeued_by_key, entry, worker_queue_key)
 if ttl and ttl > 0 then

data/lib/ci/queue/redis/reserve.lua

@@ -13,12 +13,23 @@ local current_time = ARGV[1]
 local defer_offset = tonumber(ARGV[2]) or 0
 local max_skip_attempts = 4
 
-local function insert_with_offset(test)
-  local pivot = redis.call('lrange', queue_key, -1 - defer_offset, 0 - defer_offset)[1]
-  if pivot then
-    redis.call('linsert', queue_key, 'BEFORE', pivot, test)
-  else
-    redis.call('lpush', queue_key, test)
+-- Inserts `entry` behind the `offset` + 1 entries nearest the tail, the end RPOP reserves
+-- from: where `LINSERT BEFORE <entry at index -(offset + 1)>` would put it. LINSERT finds its
+-- pivot by scanning from the head, which is O(queue length) and blocks Redis on large queues;
+-- this only touches the tail, so it is O(offset). Queues of at most `offset` + 1 entries, and
+-- offsets of zero or less, push to the head instead.
+-- Keep in sync with requeue.lua: the Python client does not resolve `-- @include`.
+local function insert_with_offset(queue_key, entry, offset)
+  if offset <= 0 or redis.call('llen', queue_key) <= offset + 1 then
+    redis.call('lpush', queue_key, entry)
+    return
+  end
+
+  local ahead = redis.call('lrange', queue_key, -1 - offset, -1)
+  redis.call('ltrim', queue_key, 0, -2 - offset)
+  redis.call('rpush', queue_key, entry)
+  for _, ahead_entry in ipairs(ahead) do
+    redis.call('rpush', queue_key, ahead_entry)
   end
 end
 
@@ -45,7 +56,7 @@ for attempt = 1, max_skip_attempts do
       return claim_test(test)
     end
 
-    insert_with_offset(test)
+    insert_with_offset(queue_key, test, defer_offset)
 
     -- If this worker only finds its own requeued tests, defer once by returning nil,
     -- then allow pickup on a subsequent reserve attempt.

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.100.0'
+    VERSION = '0.101.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.100.0
+  version: 0.101.0
 platform: ruby
 authors:
 - Jean Boussier