ci-queue 0.100.0 → 0.102.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 64510328261d26c0b3299aa0e1bb4bbbb2fdd4f784293a304cec028527d25e8b
-  data.tar.gz: 8cf5e99fcf74935b71a2ee6814cfc7565b03b1b314af8380f999f4228400417c
+  metadata.gz: 24f5d0ca5b0d0391a953c6f0014635eec1b6ea541e7ac46a944ac413952777e0
+  data.tar.gz: aeec01bc647d5fb960449d57362cc6961479f5c6da78177c915c35bd9706f861
 SHA512:
-  metadata.gz: c044a9cafe05602f22debf2ea91b5cc63c42dabc67f8689b88430c7c34ece68da7bf7ad03b62036648073c8225e3588382bce588f486359b339a65aa62e71951
-  data.tar.gz: 5d9f475df1260d6cc6cd04d1f1ceec0ca28e7b12e9d5d0b0931ac7b18a2e864a5a2386f102f878a8392722dbb535e260b8661e5ccc5722097ee8d8e866d977dc
+  metadata.gz: 896cb4ac3e81d1235581e11ce6a0693565c6089afc441546a94d7211eab24b538279dfd6eb5abe06d7e3187e1b54bfb0f5d4127baa06d49a12baf181b345a444
+  data.tar.gz: 0be6036f02910f31672c28e2d6c6eb4cdbd41f066ffb499eed8e10990b2a7a8e6d596ecf9c3da5c57c574fc0a76ac73b430d447781c2929f615a2aa669ecbe74

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ci-queue (0.100.0)
+    ci-queue (0.102.0)
       logger
 
 GEM

data/lib/ci/queue/redis/requeue.lua

@@ -12,10 +12,30 @@ local leases_key = KEYS[9]
 local max_requeues = tonumber(ARGV[1])
 local global_max_requeues = tonumber(ARGV[2])
 local entry = ARGV[3]
-local offset = ARGV[4]
+local offset = tonumber(ARGV[4]) or 0
 local ttl = tonumber(ARGV[5])
 local lease_id = ARGV[6]
 
+-- Inserts `entry` behind the `offset` + 1 entries nearest the tail, the end RPOP reserves
+-- from: where `LINSERT BEFORE <entry at index -(offset + 1)>` would put it. LINSERT finds its
+-- pivot by scanning from the head, which is O(queue length) and blocks Redis on large queues;
+-- this only touches the tail, so it is O(offset). Queues of at most `offset` + 1 entries, and
+-- offsets of zero or less, push to the head instead.
+-- Keep in sync with reserve.lua: the Python client does not resolve `-- @include`.
+local function insert_with_offset(queue_key, entry, offset)
+  if offset <= 0 or redis.call('llen', queue_key) <= offset + 1 then
+    redis.call('lpush', queue_key, entry)
+    return
+  end
+
+  local ahead = redis.call('lrange', queue_key, -1 - offset, -1)
+  redis.call('ltrim', queue_key, 0, -2 - offset)
+  redis.call('rpush', queue_key, entry)
+  for _, ahead_entry in ipairs(ahead) do
+    redis.call('rpush', queue_key, ahead_entry)
+  end
+end
+
 -- Only the current lease holder can requeue a test.
 -- If the lease was transferred (e.g. via reserve_lost), reject the stale
 -- worker's requeue so the running entry stays intact for the new holder.
@@ -42,12 +62,7 @@ redis.call('hincrby', requeues_count_key, entry, 1)
 
 redis.call('hdel', error_reports_key, entry)
 
-local pivot = redis.call('lrange', queue_key, -1 - offset, 0 - offset)[1]
-if pivot then
-  redis.call('linsert', queue_key, 'BEFORE', pivot, entry)
-else
-  redis.call('lpush', queue_key, entry)
-end
+insert_with_offset(queue_key, entry, offset)
 
 redis.call('hset', requeued_by_key, entry, worker_queue_key)
 if ttl and ttl > 0 then

data/lib/ci/queue/redis/reserve.lua

@@ -13,12 +13,23 @@ local current_time = ARGV[1]
 local defer_offset = tonumber(ARGV[2]) or 0
 local max_skip_attempts = 4
 
-local function insert_with_offset(test)
-  local pivot = redis.call('lrange', queue_key, -1 - defer_offset, 0 - defer_offset)[1]
-  if pivot then
-    redis.call('linsert', queue_key, 'BEFORE', pivot, test)
-  else
-    redis.call('lpush', queue_key, test)
+-- Inserts `entry` behind the `offset` + 1 entries nearest the tail, the end RPOP reserves
+-- from: where `LINSERT BEFORE <entry at index -(offset + 1)>` would put it. LINSERT finds its
+-- pivot by scanning from the head, which is O(queue length) and blocks Redis on large queues;
+-- this only touches the tail, so it is O(offset). Queues of at most `offset` + 1 entries, and
+-- offsets of zero or less, push to the head instead.
+-- Keep in sync with requeue.lua: the Python client does not resolve `-- @include`.
+local function insert_with_offset(queue_key, entry, offset)
+  if offset <= 0 or redis.call('llen', queue_key) <= offset + 1 then
+    redis.call('lpush', queue_key, entry)
+    return
+  end
+
+  local ahead = redis.call('lrange', queue_key, -1 - offset, -1)
+  redis.call('ltrim', queue_key, 0, -2 - offset)
+  redis.call('rpush', queue_key, entry)
+  for _, ahead_entry in ipairs(ahead) do
+    redis.call('rpush', queue_key, ahead_entry)
   end
 end
 
@@ -45,7 +56,7 @@ for attempt = 1, max_skip_attempts do
       return claim_test(test)
     end
 
-    insert_with_offset(test)
+    insert_with_offset(queue_key, test, defer_offset)
 
     -- If this worker only finds its own requeued tests, defer once by returning nil,
     -- then allow pickup on a subsequent reserve attempt.

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.100.0'
+    VERSION = '0.102.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue/runner.rb

@@ -304,7 +304,11 @@ module Minitest
         step("Waiting for workers to complete")
 
         unless supervisor.wait_for_workers { display_warnings(supervisor.build) }
-          unless supervisor.queue_initialized?
+          # A lazy-load leader only marks the queue as initialized once it has
+          # streamed every test, but workers start on the first batch, so the wait
+          # can end (e.g. on --max-test-failed) while it is still streaming.
+          # Check streaming? first: the status only moves from streaming to ready.
+          unless supervisor.streaming? || supervisor.queue_initialized?
             abort! "No leader was elected. This typically means no worker was able to start. Were there any errors during application boot?", 40
           end
 
@@ -314,7 +318,7 @@ module Minitest
             reporter.write_failure_file(queue_config.failure_file) if queue_config.failure_file
             reporter.write_flaky_tests_file(queue_config.export_flaky_tests_file) if queue_config.export_flaky_tests_file
 
-            abort!("#{supervisor.size} tests weren't run.", exit_code)
+            abort!(unrun_tests_message(supervisor), exit_code)
           end
         end
 
@@ -447,6 +451,16 @@ module Minitest
         Minitest::Queue::WorkerProfileReporter.new(supervisor).print_summary
       end
 
+      def unrun_tests_message(supervisor)
+        # Read the status before the size: tests a streaming leader hasn't pushed
+        # yet aren't in the queue, so the size is only a lower bound.
+        if supervisor.streaming?
+          "At least #{supervisor.size} tests weren't run (the leader hadn't finished streaming tests to the queue)."
+        else
+          "#{supervisor.size} tests weren't run."
+        end
+      end
+
       def parser
         @parser ||= OptionParser.new do |opts|
           opts.banner = "Usage: minitest-queue [options] COMMAND [ARGS]"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.100.0
+  version: 0.102.0
 platform: ruby
 authors:
 - Jean Boussier