ci-queue 0.82.0 → 0.84.0

data/lib/ci/queue/redis/base.rb

@@ -144,19 +144,26 @@ module CI
         end
 
         def to_a
-          test_ids.reverse.map { |k| index.fetch(k) }
+          test_ids.reverse.map do |entry|
+            index.fetch(entry) do
+              test_id = CI::Queue::QueueEntry.test_id(entry)
+              index.fetch(test_id)
+            end
+          end
         end
 
         def progress
-          total - size
+          progress = total - size
+          progress < 0 ? 0 : progress
         end
 
-        def wait_for_master(timeout: 30)
+        def wait_for_master(timeout: 30, allow_streaming: false)
           return true if master?
           return true if queue_initialized?
+          return true if allow_streaming && streaming?
 
           (timeout * 10 + 1).to_i.times do
-            if queue_initialized?
+            if queue_initialized? || (allow_streaming && streaming?)
               return true
             else
               sleep 0.1
@@ -177,6 +184,10 @@ module CI
           end
         end
 
+        def streaming?
+          master_status == 'streaming'
+        end
+
         def queue_initializing?
           master_status == 'setup'
         end
@@ -235,9 +246,21 @@ module CI
         end
 
         def read_script(name)
-          ::File.read(::File.join(CI::Queue::DEV_SCRIPTS_ROOT, "#{name}.lua"))
+          resolve_lua_includes(
+            ::File.read(::File.join(CI::Queue::DEV_SCRIPTS_ROOT, "#{name}.lua")),
+            CI::Queue::DEV_SCRIPTS_ROOT,
+          )
         rescue SystemCallError
-          ::File.read(::File.join(CI::Queue::RELEASE_SCRIPTS_ROOT, "#{name}.lua"))
+          resolve_lua_includes(
+            ::File.read(::File.join(CI::Queue::RELEASE_SCRIPTS_ROOT, "#{name}.lua")),
+            CI::Queue::RELEASE_SCRIPTS_ROOT,
+          )
+        end
+
+        def resolve_lua_includes(script, root)
+          script.gsub(/^-- @include (\S+)$/) do
+            ::File.read(::File.join(root, "#{$1}.lua"))
+          end
         end
 
         class HeartbeatProcess

data/lib/ci/queue/redis/build_record.rb

@@ -33,14 +33,14 @@ module CI
         end
 
         def failed_tests
-          redis.hkeys(key('error-reports'))
+          redis.hkeys(key('error-reports')).map { |entry| CI::Queue::QueueEntry.test_id(entry) }
         end
 
         TOTAL_KEY = "___total___"
         def requeued_tests
           requeues = redis.hgetall(key('requeues-count'))
           requeues.delete(TOTAL_KEY)
-          requeues
+          requeues.transform_keys { |entry| CI::Queue::QueueEntry.test_id(entry) }
         end
 
         def pop_warnings
@@ -56,39 +56,39 @@ module CI
           redis.rpush(key('warnings'), Marshal.dump([type, attributes]))
         end
 
-        def record_error(id, payload, stat_delta: nil)
+        def record_error(entry, payload, stat_delta: nil)
           # Run acknowledge first so we know whether we're the first to ack
-          acknowledged = @queue.acknowledge(id, error: payload)
+          acknowledged = @queue.acknowledge(entry, error: payload)
 
           if acknowledged
             # We were the first to ack; another worker already ack'd would get falsy from SADD
             @queue.increment_test_failed
             # Only the acknowledging worker's stats include this failure (others skip increment when ack=false).
             # Store so we can subtract it if another worker records success later.
-            store_error_report_delta(id, stat_delta) if stat_delta && stat_delta.any?
+            store_error_report_delta(entry, stat_delta) if stat_delta && stat_delta.any?
           end
           # Return so caller can roll back local counter when not acknowledged
           !!acknowledged
         end
 
-        def record_success(id, skip_flaky_record: false)
+        def record_success(entry, skip_flaky_record: false)
           acknowledged, error_reports_deleted_count, requeued_count, delta_json = redis.multi do |transaction|
-            @queue.acknowledge(id, pipeline: transaction)
-            transaction.hdel(key('error-reports'), id)
-            transaction.hget(key('requeues-count'), id)
-            transaction.hget(key('error-report-deltas'), id)
+            @queue.acknowledge(entry, pipeline: transaction)
+            transaction.hdel(key('error-reports'), entry)
+            transaction.hget(key('requeues-count'), entry)
+            transaction.hget(key('error-report-deltas'), entry)
           end
           # When we're replacing a failure, subtract the (single) acknowledging worker's stat contribution
           if error_reports_deleted_count.to_i > 0 && delta_json
             apply_error_report_delta_correction(delta_json)
-            redis.hdel(key('error-report-deltas'), id)
+            redis.hdel(key('error-report-deltas'), entry)
           end
-          record_flaky(id) if !skip_flaky_record && (error_reports_deleted_count.to_i > 0 || requeued_count.to_i > 0)
+          record_flaky(entry) if !skip_flaky_record && (error_reports_deleted_count.to_i > 0 || requeued_count.to_i > 0)
           # Count this run when we ack'd or when we replaced a failure (so stats delta is applied)
           !!(acknowledged || error_reports_deleted_count.to_i > 0)
         end
 
-        def record_requeue(id)
+        def record_requeue(entry)
           true
         end
 
@@ -142,11 +142,23 @@ module CI
         end
 
         def error_reports
-          redis.hgetall(key('error-reports'))
+          redis.hgetall(key('error-reports')).transform_keys { |entry| CI::Queue::QueueEntry.test_id(entry) }
         end
 
         def flaky_reports
-          redis.smembers(key('flaky-reports'))
+          redis.smembers(key('flaky-reports')).map { |entry| CI::Queue::QueueEntry.test_id(entry) }
+        end
+
+        def record_worker_profile(profile)
+          redis.pipelined do |pipeline|
+            pipeline.hset(key('worker-profiles'), config.worker_id, JSON.dump(profile))
+            pipeline.expire(key('worker-profiles'), config.redis_ttl)
+          end
+        end
+
+        def worker_profiles
+          raw = redis.hgetall(key('worker-profiles'))
+          raw.transform_values { |v| JSON.parse(v) }
         end
 
         def fetch_stats(stat_names)
@@ -175,10 +187,10 @@ module CI
           ['build', config.build_id, *args].join(':')
         end
 
-        def store_error_report_delta(test_id, stat_delta)
+        def store_error_report_delta(entry, stat_delta)
           # Only the acknowledging worker's stats include this test; store their delta for correction on success
           payload = { 'worker_id' => config.worker_id.to_s }.merge(stat_delta)
-          redis.hset(key('error-report-deltas'), test_id, JSON.generate(payload))
+          redis.hset(key('error-report-deltas'), entry, JSON.generate(payload))
           redis.expire(key('error-report-deltas'), config.redis_ttl)
         end
 

data/lib/ci/queue/redis/heartbeat.lua

@@ -5,14 +5,14 @@ local owners_key = KEYS[3]
 local worker_queue_key = KEYS[4]
 
 local current_time = ARGV[1]
-local test = ARGV[2]
+local entry = ARGV[2]
 
 -- already processed, we do not need to bump the timestamp
-if redis.call('sismember', processed_key, test) == 1 then
+if redis.call('sismember', processed_key, entry) == 1 then
   return false
 end
 
 -- we're still the owner of the test, we can bump the timestamp
-if redis.call('hget', owners_key, test) == worker_queue_key then
-  return redis.call('zadd', zset_key, current_time, test)
+if redis.call('hget', owners_key, entry) == worker_queue_key then
+  return redis.call('zadd', zset_key, current_time, entry)
 end

data/lib/ci/queue/redis/monitor.rb

@@ -56,9 +56,21 @@ module CI
         end
 
         def read_script(name)
-          ::File.read(::File.join(DEV_SCRIPTS_ROOT, "#{name}.lua"))
+          resolve_lua_includes(
+            ::File.read(::File.join(DEV_SCRIPTS_ROOT, "#{name}.lua")),
+            DEV_SCRIPTS_ROOT,
+          )
         rescue SystemCallError
-          ::File.read(::File.join(RELEASE_SCRIPTS_ROOT, "#{name}.lua"))
+          resolve_lua_includes(
+            ::File.read(::File.join(RELEASE_SCRIPTS_ROOT, "#{name}.lua")),
+            RELEASE_SCRIPTS_ROOT,
+          )
+        end
+
+        def resolve_lua_includes(script, root)
+          script.gsub(/^-- @include (\S+)$/) do
+            ::File.read(::File.join(root, "#{$1}.lua"))
+          end
         end
 
         HEADER = 'L'

data/lib/ci/queue/redis/requeue.lua

@@ -6,17 +6,19 @@ local zset_key = KEYS[4]
 local worker_queue_key = KEYS[5]
 local owners_key = KEYS[6]
 local error_reports_key = KEYS[7]
+local requeued_by_key = KEYS[8]
 
 local max_requeues = tonumber(ARGV[1])
 local global_max_requeues = tonumber(ARGV[2])
-local test = ARGV[3]
+local entry = ARGV[3]
 local offset = ARGV[4]
+local ttl = tonumber(ARGV[5])
 
-if redis.call('hget', owners_key, test) == worker_queue_key then
-   redis.call('hdel', owners_key, test)
+if redis.call('hget', owners_key, entry) == worker_queue_key then
+   redis.call('hdel', owners_key, entry)
 end
 
-if redis.call('sismember', processed_key, test) == 1 then
+if redis.call('sismember', processed_key, entry) == 1 then
   return false
 end
 
@@ -25,23 +27,28 @@ if global_requeues and global_requeues >= tonumber(global_max_requeues) then
   return false
 end
 
-local requeues = tonumber(redis.call('hget', requeues_count_key, test))
+local requeues = tonumber(redis.call('hget', requeues_count_key, entry))
 if requeues and requeues >= max_requeues then
   return false
 end
 
 redis.call('hincrby', requeues_count_key, '___total___', 1)
-redis.call('hincrby', requeues_count_key, test, 1)
+redis.call('hincrby', requeues_count_key, entry, 1)
 
-redis.call('hdel', error_reports_key, test)
+redis.call('hdel', error_reports_key, entry)
 
 local pivot = redis.call('lrange', queue_key, -1 - offset, 0 - offset)[1]
 if pivot then
-  redis.call('linsert', queue_key, 'BEFORE', pivot, test)
+  redis.call('linsert', queue_key, 'BEFORE', pivot, entry)
 else
-  redis.call('lpush', queue_key, test)
+  redis.call('lpush', queue_key, entry)
 end
 
-redis.call('zrem', zset_key, test)
+redis.call('hset', requeued_by_key, entry, worker_queue_key)
+if ttl and ttl > 0 then
+  redis.call('expire', requeued_by_key, ttl)
+end
+
+redis.call('zrem', zset_key, entry)
 
 return true

data/lib/ci/queue/redis/reserve.lua

@@ -4,15 +4,54 @@ local zset_key = KEYS[2]
 local processed_key = KEYS[3]
 local worker_queue_key = KEYS[4]
 local owners_key = KEYS[5]
+local requeued_by_key = KEYS[6]
+local workers_key = KEYS[7]
 
 local current_time = ARGV[1]
+local defer_offset = tonumber(ARGV[2]) or 0
+local max_skip_attempts = 4
 
-local test = redis.call('rpop', queue_key)
-if test then
-  redis.call('zadd', zset_key, current_time, test)
-  redis.call('lpush', worker_queue_key, test)
-  redis.call('hset', owners_key, test, worker_queue_key)
-  return test
-else
-  return nil
+local function insert_with_offset(test)
+  local pivot = redis.call('lrange', queue_key, -1 - defer_offset, 0 - defer_offset)[1]
+  if pivot then
+    redis.call('linsert', queue_key, 'BEFORE', pivot, test)
+  else
+    redis.call('lpush', queue_key, test)
+  end
 end
+
+for attempt = 1, max_skip_attempts do
+  local test = redis.call('rpop', queue_key)
+  if not test then
+    return nil
+  end
+
+  local requeued_by = redis.call('hget', requeued_by_key, test)
+  if requeued_by == worker_queue_key then
+    -- If this build only has one worker, allow immediate self-pickup.
+    if redis.call('scard', workers_key) <= 1 then
+      redis.call('hdel', requeued_by_key, test)
+      redis.call('zadd', zset_key, current_time, test)
+      redis.call('lpush', worker_queue_key, test)
+      redis.call('hset', owners_key, test, worker_queue_key)
+      return test
+    end
+
+    insert_with_offset(test)
+
+    -- If this worker only finds its own requeued tests, defer once by returning nil,
+    -- then allow pickup on a subsequent reserve attempt.
+    if attempt == max_skip_attempts then
+      redis.call('hdel', requeued_by_key, test)
+      return nil
+    end
+  else
+    redis.call('hdel', requeued_by_key, test)
+    redis.call('zadd', zset_key, current_time, test)
+    redis.call('lpush', worker_queue_key, test)
+    redis.call('hset', owners_key, test, worker_queue_key)
+    return test
+  end
+end
+
+return nil

data/lib/ci/queue/redis/supervisor.rb

@@ -9,7 +9,7 @@ module CI
         end
 
         def total
-          wait_for_master(timeout: config.queue_init_timeout)
+          wait_for_master(timeout: config.queue_init_timeout, allow_streaming: true)
           redis.get(key('total')).to_i
         end
 
@@ -19,7 +19,7 @@ module CI
 
         def wait_for_workers
           duration = measure do
-            wait_for_master(timeout: config.queue_init_timeout)
+            wait_for_master(timeout: config.queue_init_timeout, allow_streaming: true)
           end
 
           yield if block_given?
@@ -30,7 +30,7 @@ module CI
             @time_left -= 1
             sleep 1
 
-            if active_workers?
+            if active_workers? || streaming?
               @time_left_with_no_workers = config.inactive_workers_timeout
             else
               @time_left_with_no_workers -= 1