ci-queue 0.81.0 → 0.83.0

data/lib/ci/queue/redis/build_record.rb

@@ -56,30 +56,71 @@ module CI
           redis.rpush(key('warnings'), Marshal.dump([type, attributes]))
         end
 
-        def record_error(id, payload, stats: nil)
-          acknowledged, _ = redis.pipelined do |pipeline|
-            @queue.acknowledge(id, error: payload, pipeline: pipeline)
-            record_stats(stats, pipeline: pipeline)
+        def record_error(id, payload, stat_delta: nil)
+          # Run acknowledge first so we know whether we're the first to ack
+          acknowledged = @queue.acknowledge(id, error: payload)
+
+          if acknowledged
+            # We were the first to ack; another worker already ack'd would get falsy from SADD
+            @queue.increment_test_failed
+            # Only the acknowledging worker's stats include this failure (others skip increment when ack=false).
+            # Store so we can subtract it if another worker records success later.
+            store_error_report_delta(id, stat_delta) if stat_delta && stat_delta.any?
           end
-
-          @queue.increment_test_failed if acknowledged == 1
-          nil
+          # Return so caller can roll back local counter when not acknowledged
+          !!acknowledged
         end
 
-        def record_success(id, stats: nil, skip_flaky_record: false)
-          _, error_reports_deleted_count, requeued_count, _ = redis.multi do |transaction|
+        def record_success(id, skip_flaky_record: false)
+          acknowledged, error_reports_deleted_count, requeued_count, delta_json = redis.multi do |transaction|
             @queue.acknowledge(id, pipeline: transaction)
             transaction.hdel(key('error-reports'), id)
             transaction.hget(key('requeues-count'), id)
-            record_stats(stats, pipeline: transaction)
+            transaction.hget(key('error-report-deltas'), id)
+          end
+          # When we're replacing a failure, subtract the (single) acknowledging worker's stat contribution
+          if error_reports_deleted_count.to_i > 0 && delta_json
+            apply_error_report_delta_correction(delta_json)
+            redis.hdel(key('error-report-deltas'), id)
           end
           record_flaky(id) if !skip_flaky_record && (error_reports_deleted_count.to_i > 0 || requeued_count.to_i > 0)
-          nil
+          # Count this run when we ack'd or when we replaced a failure (so stats delta is applied)
+          !!(acknowledged || error_reports_deleted_count.to_i > 0)
         end
 
-        def record_requeue(id, stats: nil)
-          redis.pipelined do |pipeline|
-           record_stats(stats, pipeline: pipeline)
+        def record_requeue(id)
+          true
+        end
+
+        def record_stats(stats = nil, pipeline: nil)
+          return unless stats
+          if pipeline
+            stats.each do |stat_name, stat_value|
+              pipeline.hset(key(stat_name), config.worker_id, stat_value)
+              pipeline.expire(key(stat_name), config.redis_ttl)
+            end
+          else
+            redis.pipelined do |p|
+              record_stats(stats, pipeline: p)
+            end
+          end
+        end
+
+        # Apply a delta to this worker's stats in Redis (HINCRBY). Use this instead of
+        # record_stats when recording per-test so we never overwrite and correction sticks.
+        def record_stats_delta(delta, pipeline: nil)
+          return if delta.nil? || delta.empty?
+          apply_delta = lambda do |p|
+            delta.each do |stat_name, value|
+              next unless value.is_a?(Numeric) || value.to_s.match?(/\A-?\d+\.?\d*\z/)
+              p.hincrbyfloat(key(stat_name), config.worker_id.to_s, value.to_f)
+              p.expire(key(stat_name), config.redis_ttl)
+            end
+          end
+          if pipeline
+            apply_delta.call(pipeline)
+          else
+            redis.pipelined { |p| apply_delta.call(p) }
           end
         end
 
@@ -108,6 +149,18 @@ module CI
           redis.smembers(key('flaky-reports'))
         end
 
+        def record_worker_profile(profile)
+          redis.pipelined do |pipeline|
+            pipeline.hset(key('worker-profiles'), config.worker_id, JSON.dump(profile))
+            pipeline.expire(key('worker-profiles'), config.redis_ttl)
+          end
+        end
+
+        def worker_profiles
+          raw = redis.hgetall(key('worker-profiles'))
+          raw.transform_values { |v| JSON.parse(v) }
+        end
+
         def fetch_stats(stat_names)
           counts = redis.pipelined do |pipeline|
             stat_names.each { |c| pipeline.hvals(key(c)) }
@@ -130,16 +183,30 @@ module CI
 
         attr_reader :config, :redis
 
-        def record_stats(stats, pipeline: redis)
-          return unless stats
-          stats.each do |stat_name, stat_value|
-            pipeline.hset(key(stat_name), config.worker_id, stat_value)
-            pipeline.expire(key(stat_name), config.redis_ttl)
-          end
+        def key(*args)
+          ['build', config.build_id, *args].join(':')
         end
 
-        def key(*args)
-          KeyShortener.key(config.build_id, *args)
+        def store_error_report_delta(test_id, stat_delta)
+          # Only the acknowledging worker's stats include this test; store their delta for correction on success
+          payload = { 'worker_id' => config.worker_id.to_s }.merge(stat_delta)
+          redis.hset(key('error-report-deltas'), test_id, JSON.generate(payload))
+          redis.expire(key('error-report-deltas'), config.redis_ttl)
+        end
+
+        def apply_error_report_delta_correction(delta_json)
+          delta = JSON.parse(delta_json)
+          worker_id = delta.delete('worker_id')&.to_s
+          return if worker_id.nil? || worker_id.empty? || delta.empty?
+
+          redis.pipelined do |pipeline|
+            delta.each do |stat_name, value|
+              next unless value.is_a?(Numeric) || value.to_s.match?(/\A-?\d+\.?\d*\z/)
+
+              pipeline.hincrbyfloat(key(stat_name), worker_id, -value.to_f)
+              pipeline.expire(key(stat_name), config.redis_ttl)
+            end
+          end
         end
       end
     end

data/lib/ci/queue/redis/grind_record.rb

@@ -10,20 +10,32 @@ module CI
           @config = config
         end
 
-        def record_error(payload, stats: nil)
+        def record_error(payload)
           redis.pipelined do |pipeline|
             pipeline.lpush(
               key('error-reports'),
               payload,
             )
             pipeline.expire(key('error-reports'), config.redis_ttl)
-            record_stats(stats, pipeline: pipeline)
           end
           nil
         end
 
-        def record_success(stats: nil)
-          record_stats(stats)
+        def record_success
+        end
+
+        def record_stats(stats, pipeline: nil)
+          return unless stats
+          if pipeline
+            stats.each do |stat_name, stat_value|
+              pipeline.hset(key(stat_name), config.worker_id, stat_value)
+              pipeline.expire(key(stat_name), config.redis_ttl)
+            end
+          else
+            redis.pipelined do |p|
+              record_stats(stats, pipeline: p)
+            end
+          end
         end
 
         def record_warning(_,_)
@@ -52,15 +64,7 @@ module CI
         attr_reader :redis, :config
 
         def key(*args)
-          KeyShortener.key(config.build_id, *args)
-        end
-
-        def record_stats(stats, pipeline: redis)
-          return unless stats
-          stats.each do |stat_name, stat_value|
-            pipeline.hset(key(stat_name), config.worker_id, stat_value)
-            pipeline.expire(key(stat_name), config.redis_ttl)
-          end
+          ['build', config.build_id, *args].join(':')
         end
       end
     end

data/lib/ci/queue/redis/heartbeat.lua

@@ -1,18 +1,23 @@
 -- AUTOGENERATED FILE DO NOT EDIT DIRECTLY
+-- @include _entry_helpers
+
 local zset_key = KEYS[1]
 local processed_key = KEYS[2]
 local owners_key = KEYS[3]
 local worker_queue_key = KEYS[4]
 
 local current_time = ARGV[1]
-local test = ARGV[2]
+local entry = ARGV[2]
+local entry_delimiter = ARGV[3]
+
+local test_id = test_id_from_entry(entry, entry_delimiter)
 
 -- already processed, we do not need to bump the timestamp
-if redis.call('sismember', processed_key, test) == 1 then
+if redis.call('sismember', processed_key, test_id) == 1 then
   return false
 end
 
 -- we're still the owner of the test, we can bump the timestamp
-if redis.call('hget', owners_key, test) == worker_queue_key then
-  return redis.call('zadd', zset_key, current_time, test)
+if redis.call('hget', owners_key, entry) == worker_queue_key then
+  return redis.call('zadd', zset_key, current_time, entry)
 end

data/lib/ci/queue/redis/monitor.rb

@@ -13,11 +13,12 @@ module CI
         DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../../redis', __FILE__)
         RELEASE_SCRIPTS_ROOT = ::File.expand_path('../../redis', __FILE__)
 
-        def initialize(pipe, logger, redis_url, zset_key, processed_key, owners_key, worker_queue_key)
+        def initialize(pipe, logger, redis_url, zset_key, processed_key, owners_key, worker_queue_key, entry_delimiter)
           @zset_key = zset_key
           @processed_key = processed_key
           @owners_key = owners_key
           @worker_queue_key = worker_queue_key
+          @entry_delimiter = entry_delimiter
           @logger = logger
           @redis = ::Redis.new(url: redis_url, reconnect_attempts: [0, 0, 0.1, 0.5, 1, 3, 5])
           @shutdown = false
@@ -40,7 +41,7 @@ module CI
           eval_script(
             :heartbeat,
             keys: [@zset_key, @processed_key, @owners_key, @worker_queue_key],
-            argv: [Time.now.to_f, id]
+            argv: [Time.now.to_f, id, @entry_delimiter]
           )
         rescue => error
           @logger.info(error)
@@ -56,9 +57,21 @@ module CI
         end
 
         def read_script(name)
-          ::File.read(::File.join(DEV_SCRIPTS_ROOT, "#{name}.lua"))
+          resolve_lua_includes(
+            ::File.read(::File.join(DEV_SCRIPTS_ROOT, "#{name}.lua")),
+            DEV_SCRIPTS_ROOT,
+          )
         rescue SystemCallError
-          ::File.read(::File.join(RELEASE_SCRIPTS_ROOT, "#{name}.lua"))
+          resolve_lua_includes(
+            ::File.read(::File.join(RELEASE_SCRIPTS_ROOT, "#{name}.lua")),
+            RELEASE_SCRIPTS_ROOT,
+          )
+        end
+
+        def resolve_lua_includes(script, root)
+          script.gsub(/^-- @include (\S+)$/) do
+            ::File.read(::File.join(root, "#{$1}.lua"))
+          end
         end
 
         HEADER = 'L'
@@ -142,9 +155,10 @@ zset_key = ARGV[1]
 processed_key = ARGV[2]
 owners_key = ARGV[3]
 worker_queue_key = ARGV[4]
+entry_delimiter = ARGV[5]
 
 logger.debug("Starting monitor: #{redis_url} #{zset_key} #{processed_key}")
-manager = CI::Queue::Redis::Monitor.new($stdin, logger, redis_url, zset_key, processed_key, owners_key, worker_queue_key)
+manager = CI::Queue::Redis::Monitor.new($stdin, logger, redis_url, zset_key, processed_key, owners_key, worker_queue_key, entry_delimiter)
 
 # Notify the parent we're ready
 $stdout.puts(".")

data/lib/ci/queue/redis/requeue.lua

@@ -6,17 +6,20 @@ local zset_key = KEYS[4]
 local worker_queue_key = KEYS[5]
 local owners_key = KEYS[6]
 local error_reports_key = KEYS[7]
+local requeued_by_key = KEYS[8]
 
 local max_requeues = tonumber(ARGV[1])
 local global_max_requeues = tonumber(ARGV[2])
-local test = ARGV[3]
-local offset = ARGV[4]
+local entry = ARGV[3]
+local test_id = ARGV[4]
+local offset = ARGV[5]
+local ttl = tonumber(ARGV[6])
 
-if redis.call('hget', owners_key, test) == worker_queue_key then
-   redis.call('hdel', owners_key, test)
+if redis.call('hget', owners_key, entry) == worker_queue_key then
+   redis.call('hdel', owners_key, entry)
 end
 
-if redis.call('sismember', processed_key, test) == 1 then
+if redis.call('sismember', processed_key, test_id) == 1 then
   return false
 end
 
@@ -25,23 +28,28 @@ if global_requeues and global_requeues >= tonumber(global_max_requeues) then
   return false
 end
 
-local requeues = tonumber(redis.call('hget', requeues_count_key, test))
+local requeues = tonumber(redis.call('hget', requeues_count_key, test_id))
 if requeues and requeues >= max_requeues then
   return false
 end
 
 redis.call('hincrby', requeues_count_key, '___total___', 1)
-redis.call('hincrby', requeues_count_key, test, 1)
+redis.call('hincrby', requeues_count_key, test_id, 1)
 
-redis.call('hdel', error_reports_key, test)
+redis.call('hdel', error_reports_key, test_id)
 
 local pivot = redis.call('lrange', queue_key, -1 - offset, 0 - offset)[1]
 if pivot then
-  redis.call('linsert', queue_key, 'BEFORE', pivot, test)
+  redis.call('linsert', queue_key, 'BEFORE', pivot, entry)
 else
-  redis.call('lpush', queue_key, test)
+  redis.call('lpush', queue_key, entry)
 end
 
-redis.call('zrem', zset_key, test)
+redis.call('hset', requeued_by_key, entry, worker_queue_key)
+if ttl and ttl > 0 then
+  redis.call('expire', requeued_by_key, ttl)
+end
+
+redis.call('zrem', zset_key, entry)
 
 return true

data/lib/ci/queue/redis/reserve.lua

@@ -4,15 +4,54 @@ local zset_key = KEYS[2]
 local processed_key = KEYS[3]
 local worker_queue_key = KEYS[4]
 local owners_key = KEYS[5]
+local requeued_by_key = KEYS[6]
+local workers_key = KEYS[7]
 
 local current_time = ARGV[1]
+local defer_offset = tonumber(ARGV[2]) or 0
+local max_skip_attempts = 4
 
-local test = redis.call('rpop', queue_key)
-if test then
-  redis.call('zadd', zset_key, current_time, test)
-  redis.call('lpush', worker_queue_key, test)
-  redis.call('hset', owners_key, test, worker_queue_key)
-  return test
-else
-  return nil
+local function insert_with_offset(test)
+  local pivot = redis.call('lrange', queue_key, -1 - defer_offset, 0 - defer_offset)[1]
+  if pivot then
+    redis.call('linsert', queue_key, 'BEFORE', pivot, test)
+  else
+    redis.call('lpush', queue_key, test)
+  end
 end
+
+for attempt = 1, max_skip_attempts do
+  local test = redis.call('rpop', queue_key)
+  if not test then
+    return nil
+  end
+
+  local requeued_by = redis.call('hget', requeued_by_key, test)
+  if requeued_by == worker_queue_key then
+    -- If this build only has one worker, allow immediate self-pickup.
+    if redis.call('scard', workers_key) <= 1 then
+      redis.call('hdel', requeued_by_key, test)
+      redis.call('zadd', zset_key, current_time, test)
+      redis.call('lpush', worker_queue_key, test)
+      redis.call('hset', owners_key, test, worker_queue_key)
+      return test
+    end
+
+    insert_with_offset(test)
+
+    -- If this worker only finds its own requeued tests, defer once by returning nil,
+    -- then allow pickup on a subsequent reserve attempt.
+    if attempt == max_skip_attempts then
+      redis.call('hdel', requeued_by_key, test)
+      return nil
+    end
+  else
+    redis.call('hdel', requeued_by_key, test)
+    redis.call('zadd', zset_key, current_time, test)
+    redis.call('lpush', worker_queue_key, test)
+    redis.call('hset', owners_key, test, worker_queue_key)
+    return test
+  end
+end
+
+return nil

data/lib/ci/queue/redis/reserve_lost.lua

@@ -1,4 +1,6 @@
 -- AUTOGENERATED FILE DO NOT EDIT DIRECTLY
+-- @include _entry_helpers
+
 local zset_key = KEYS[1]
 local processed_key = KEYS[2]
 local worker_queue_key = KEYS[3]
@@ -6,10 +8,12 @@ local owners_key = KEYS[4]
 
 local current_time = ARGV[1]
 local timeout = ARGV[2]
+local entry_delimiter = ARGV[3]
 
 local lost_tests = redis.call('zrangebyscore', zset_key, 0, current_time - timeout)
 for _, test in ipairs(lost_tests) do
-  if redis.call('sismember', processed_key, test) == 0 then
+  local test_id = test_id_from_entry(test, entry_delimiter)
+  if redis.call('sismember', processed_key, test_id) == 0 then
     redis.call('zadd', zset_key, current_time, test)
     redis.call('lpush', worker_queue_key, test)
     redis.call('hset', owners_key, test, worker_queue_key) -- Take ownership

data/lib/ci/queue/redis/supervisor.rb

@@ -9,7 +9,7 @@ module CI
         end
 
         def total
-          wait_for_master(timeout: config.queue_init_timeout)
+          wait_for_master(timeout: config.queue_init_timeout, allow_streaming: true)
           redis.get(key('total')).to_i
         end
 
@@ -19,7 +19,7 @@ module CI
 
         def wait_for_workers
           duration = measure do
-            wait_for_master(timeout: config.queue_init_timeout)
+            wait_for_master(timeout: config.queue_init_timeout, allow_streaming: true)
           end
 
           yield if block_given?
@@ -30,7 +30,7 @@ module CI
             @time_left -= 1
             sleep 1
 
-            if active_workers?
+            if active_workers? || streaming?
               @time_left_with_no_workers = config.inactive_workers_timeout
             else
               @time_left_with_no_workers -= 1