ci-queue 0.80.0 → 0.82.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b8f6b51411e322411c49898ac1a082148ca8b584769d8d3e52ddfd87b11cbb96
-  data.tar.gz: ee4e4beba8f6d2f97400631f7ac7b47ed8561c2af223e369ad9fcdbe88f4c6e3
+  metadata.gz: 3db30c3671ffa851f03a771eaf73a1210b51213b4039cdb17bfa29489a5b21ec
+  data.tar.gz: 5ae004105d90eb2d1383439f71ce355643c67b5e1e35a75782fb6b94815a62b1
 SHA512:
-  metadata.gz: d8f03c40ee904cc069300219ffd71262237de5d291eae1ab682289faae8108f90ec127cbd327b5ea1c0afe9a674e493670e3097922f8848bc3757ebbcb8c38f4
-  data.tar.gz: 8c9904ae453b65bd1be76f5cf3c49098f33a2cc1270699ec7569477465ff3c6369931da7e9a2eae789fc1949b07341148d2e8e73276d0362568bd65801fc6723
+  metadata.gz: 4a1c6ba221d268f37b32d804fde95827159e79dceaf51a156af5abf774e57a04c4dc98d781f3f7c9bcd065006b1b11e70ee2237e3e6109eb9f32c3bef9da2f3b
+  data.tar.gz: a62858950ae3c22ad1f7ae0dba2f85f5daa6de2fa17730efe2faf91b32b1e3be42ee750f65febc48cc53757e417a9f449d177cecf4dcaffb7b03736825a1d7d4

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ci-queue (0.80.0)
+    ci-queue (0.82.0)
       logger
 
 GEM

data/README.md

@@ -71,6 +71,28 @@ rspec-queue --queue redis://example.com --timeout 600 --report
 
 Because of how `ci-queue` executes the examples, `before(:all)` and `after(:all)` hooks are not supported. `rspec-queue` will explicitly reject them.
 
+## Releasing a New Version
+
+After merging changes to `main`, follow these steps to release and propagate the update:
+
+1. **Bump the version** in `ruby/lib/ci/queue/version.rb`:
+
+    ```ruby
+    VERSION = '0.XX.0'
+    ```
+
+2. **Update `Gemfile.lock`** by running `bundle install` in the `ruby/` directory (or manually updating the version string in `Gemfile.lock` if native dependencies prevent `bundle install`).
+
+3. **Commit and merge** the version bump to `main`. ShipIt will automatically publish the gem to RubyGems.
+
+4. **Update dependent apps/zones**: Any application that depends on `ci-queue` (e.g. via its `Gemfile`) needs to pick up the new version by running:
+
+    ```bash
+    bundle update ci-queue
+    ```
+
+    This updates the app's `Gemfile.lock` to reference the new `ci-queue` version. Commit the updated `Gemfile.lock` and deploy.
+
 ## Custom Redis Expiry
 
 `ci-queue` expects the Redis server to have an [eviction policy](https://redis.io/docs/manual/eviction/#eviction-policies) of `allkeys-lru`.

data/lib/ci/queue/build_record.rb

@@ -18,18 +18,35 @@ module CI
         @queue.exhausted?
       end
 
-      def record_error(id, payload, stats: nil)
+      def record_error(id, payload, stat_delta: nil)
         error_reports[id] = payload
-        record_stats(stats)
+        true
       end
 
-      def record_success(id, stats: nil, skip_flaky_record: false, acknowledge: true)
+      def record_success(id, skip_flaky_record: false, acknowledge: true)
         error_reports.delete(id)
-        record_stats(stats)
+        true
+      end
+
+      def record_requeue(id)
+        true
+      end
+
+      def record_stats(builds_stats)
+        return unless builds_stats
+        stats.merge!(builds_stats)
+      end
+
+      def record_stats_delta(delta, pipeline: nil)
+        return if delta.nil? || delta.empty?
+        delta.each do |stat_name, value|
+          next unless value.is_a?(Numeric) || value.to_s.match?(/\A-?\d+\.?\d*\z/)
+          stats[stat_name] = (stats[stat_name] || 0).to_f + value.to_f
+        end
       end
 
       def fetch_stats(stat_names)
-        stat_names.zip(stats.values_at(*stat_names).map(&:to_f))
+        stat_names.zip(stats.values_at(*stat_names).map(&:to_f)).to_h
       end
 
       def reset_stats(stat_names)
@@ -47,11 +64,6 @@ module CI
       private
 
       attr_reader :stats
-
-      def record_stats(builds_stats)
-        return unless builds_stats
-        stats.merge!(builds_stats)
-      end
     end
   end
 end

data/lib/ci/queue/redis/base.rb

@@ -214,7 +214,7 @@ module CI
         end
 
         def key(*args)
-          KeyShortener.key(config.build_id, *args)
+          ['build', build_id, *args].join(':')
         end
 
         def build_id

data/lib/ci/queue/redis/build_record.rb

@@ -56,30 +56,71 @@ module CI
           redis.rpush(key('warnings'), Marshal.dump([type, attributes]))
         end
 
-        def record_error(id, payload, stats: nil)
-          acknowledged, _ = redis.pipelined do |pipeline|
-            @queue.acknowledge(id, error: payload, pipeline: pipeline)
-            record_stats(stats, pipeline: pipeline)
+        def record_error(id, payload, stat_delta: nil)
+          # Run acknowledge first so we know whether we're the first to ack
+          acknowledged = @queue.acknowledge(id, error: payload)
+
+          if acknowledged
+            # We were the first to ack; another worker already ack'd would get falsy from SADD
+            @queue.increment_test_failed
+            # Only the acknowledging worker's stats include this failure (others skip increment when ack=false).
+            # Store so we can subtract it if another worker records success later.
+            store_error_report_delta(id, stat_delta) if stat_delta && stat_delta.any?
           end
-
-          @queue.increment_test_failed if acknowledged == 1
-          nil
+          # Return so caller can roll back local counter when not acknowledged
+          !!acknowledged
         end
 
-        def record_success(id, stats: nil, skip_flaky_record: false)
-          _, error_reports_deleted_count, requeued_count, _ = redis.multi do |transaction|
+        def record_success(id, skip_flaky_record: false)
+          acknowledged, error_reports_deleted_count, requeued_count, delta_json = redis.multi do |transaction|
             @queue.acknowledge(id, pipeline: transaction)
             transaction.hdel(key('error-reports'), id)
             transaction.hget(key('requeues-count'), id)
-            record_stats(stats, pipeline: transaction)
+            transaction.hget(key('error-report-deltas'), id)
+          end
+          # When we're replacing a failure, subtract the (single) acknowledging worker's stat contribution
+          if error_reports_deleted_count.to_i > 0 && delta_json
+            apply_error_report_delta_correction(delta_json)
+            redis.hdel(key('error-report-deltas'), id)
           end
           record_flaky(id) if !skip_flaky_record && (error_reports_deleted_count.to_i > 0 || requeued_count.to_i > 0)
-          nil
+          # Count this run when we ack'd or when we replaced a failure (so stats delta is applied)
+          !!(acknowledged || error_reports_deleted_count.to_i > 0)
         end
 
-        def record_requeue(id, stats: nil)
-          redis.pipelined do |pipeline|
-           record_stats(stats, pipeline: pipeline)
+        def record_requeue(id)
+          true
+        end
+
+        def record_stats(stats = nil, pipeline: nil)
+          return unless stats
+          if pipeline
+            stats.each do |stat_name, stat_value|
+              pipeline.hset(key(stat_name), config.worker_id, stat_value)
+              pipeline.expire(key(stat_name), config.redis_ttl)
+            end
+          else
+            redis.pipelined do |p|
+              record_stats(stats, pipeline: p)
+            end
+          end
+        end
+
+        # Apply a delta to this worker's stats in Redis (HINCRBY). Use this instead of
+        # record_stats when recording per-test so we never overwrite and correction sticks.
+        def record_stats_delta(delta, pipeline: nil)
+          return if delta.nil? || delta.empty?
+          apply_delta = lambda do |p|
+            delta.each do |stat_name, value|
+              next unless value.is_a?(Numeric) || value.to_s.match?(/\A-?\d+\.?\d*\z/)
+              p.hincrbyfloat(key(stat_name), config.worker_id.to_s, value.to_f)
+              p.expire(key(stat_name), config.redis_ttl)
+            end
+          end
+          if pipeline
+            apply_delta.call(pipeline)
+          else
+            redis.pipelined { |p| apply_delta.call(p) }
           end
         end
 
@@ -130,16 +171,30 @@ module CI
 
         attr_reader :config, :redis
 
-        def record_stats(stats, pipeline: redis)
-          return unless stats
-          stats.each do |stat_name, stat_value|
-            pipeline.hset(key(stat_name), config.worker_id, stat_value)
-            pipeline.expire(key(stat_name), config.redis_ttl)
-          end
+        def key(*args)
+          ['build', config.build_id, *args].join(':')
         end
 
-        def key(*args)
-          KeyShortener.key(config.build_id, *args)
+        def store_error_report_delta(test_id, stat_delta)
+          # Only the acknowledging worker's stats include this test; store their delta for correction on success
+          payload = { 'worker_id' => config.worker_id.to_s }.merge(stat_delta)
+          redis.hset(key('error-report-deltas'), test_id, JSON.generate(payload))
+          redis.expire(key('error-report-deltas'), config.redis_ttl)
+        end
+
+        def apply_error_report_delta_correction(delta_json)
+          delta = JSON.parse(delta_json)
+          worker_id = delta.delete('worker_id')&.to_s
+          return if worker_id.nil? || worker_id.empty? || delta.empty?
+
+          redis.pipelined do |pipeline|
+            delta.each do |stat_name, value|
+              next unless value.is_a?(Numeric) || value.to_s.match?(/\A-?\d+\.?\d*\z/)
+
+              pipeline.hincrbyfloat(key(stat_name), worker_id, -value.to_f)
+              pipeline.expire(key(stat_name), config.redis_ttl)
+            end
+          end
         end
       end
     end

data/lib/ci/queue/redis/grind_record.rb

@@ -10,20 +10,32 @@ module CI
           @config = config
         end
 
-        def record_error(payload, stats: nil)
+        def record_error(payload)
           redis.pipelined do |pipeline|
             pipeline.lpush(
               key('error-reports'),
               payload,
             )
             pipeline.expire(key('error-reports'), config.redis_ttl)
-            record_stats(stats, pipeline: pipeline)
           end
           nil
         end
 
-        def record_success(stats: nil)
-          record_stats(stats)
+        def record_success
+        end
+
+        def record_stats(stats, pipeline: nil)
+          return unless stats
+          if pipeline
+            stats.each do |stat_name, stat_value|
+              pipeline.hset(key(stat_name), config.worker_id, stat_value)
+              pipeline.expire(key(stat_name), config.redis_ttl)
+            end
+          else
+            redis.pipelined do |p|
+              record_stats(stats, pipeline: p)
+            end
+          end
         end
 
         def record_warning(_,_)
@@ -52,15 +64,7 @@ module CI
         attr_reader :redis, :config
 
         def key(*args)
-          KeyShortener.key(config.build_id, *args)
-        end
-
-        def record_stats(stats, pipeline: redis)
-          return unless stats
-          stats.each do |stat_name, stat_value|
-            pipeline.hset(key(stat_name), config.worker_id, stat_value)
-            pipeline.expire(key(stat_name), config.redis_ttl)
-          end
+          ['build', config.build_id, *args].join(':')
         end
       end
     end

data/lib/ci/queue/redis.rb

@@ -11,7 +11,6 @@ require 'ci/queue/redis/retry'
 require 'ci/queue/redis/supervisor'
 require 'ci/queue/redis/grind_supervisor'
 require 'ci/queue/redis/test_time_record'
-require 'ci/queue/redis/key_shortener'
 
 module CI
   module Queue

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.80.0'
+    VERSION = '0.82.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue/build_status_recorder.rb

@@ -38,28 +38,46 @@ module Minitest
         super
 
         self.total_time = Minitest.clock_time - start_time
-        if test.requeued?
-          self.requeues += 1
-        elsif test.skipped?
-          self.skips += 1
-        elsif test.error?
-          self.errors += 1
-        elsif test.failure
-          self.failures += 1
-        end
+        
+        # Determine what type of result this is and record it
+        test_id = "#{test.klass}##{test.name}"
+        delta = delta_for(test)
 
-        stats = COUNTERS.zip(COUNTERS.map { |c| send(c) }).to_h
-        if (test.failure || test.error?) && !test.skipped?
-          build.record_error("#{test.klass}##{test.name}", dump(test), stats: stats)
+        acknowledged = if (test.failure || test.error?) && !test.skipped?
+          build.record_error(test_id, dump(test), stat_delta: delta)
         elsif test.requeued?
-          build.record_requeue("#{test.klass}##{test.name}", stats: stats)
+          build.record_requeue(test_id)
         else
-          build.record_success("#{test.klass}##{test.name}", stats: stats, skip_flaky_record: test.skipped?)
+          build.record_success(test_id, skip_flaky_record: test.skipped?)
+        end
+
+        if acknowledged
+          if (test.failure || test.error?) && !test.skipped?
+            test.error? ? self.errors += 1 : self.failures += 1
+          elsif test.requeued?
+            self.requeues += 1
+          elsif test.skipped?
+            self.skips += 1
+          end
+          # Apply delta to Redis (record_success returns true when ack'd or when we replaced a failure)
+          build.record_stats_delta(delta)
         end
       end
 
       private
 
+      def delta_for(test)
+        h = { 'assertions' => (test.assertions || 0).to_i, 'errors' => 0, 'failures' => 0, 'skips' => 0, 'requeues' => 0, 'total_time' => test.time.to_f }
+        if (test.failure || test.error?) && !test.skipped?
+          test.error? ? h['errors'] = 1 : h['failures'] = 1
+        elsif test.requeued?
+          h['requeues'] = 1
+        elsif test.skipped?
+          h['skips'] = 1
+        end
+        h
+      end
+
       def dump(test)
         ErrorReport.new(self.class.failure_formatter.new(test).to_h).dump
       end

data/lib/minitest/queue/failure_formatter.rb

@@ -13,11 +13,9 @@ module Minitest
       end
 
       def to_s
-        [
-          header,
-          body,
-          "\n"
-        ].flatten.compact.join("\n")
+        s = +"#{header}\n#{body}\n\n"
+        s.encode!(Encoding::UTF_8, invalid: :replace, undef: :replace)
+        s
       end
 
       def to_h

data/lib/minitest/queue/grind_recorder.rb

@@ -32,12 +32,12 @@ module Minitest
       private
 
       def record_test(test)
-        stats = self.class.counters
         if (test.failure || test.error?) && !test.skipped?
-          build.record_error(dump(test), stats: stats)
+          build.record_error(dump(test))
         else
-          build.record_success(stats: stats)
+          build.record_success
         end
+        build.record_stats(self.class.counters)
       end
 
       def increment_counter(test)

data/lib/minitest/queue/runner.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 require 'optparse'
 require 'json'
+require 'fileutils'
 require 'minitest/queue'
 require 'ci/queue'
 require 'digest/md5'
@@ -242,16 +243,16 @@ module Minitest
           puts
 
           File.write('log/test_order.log', failing_order.to_a.map(&:id).join("\n"))
-          
+
           bisect_test_details = failing_order.to_a.map do |test|
             source_location = test.source_location
             file_path = source_location&.first || 'unknown'
             line_number = source_location&.last || -1
             "#{test.id} #{file_path}:#{line_number}"
           end
-          
+
           File.write('log/bisect_test_details.log', bisect_test_details.join("\n"))
-          
+
           exit! 0
         end
       end
@@ -336,8 +337,22 @@ module Minitest
         warnings = build.pop_warnings.map do |type, attributes|
           attributes.merge(type: type)
         end.compact
-        File.open(queue_config.warnings_file, 'w') do |f|
-          JSON.dump(warnings, f)
+
+        return if warnings.empty?
+
+        begin
+          # Ensure directory exists
+          dir = File.dirname(queue_config.warnings_file)
+          FileUtils.mkdir_p(dir) unless File.directory?(dir)
+
+          # Write each warning as a separate JSON line (JSONL format)
+          File.open(queue_config.warnings_file, 'a') do |f|
+            warnings.each do |warning|
+              f.puts(JSON.dump(warning))
+            end
+          end
+        rescue => error
+          STDERR.puts "Failed to write warnings: #{error.message}"
         end
       end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.80.0
+  version: 0.82.0
 platform: ruby
 authors:
 - Jean Boussier
@@ -203,7 +203,6 @@ files:
 - lib/ci/queue/redis/grind_record.rb
 - lib/ci/queue/redis/grind_supervisor.rb
 - lib/ci/queue/redis/heartbeat.lua
-- lib/ci/queue/redis/key_shortener.rb
 - lib/ci/queue/redis/monitor.rb
 - lib/ci/queue/redis/release.lua
 - lib/ci/queue/redis/requeue.lua
@@ -257,7 +256,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.7.2
+rubygems_version: 4.0.6
 specification_version: 4
 summary: Distribute tests over many workers using a queue
 test_files: []

data/lib/ci/queue/redis/key_shortener.rb

@@ -1,53 +0,0 @@
-# frozen_string_literal: true
-require 'digest/md5'
-
-module CI
-  module Queue
-    module Redis
-      module KeyShortener
-        # Suffix mapping for common key patterns
-        SUFFIX_ALIASES = {
-          'running' => 'r',
-          'processed' => 'p',
-          'queue' => 'q',
-          'owners' => 'o',
-          'error-reports' => 'e',
-          'requeues-count' => 'rc',
-          'assertions' => 'a',
-          'errors' => 'er',
-          'failures' => 'f',
-          'skips' => 's',
-          'requeues' => 'rq',
-          'total_time' => 't',
-          'test_failed_count' => 'fc',
-          'completed' => 'c',
-          'master-status' => 'm',
-          'created-at' => 'ca',
-          'workers' => 'w',
-          'worker' => 'w',
-          'warnings' => 'wn',
-          'worker-errors' => 'we',
-          'flaky-reports' => 'fl',
-        }.freeze
-
-        # We're transforming the key to a shorter format to minimize network traffic.
-        #
-        # Strategy:
-        # - Shorten prefix: 'b' instead of 'build'
-        # - Hash UUID: 8-char MD5 instead of 36-char UUID
-        # - Alias suffixes: single letters instead of full words
-        #
-        # Example:
-        #   build:unit:019aef0e-c010-433e-b706-c658d3c16372:running (55 bytes)
-        #   -> b:f03d3bef:r (13 bytes, 76% reduction)
-
-        def self.key(build_id, *args)
-          digest = Digest::MD5.hexdigest(build_id)[0..7]
-          shortened_args = args.map { |arg| SUFFIX_ALIASES[arg] || arg }
-
-          ['b', digest, *shortened_args].join(':')
-        end
-      end
-    end
-  end
-end