ci-queue 0.81.0 → 0.83.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6e62b7e80a81a6ec61a80810479ba63d150b23fabbd89eb29ee0cd9328158fc9
-  data.tar.gz: ba9f408cb20679dc11476aa6b0207c8ad5cd1922778ee075182d4390f7b90ec8
+  metadata.gz: '08152aacde705d2472151b351e690db68a0c067c8930bf9574e1835825ee225b'
+  data.tar.gz: 37fc5a7e9174d9188ff39568a0e4a5bf4f1ae0e3bce0f08e4d9d66e2a5a5264b
 SHA512:
-  metadata.gz: 426464c8b7ad0a5c5d2652484a24510c522addd20dbbaf7651fe0cb5624226809a7c8013ee3dacf64e1c0580616d1428c6ef070306fa008a4dbf89eb825c59b0
-  data.tar.gz: '08926dd910a43f73e118880f8929e34844a81502f4cb28f3e9c144f5e584de6db2c9eacc6a5e463f50f0a9fd5ec76ca0b2835880e4dd43339325ac580365a390'
+  metadata.gz: 0ca31505b3ca58115b632d1a23e7884e2bbc25ae2360f4228b08192705200303ddecd03220ff86ebb6c6c82b7e580f83f7d86e5f79d10d59cd6949c4b6947d42
+  data.tar.gz: f47440362656cc4ddf8ad52a9c49c9289b70f0a64312eb2c0c3f82e6bc9a980e6414a2c03ccb6613508975bf7b64e6b6232d1083fbbfc4c159b6eac0cb522e51

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ci-queue (0.81.0)
+    ci-queue (0.83.0)
       logger
 
 GEM

data/README.md

@@ -38,6 +38,93 @@ minitest-queue --queue redis://example.com run -Itest test/**/*_test.rb
 
 Additionally you can configure the requeue settings (see main README) with `--max-requeues` and `--requeue-tolerance`.
 
+#### Lazy loading (opt-in)
+
+By default, all test files are loaded upfront before any tests run. Lazy loading changes this
+so that test files are loaded on-demand as each test is dequeued, reducing peak memory usage.
+This is supported only by `minitest-queue` (not `rspec-queue`).
+
+```bash
+minitest-queue --queue redis://example.com --lazy-load run -Itest test/**/*_test.rb
+```
+
+The leader discovers tests from the provided files, streams them to Redis in batches, and
+workers start running tests as soon as the first batch arrives. Each worker only loads the
+test files it actually needs.
+
+In lazy-load mode, test files are not loaded at startup. If your test suite requires a boot
+file (e.g., `test/test_helper.rb` for Rails), specify it so all workers load it before
+running tests.
+
+**CLI flags:**
+
+| Flag | Description |
+|---|---|
+| `--lazy-load` | Enable lazy loading mode |
+| `--lazy-load-stream-batch-size SIZE` | Number of tests per batch streamed to Redis (default: 5000) |
+| `--lazy-load-stream-timeout SECONDS` | Max time for the leader to finish streaming (default: 300s or `--queue-init-timeout`, whichever is larger) |
+| `--test-files FILE` | Read test file paths from FILE (one per line) instead of positional args. Avoids ARG_MAX limits for large suites (36K+ files). |
+
+**Environment variables:**
+
+| Variable | Description |
+|---|---|
+| `CI_QUEUE_LAZY_LOAD=1` | Enable lazy loading (equivalent to `--lazy-load`) |
+| `CI_QUEUE_LAZY_LOAD_STREAM_BATCH_SIZE=N` | Same as `--lazy-load-stream-batch-size` |
+| `CI_QUEUE_LAZY_LOAD_STREAM_TIMEOUT=N` | Same as `--lazy-load-stream-timeout` |
+| `CI_QUEUE_LAZY_LOAD_TEST_HELPERS=path` | Comma-separated list of helper files to load at startup on all workers (e.g., `test/test_helper.rb`). No CLI equivalent. |
+
+Backward-compatible env var aliases: `CI_QUEUE_STREAM_BATCH_SIZE`, `CI_QUEUE_STREAM_TIMEOUT`, `CI_QUEUE_TEST_HELPERS`.
+
+When `CI_QUEUE_DEBUG=1` is set, file loading stats are printed at the end of the run.
+
+#### Preresolved test names (opt-in)
+
+For large test suites, you can pre-compute the full list of test names on a stable branch
+(e.g., `main`) and cache it. On feature branches, ci-queue reads test names from the cache
+instead of loading all test files to discover them. This eliminates the upfront discovery
+cost and implies lazy-load mode for all workers.
+
+```bash
+minitest-queue --queue redis://example.com run \
+  --preresolved-tests test_names.txt \
+  -I. -Itest
+```
+
+The file format is one test per line: `TestClass#method_name|path/to/test_file.rb`.
+The pipe-delimited file path tells ci-queue which file to load when a worker picks up that test.
+The leader streams entries directly to Redis without loading any test files.
+
+**Reconciliation**: The cached test list may become stale when test files change between
+the cache build and the branch build (methods added, removed, or renamed). To handle this,
+pass `--test-files` with a list of changed test files. The leader will discard preresolved
+entries for those files and re-discover their current test methods by loading them:
+
+```bash
+minitest-queue --queue redis://example.com run \
+  --preresolved-tests cached_test_names.txt \
+  --test-files changed_test_files.txt \
+  -I. -Itest
+```
+
+Note: `--test-files` serves double duty. In plain lazy-load mode it provides the list of
+test files to discover. In preresolved mode it acts as the reconciliation set.
+
+**Stale entry handling**: Even with reconciliation, some preresolved entries may refer to
+test methods that no longer exist (e.g., a helper file changed the set of dynamically
+generated methods). By default, these cause an error on the worker. To skip them gracefully
+as `Minitest::Skip` instead, set:
+
+| Variable | Description |
+|---|---|
+| `CI_QUEUE_SKIP_STALE_TESTS=1` | Report stale preresolved entries as skips instead of errors. No CLI equivalent. |
+
+**CLI flags:**
+
+| Flag | Description |
+|---|---|
+| `--preresolved-tests FILE` | Read pre-computed test names from FILE. Implies `--lazy-load`. No env var equivalent. |
+| `--test-files FILE` | In preresolved mode: reconciliation set of changed files to re-discover. |
 
 If you'd like to centralize the error reporting you can do so with:
 
@@ -71,6 +158,28 @@ rspec-queue --queue redis://example.com --timeout 600 --report
 
 Because of how `ci-queue` executes the examples, `before(:all)` and `after(:all)` hooks are not supported. `rspec-queue` will explicitly reject them.
 
+## Releasing a New Version
+
+After merging changes to `main`, follow these steps to release and propagate the update:
+
+1. **Bump the version** in `ruby/lib/ci/queue/version.rb`:
+
+    ```ruby
+    VERSION = '0.XX.0'
+    ```
+
+2. **Update `Gemfile.lock`** by running `bundle install` in the `ruby/` directory (or manually updating the version string in `Gemfile.lock` if native dependencies prevent `bundle install`).
+
+3. **Commit and merge** the version bump to `main`. ShipIt will automatically publish the gem to RubyGems.
+
+4. **Update dependent apps/zones**: Any application that depends on `ci-queue` (e.g. via its `Gemfile`) needs to pick up the new version by running:
+
+    ```bash
+    bundle update ci-queue
+    ```
+
+    This updates the app's `Gemfile.lock` to reference the new `ci-queue` version. Commit the updated `Gemfile.lock` and deploy.
+
 ## Custom Redis Expiry
 
 `ci-queue` expects the Redis server to have an [eviction policy](https://redis.io/docs/manual/eviction/#eviction-policies) of `allkeys-lru`.

data/lib/ci/queue/build_record.rb

@@ -18,18 +18,35 @@ module CI
         @queue.exhausted?
       end
 
-      def record_error(id, payload, stats: nil)
+      def record_error(id, payload, stat_delta: nil)
         error_reports[id] = payload
-        record_stats(stats)
+        true
       end
 
-      def record_success(id, stats: nil, skip_flaky_record: false, acknowledge: true)
+      def record_success(id, skip_flaky_record: false, acknowledge: true)
         error_reports.delete(id)
-        record_stats(stats)
+        true
+      end
+
+      def record_requeue(id)
+        true
+      end
+
+      def record_stats(builds_stats)
+        return unless builds_stats
+        stats.merge!(builds_stats)
+      end
+
+      def record_stats_delta(delta, pipeline: nil)
+        return if delta.nil? || delta.empty?
+        delta.each do |stat_name, value|
+          next unless value.is_a?(Numeric) || value.to_s.match?(/\A-?\d+\.?\d*\z/)
+          stats[stat_name] = (stats[stat_name] || 0).to_f + value.to_f
+        end
       end
 
       def fetch_stats(stat_names)
-        stat_names.zip(stats.values_at(*stat_names).map(&:to_f))
+        stat_names.zip(stats.values_at(*stat_names).map(&:to_f)).to_h
       end
 
       def reset_stats(stat_names)
@@ -47,11 +64,6 @@ module CI
       private
 
       attr_reader :stats
-
-      def record_stats(builds_stats)
-        return unless builds_stats
-        stats.merge!(builds_stats)
-      end
     end
   end
 end

data/lib/ci/queue/class_resolver.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module CI
+  module Queue
+    module ClassResolver
+      def self.resolve(class_name, file_path: nil, loader: nil)
+        klass = try_direct_lookup(class_name)
+        return klass if klass
+
+        if file_path && loader
+          loader.load_file(file_path)
+          klass = try_direct_lookup(class_name)
+          return klass if klass
+        end
+
+        raise ClassNotFoundError, "Unable to resolve class #{class_name}"
+      end
+
+      def self.try_direct_lookup(class_name)
+        parts = class_name.sub(/\A::/, '').split('::')
+        current = Object
+
+        parts.each do |name|
+          return nil unless current.const_defined?(name, false)
+
+          current = current.const_get(name, false)
+        end
+
+        return nil unless current.is_a?(Class)
+
+        current
+      rescue NameError
+        nil
+      end
+      private_class_method :try_direct_lookup
+    end
+  end
+end

data/lib/ci/queue/configuration.rb

@@ -6,12 +6,18 @@ module CI
       attr_accessor :requeue_tolerance, :namespace, :failing_test, :statsd_endpoint
       attr_accessor :max_test_duration, :max_test_duration_percentile, :track_test_duration
       attr_accessor :max_test_failed, :redis_ttl, :warnings_file, :debug_log, :max_missed_heartbeat_seconds
+      attr_accessor :lazy_load, :lazy_load_stream_batch_size
+      attr_writer :lazy_load_streaming_timeout
+      attr_accessor :lazy_load_test_helpers
+      attr_accessor :skip_stale_tests
       attr_reader :circuit_breakers
       attr_writer :seed, :build_id
       attr_writer :queue_init_timeout, :report_timeout, :inactive_workers_timeout
 
       class << self
         def from_env(env)
+          lazy_load_value = env['CI_QUEUE_LAZY_LOAD']
+          lazy_load = lazy_load_value && !lazy_load_value.strip.empty? && !%w(0 false).include?(lazy_load_value.strip.downcase)
           new(
             build_id: env['CIRCLE_BUILD_URL'] || env['BUILDKITE_BUILD_ID'] || env['TRAVIS_BUILD_ID'] || env['HEROKU_TEST_RUN_ID'] || env['SEMAPHORE_PIPELINE_ID'],
             worker_id: env['CIRCLE_NODE_INDEX'] || env['BUILDKITE_PARALLEL_JOB'] || env['CI_NODE_INDEX'] || env['SEMAPHORE_JOB_ID'],
@@ -22,6 +28,11 @@ module CI
             debug_log: env['CI_QUEUE_DEBUG_LOG'],
             max_requeues: env['CI_QUEUE_MAX_REQUEUES']&.to_i || 0,
             requeue_tolerance: env['CI_QUEUE_REQUEUE_TOLERANCE']&.to_f || 0,
+            lazy_load: lazy_load || false,
+            lazy_load_stream_batch_size: (env['CI_QUEUE_LAZY_LOAD_STREAM_BATCH_SIZE'] || env['CI_QUEUE_STREAM_BATCH_SIZE'])&.to_i,
+            lazy_load_streaming_timeout: (env['CI_QUEUE_LAZY_LOAD_STREAM_TIMEOUT'] || env['CI_QUEUE_STREAM_TIMEOUT'])&.to_i,
+            lazy_load_test_helpers: env['CI_QUEUE_LAZY_LOAD_TEST_HELPERS'] || env['CI_QUEUE_TEST_HELPERS'],
+            skip_stale_tests: %w(1 true).include?(env['CI_QUEUE_SKIP_STALE_TESTS']&.strip&.downcase),
           )
         end
 
@@ -46,7 +57,9 @@ module CI
         grind_count: nil, max_duration: nil, failure_file: nil, max_test_duration: nil,
         max_test_duration_percentile: 0.5, track_test_duration: false, max_test_failed: nil,
         queue_init_timeout: nil, redis_ttl: 8 * 60 * 60, report_timeout: nil, inactive_workers_timeout: nil,
-        export_flaky_tests_file: nil, warnings_file: nil, debug_log: nil, max_missed_heartbeat_seconds: nil)
+        export_flaky_tests_file: nil, warnings_file: nil, debug_log: nil, max_missed_heartbeat_seconds: nil,
+        lazy_load: false, lazy_load_stream_batch_size: nil, lazy_load_streaming_timeout: nil, lazy_load_test_helpers: nil,
+        skip_stale_tests: false)
         @build_id = build_id
         @circuit_breakers = [CircuitBreaker::Disabled]
         @failure_file = failure_file
@@ -73,6 +86,17 @@ module CI
         @warnings_file = warnings_file
         @debug_log = debug_log
         @max_missed_heartbeat_seconds = max_missed_heartbeat_seconds
+        @lazy_load = lazy_load
+        @lazy_load_stream_batch_size = lazy_load_stream_batch_size || 5_000
+        @lazy_load_streaming_timeout = lazy_load_streaming_timeout
+        @lazy_load_test_helpers = lazy_load_test_helpers
+        @skip_stale_tests = skip_stale_tests
+      end
+
+      def lazy_load_test_helper_paths
+        return [] unless @lazy_load_test_helpers
+
+        @lazy_load_test_helpers.split(',').map(&:strip)
       end
 
       def queue_init_timeout
@@ -83,6 +107,43 @@ module CI
         @report_timeout || timeout
       end
 
+      def lazy_load_streaming_timeout
+        if @lazy_load_streaming_timeout && @lazy_load_streaming_timeout > 0
+          @lazy_load_streaming_timeout
+        else
+          [queue_init_timeout, 300].max
+        end
+      end
+
+      # Backward-compatible aliases for existing callers.
+      def stream_batch_size
+        lazy_load_stream_batch_size
+      end
+
+      def stream_batch_size=(value)
+        self.lazy_load_stream_batch_size = value
+      end
+
+      def streaming_timeout
+        lazy_load_streaming_timeout
+      end
+
+      def streaming_timeout=(value)
+        self.lazy_load_streaming_timeout = value
+      end
+
+      def test_helpers
+        lazy_load_test_helpers
+      end
+
+      def test_helpers=(value)
+        self.lazy_load_test_helpers = value
+      end
+
+      def test_helper_paths
+        lazy_load_test_helper_paths
+      end
+
       def inactive_workers_timeout
         @inactive_workers_timeout || timeout
       end

data/lib/ci/queue/file_loader.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require 'set'
+
+module CI
+  module Queue
+    class FileLoader
+      attr_reader :load_stats
+
+      def initialize
+        @loaded_files = Set.new
+        @failed_files = {}
+        @pid = Process.pid
+        @forked = false
+        @load_stats = {}
+        @loaded_features = nil
+      end
+
+      def load_file(file_path)
+        detect_fork!
+        expanded = ::File.expand_path(file_path)
+        return if @loaded_files.include?(expanded)
+
+        if (cached_error = @failed_files[expanded])
+          raise cached_error
+        end
+
+        start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        error = nil
+
+        begin
+          required = with_warning_suppression { require expanded }
+          if should_force_load_after_fork?(required, expanded)
+            with_warning_suppression { load expanded }
+          end
+        rescue Exception => e
+          raise if e.is_a?(SignalException) || e.is_a?(SystemExit)
+          error = e
+        ensure
+          duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
+          @load_stats[expanded] = duration
+        end
+
+        if error
+          load_error = FileLoadError.new(file_path, error)
+          @failed_files[expanded] = load_error
+          raise load_error
+        end
+
+        remember_loaded_feature(expanded)
+        @loaded_files.add(expanded)
+        nil
+      end
+
+      def total_load_time
+        load_stats.values.sum
+      end
+
+      def slowest_files(limit = 10)
+        load_stats.sort_by { |_, duration| -duration }.take(limit)
+      end
+
+      private
+
+      def detect_fork!
+        return if @pid == Process.pid
+
+        @pid = Process.pid
+        @forked = true
+        @loaded_files.clear
+        @failed_files.clear
+        @load_stats.clear
+        @loaded_features = nil
+      end
+
+      def file_in_loaded_features?(file_path)
+        loaded_features.include?(::File.expand_path(file_path))
+      end
+
+      def loaded_features
+        @loaded_features ||= Set.new($LOADED_FEATURES.map { |loaded| ::File.expand_path(loaded) })
+      end
+
+      def remember_loaded_feature(file_path)
+        loaded_features.add(::File.expand_path(file_path))
+      end
+
+      def should_force_load_after_fork?(required, file_path)
+        @forked && !required && file_in_loaded_features?(file_path)
+      end
+
+      def with_warning_suppression
+        previous = $VERBOSE
+        $VERBOSE = nil
+        yield
+      ensure
+        $VERBOSE = previous
+      end
+    end
+  end
+end

data/lib/ci/queue/queue_entry.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'base64'
+require 'json'
+
+module CI
+  module Queue
+    module QueueEntry
+      DELIMITER = "\t"
+      LOAD_ERROR_PREFIX = '__ciq_load_error__:'.freeze
+
+      def self.test_id(entry)
+        pos = entry.index(DELIMITER)
+        pos ? entry[0, pos] : entry
+      end
+
+      def self.parse(entry)
+        return { test_id: entry, file_path: nil } unless entry.include?(DELIMITER)
+
+        test_id, file_path = entry.split(DELIMITER, 2)
+        file_path = nil if file_path == ""
+        { test_id: test_id, file_path: file_path }
+      end
+
+      def self.format(test_id, file_path)
+        return test_id if file_path.nil? || file_path == ""
+
+        "#{test_id}#{DELIMITER}#{file_path}"
+      end
+
+      def self.load_error_payload?(file_path)
+        file_path&.start_with?(LOAD_ERROR_PREFIX)
+      end
+
+      def self.encode_load_error(file_path, error)
+        original = error.respond_to?(:original_error) ? error.original_error : error
+        payload = {
+          'file_path' => file_path,
+          'error_class' => original.class.name,
+          'error_message' => original.message,
+          'backtrace' => original.backtrace,
+        }
+        "#{LOAD_ERROR_PREFIX}#{Base64.strict_encode64(JSON.dump(payload))}"
+      end
+
+      def self.decode_load_error(file_path)
+        return nil unless load_error_payload?(file_path)
+
+        encoded = file_path.sub(LOAD_ERROR_PREFIX, '')
+        JSON.parse(Base64.strict_decode64(encoded))
+      rescue ArgumentError, JSON::ParserError
+        nil
+      end
+    end
+  end
+end

data/lib/ci/queue/redis/_entry_helpers.lua

@@ -0,0 +1,10 @@
+-- AUTOGENERATED FILE DO NOT EDIT DIRECTLY
+local function test_id_from_entry(value, delimiter)
+  if delimiter then
+    local pos = string.find(value, delimiter, 1, true)
+    if pos then
+      return string.sub(value, 1, pos - 1)
+    end
+  end
+  return value
+end

data/lib/ci/queue/redis/acknowledge.lua

@@ -3,16 +3,19 @@ local zset_key = KEYS[1]
 local processed_key = KEYS[2]
 local owners_key = KEYS[3]
 local error_reports_key = KEYS[4]
+local requeued_by_key = KEYS[5]
 
-local test = ARGV[1]
-local error = ARGV[2]
-local ttl = ARGV[3]
-redis.call('zrem', zset_key, test)
-redis.call('hdel', owners_key, test)  -- Doesn't matter if it was reclaimed by another workers
-local acknowledged = redis.call('sadd', processed_key, test) == 1
+local entry = ARGV[1]
+local test_id = ARGV[2]
+local error = ARGV[3]
+local ttl = ARGV[4]
+redis.call('zrem', zset_key, entry)
+redis.call('hdel', owners_key, entry)  -- Doesn't matter if it was reclaimed by another workers
+redis.call('hdel', requeued_by_key, entry)
+local acknowledged = redis.call('sadd', processed_key, test_id) == 1
 
 if acknowledged and error ~= "" then
-  redis.call('hset', error_reports_key, test, error)
+  redis.call('hset', error_reports_key, test_id, error)
   redis.call('expire', error_reports_key, ttl)
 end
 

data/lib/ci/queue/redis/base.rb

@@ -144,19 +144,26 @@ module CI
         end
 
         def to_a
-          test_ids.reverse.map { |k| index.fetch(k) }
+          test_ids.reverse.map do |entry|
+            index.fetch(entry) do
+              test_id = CI::Queue::QueueEntry.test_id(entry)
+              index.fetch(test_id)
+            end
+          end
         end
 
         def progress
-          total - size
+          progress = total - size
+          progress < 0 ? 0 : progress
         end
 
-        def wait_for_master(timeout: 30)
+        def wait_for_master(timeout: 30, allow_streaming: false)
           return true if master?
           return true if queue_initialized?
+          return true if allow_streaming && streaming?
 
           (timeout * 10 + 1).to_i.times do
-            if queue_initialized?
+            if queue_initialized? || (allow_streaming && streaming?)
               return true
             else
               sleep 0.1
@@ -177,6 +184,10 @@ module CI
           end
         end
 
+        def streaming?
+          master_status == 'streaming'
+        end
+
         def queue_initializing?
           master_status == 'setup'
         end
@@ -214,7 +225,7 @@ module CI
         end
 
         def key(*args)
-          KeyShortener.key(config.build_id, *args)
+          ['build', build_id, *args].join(':')
         end
 
         def build_id
@@ -235,18 +246,31 @@ module CI
         end
 
         def read_script(name)
-          ::File.read(::File.join(CI::Queue::DEV_SCRIPTS_ROOT, "#{name}.lua"))
+          resolve_lua_includes(
+            ::File.read(::File.join(CI::Queue::DEV_SCRIPTS_ROOT, "#{name}.lua")),
+            CI::Queue::DEV_SCRIPTS_ROOT,
+          )
         rescue SystemCallError
-          ::File.read(::File.join(CI::Queue::RELEASE_SCRIPTS_ROOT, "#{name}.lua"))
+          resolve_lua_includes(
+            ::File.read(::File.join(CI::Queue::RELEASE_SCRIPTS_ROOT, "#{name}.lua")),
+            CI::Queue::RELEASE_SCRIPTS_ROOT,
+          )
+        end
+
+        def resolve_lua_includes(script, root)
+          script.gsub(/^-- @include (\S+)$/) do
+            ::File.read(::File.join(root, "#{$1}.lua"))
+          end
         end
 
         class HeartbeatProcess
-          def initialize(redis_url, zset_key, processed_key, owners_key, worker_queue_key)
+          def initialize(redis_url, zset_key, processed_key, owners_key, worker_queue_key, entry_delimiter:)
             @redis_url = redis_url
             @zset_key = zset_key
             @processed_key = processed_key
             @owners_key = owners_key
             @worker_queue_key = worker_queue_key
+            @entry_delimiter = entry_delimiter
           end
 
           def boot!
@@ -261,6 +285,7 @@ module CI
               @processed_key,
               @owners_key,
               @worker_queue_key,
+              @entry_delimiter,
               in: child_read,
               out: child_write,
             )
@@ -335,6 +360,7 @@ module CI
             key('processed'),
             key('owners'),
             key('worker', worker_id, 'queue'),
+            entry_delimiter: CI::Queue::QueueEntry::DELIMITER,
           )
         end