ci-queue 0.47.0 → 0.49.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 44d3c9d768457c849133f6886a33d756823e5d59393f045ffd22222538745e09
-  data.tar.gz: 416acb36f1b32c8cbb170e71346522d735bf9375c7032f7337ef4fb5901b9e85
+  metadata.gz: 946f3f483ffbf8d89553c575d733767bd0851dffaa1439463ce43508f1727eb1
+  data.tar.gz: b74f1059c33686eacc6ef613900595ce8ff74d7fdd55f5f14c856c33fe35c2b3
 SHA512:
-  metadata.gz: 41d246408b2a2cbc619978a37e67de356169ce048216359cf063b1f623b03a8073be4cf117cbedc6c19719d2ff13db94e07ad6cdc804fba104fea9ee5c236614
-  data.tar.gz: bbe69b32ccc8253323ed181852fdea8c4a22cf6b69e0136bfafc0722421efa94ecf171ca669e6a454c7d0adc80d55f7b59ced7585e763047122ca0d01131f4c4
+  metadata.gz: 4eca10a80566c11cb600f133008848bde44adec4347f16ec174300e24fb7fafc8727f2dbeae90884b41c60ed784ea8477c5ac896fefac3a7170c84e6b87249fd
+  data.tar.gz: e8e04782df56a2b1cfd873e3ec9295f4e91b848f7f02cc99df41a349a45115a6ceb0fcd2263fc19e54dfe0b72fb870afc80bd74970f443920c0e31a7c5af8937

data/.gitignore

@@ -4,7 +4,6 @@ test/fixtures/log/test_data.json
 /.byebug_history
 /.bundle/
 /.yardoc
-/Gemfile.lock
 /_yardoc/
 /coverage/
 /doc/

data/.ruby-version

@@ -0,0 +1 @@
+3.3.0

data/Gemfile.lock

@@ -0,0 +1,111 @@
+PATH
+  remote: .
+  specs:
+    ci-queue (0.49.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (7.1.3.2)
+      base64
+      bigdecimal
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      connection_pool (>= 2.2.5)
+      drb
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      mutex_m
+      tzinfo (~> 2.0)
+    ansi (1.5.0)
+    ast (2.4.2)
+    base64 (0.2.0)
+    bigdecimal (3.1.7)
+    builder (3.2.4)
+    concurrent-ruby (1.2.3)
+    connection_pool (2.4.1)
+    diff-lcs (1.5.1)
+    docile (1.4.0)
+    drb (2.2.1)
+    i18n (1.14.4)
+      concurrent-ruby (~> 1.0)
+    json (2.7.1)
+    language_server-protocol (3.17.0.3)
+    minitest (5.22.3)
+    minitest-reporters (1.6.1)
+      ansi
+      builder
+      minitest (>= 5.0)
+      ruby-progressbar
+    msgpack (1.7.2)
+    mutex_m (0.2.0)
+    parallel (1.24.0)
+    parser (3.3.0.5)
+      ast (~> 2.4.1)
+      racc
+    racc (1.7.3)
+    rainbow (3.1.1)
+    rake (13.1.0)
+    redis (5.1.0)
+      redis-client (>= 0.17.0)
+    redis-client (0.21.1)
+      connection_pool
+    regexp_parser (2.9.0)
+    rexml (3.2.6)
+    rspec (3.13.0)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.0)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.1)
+    rubocop (1.62.1)
+      json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
+      parallel (~> 1.10)
+      parser (>= 3.3.0.2)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.31.2)
+      parser (>= 3.3.0.4)
+    ruby-progressbar (1.13.0)
+    simplecov (0.22.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.4)
+    snappy (0.4.0)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (2.5.0)
+
+PLATFORMS
+  arm64-darwin-23
+  ruby
+
+DEPENDENCIES
+  activesupport
+  bundler
+  ci-queue!
+  minitest (~> 5.11)
+  minitest-reporters (~> 1.1)
+  msgpack
+  rake
+  redis
+  rspec (~> 3.10)
+  rubocop
+  simplecov (~> 0.12)
+  snappy
+
+BUNDLED WITH
+   2.5.3

data/ci-queue.gemspec

@@ -19,6 +19,8 @@ Gem::Specification.new do |spec|
   spec.homepage      = 'https://github.com/Shopify/ci-queue'
   spec.license       = 'MIT'
 
+  spec.required_ruby_version = '>= 2.7'
+
   spec.files         = lua_scripts + `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
   end

data/dev.yml

@@ -3,9 +3,9 @@
 name: ci-queue
 
 up:
-- ruby: 3.3.0
+- ruby
 - bundler
 - redis
 
 commands:
-  test: REDIS_URL=${REDIS_URL:-redis://ci-queue.railgun/0} bundle exec rake test TEST_FILES="$*"
+  test: REDIS_URL=${REDIS_URL} bundle exec rake test TEST_FILES="$*"

data/lib/ci/queue/redis/supervisor.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module CI
   module Queue
     module Redis
@@ -17,11 +18,13 @@ module CI
         end
 
         def wait_for_workers
-          wait_for_master(timeout: config.queue_init_timeout)
+          duration = measure do
+            wait_for_master(timeout: config.queue_init_timeout)
+          end
 
           yield if block_given?
 
-          time_left = config.report_timeout
+          time_left = config.report_timeout - duration.to_i
           time_left_with_no_workers = config.inactive_workers_timeout
           until exhausted? || time_left <= 0 || max_test_failed? || time_left_with_no_workers <= 0
             time_left -= 1
@@ -44,6 +47,12 @@ module CI
 
         private
 
+        def measure
+          starting = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          yield
+          Process.clock_gettime(Process::CLOCK_MONOTONIC) - starting
+        end
+
         def active_workers?
           # if there are running jobs we assume there are still agents active
           redis.zrangebyscore(key('running'), CI::Queue.time_now.to_f - config.timeout, "+inf", limit: [0,1]).count > 0

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.47.0'
+    VERSION = '0.49.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue/runner.rb

@@ -49,7 +49,8 @@ module Minitest
 
       def run_command
         require_worker_id!
-        if queue.retrying? || retry?
+        # if it's an automatic job retry we should process the main queue
+        if manual_retry?
           if queue.expired?
             abort! "The test run is too old and can't be retried"
           end
@@ -207,6 +208,12 @@ module Minitest
           puts
         end
 
+        if run_index == 0
+          step(yellow("The failing test was the first test in the test order so there is nothing to bisect."))
+          File.write('log/test_order.log', "")
+          exit! 1
+        end
+
         failing_order = queue.candidates
         step("Final validation")
         status = if run_tests_in_fork(failing_order)
@@ -220,7 +227,7 @@ module Minitest
           command += argv
 
           puts
-          puts "cat <<EOF |\n#{failing_order.to_a.map(&:id).join("\n")}\nEOF\n#{command.join(' ')}"
+          puts "cat <<'EOF' |\n#{failing_order.to_a.map(&:id).join("\n")}\nEOF\n#{command.join(' ')}"
           puts
 
           File.write('log/test_order.log', failing_order.to_a.map(&:id).join("\n"))
@@ -665,6 +672,12 @@ module Minitest
         exit! exit_status # exit! is required to avoid minitest at_exit callback
       end
 
+      def manual_retry?
+        # this env variable only exists on Buildkite so we should default to manual for backward compatibility
+        (retry? || queue.retrying?) &&
+          ENV.fetch("BUILDKITE_RETRY_TYPE", "manual") == "manual"
+      end
+
       def retry?
         ENV["BUILDKITE_RETRY_COUNT"].to_i > 0 ||
           ENV["SEMAPHORE_PIPELINE_RERUN"] == "true"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.47.0
+  version: 0.49.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-02-08 00:00:00.000000000 Z
+date: 2024-04-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -160,7 +160,9 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".ruby-version"
 - Gemfile
+- Gemfile.lock
 - README.md
 - Rakefile
 - bin/bundler
@@ -171,7 +173,6 @@ files:
 - dev.yml
 - exe/minitest-queue
 - exe/rspec-queue
-- isogun.yml
 - lib/ci/queue.rb
 - lib/ci/queue/bisect.rb
 - lib/ci/queue/build_record.rb
@@ -234,14 +235,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.6
+rubygems_version: 3.5.7
 signing_key: 
 specification_version: 4
 summary: Distribute tests over many workers using a queue

data/isogun.yml

@@ -1,10 +0,0 @@
-# This file is for Shopify employees development environment.
-# If you are an external contributor you don't have to bother with it.
-name: ci-queue
-
-vm:
-  ip_address: 192.168.64.245
-  memory: 1G
-  cores: 2
-services:
-   - redis