knapsack_pro 7.12.1 → 7.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 204e1595877682d96997ee193c8b0aa0230d6f5e1c22d37ca515e1e94a40d404
-  data.tar.gz: 4a373acd12bdd3451bda93d7186a69a217aeeea593c3ad1d8cb425d1f215d0cc
+  metadata.gz: ddf5678a555f4e33ac61a02ba0f05b916a5e65dd8dca2b5a06e7d2bea74da028
+  data.tar.gz: aaa74b69c3753b8f70aa8deff5528413db49c53b731ce65db571d5d768c0d4f0
 SHA512:
-  metadata.gz: 9f2310862a878b55c4001beaf3beda4c2249946a7fc04fe40f3b4e5911c17aff1ffc1df2437df392aa597681396cc1dc1e7a0deac67936d6e5cbf399bda830c5
-  data.tar.gz: a3d8b7bf464a71a5a0600333f017a981562ecab163c52f1b67747c5aaad444aab834ab7d0a93ef00cad012e2d4f66163ec1cfa9b9204e4bb13477b1fdce263de
+  metadata.gz: dbc287b2088b1fb96a76e9ca59d3e37c40e426644b21c0b85fd373737847b0966f56b70309b5ba2afa32c26279807ea858d99e09ab9e9feb38ed0f1819320dcd
+  data.tar.gz: fc31eb5aa8f02cb4a77221d04d5260e607a8af70e3174b89c0d578c67f2616c4bee76a098d7f405942a0bcf678399b9d270b42e50d1e89902233f1e7203ea8f1

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+### 7.13.0
+
+* Add `KNAPSACK_PRO_FALLBACK_MODE_ERROR_EXIT_CODE` to specify a custom exit code whenever Knapsack Pro fails because Fallback Mode cannot be used
+
+    https://github.com/KnapsackPro/knapsack_pro-ruby/pull/284
+
+https://github.com/KnapsackPro/knapsack_pro-ruby/compare/v7.12.1...v7.13.0
+
 ### 7.12.1
 
 * fix(RSpec split by examples): properly disable split by test examples on a single node to speed up tests

data/lib/knapsack_pro/allocator.rb

@@ -29,14 +29,16 @@ module KnapsackPro
       elsif !KnapsackPro::Config::Env.fallback_mode_enabled?
         message = "Fallback Mode was disabled with KNAPSACK_PRO_FALLBACK_MODE_ENABLED=false. Please restart this CI node to retry tests. Most likely Fallback Mode was disabled due to #{KnapsackPro::Urls::REGULAR_MODE__CONNECTION_ERROR_WITH_FALLBACK_ENABLED_FALSE}"
         KnapsackPro.logger.error(message)
-        raise message
+        exit_code = KnapsackPro::Config::Env.fallback_mode_error_exit_code
+        Kernel.exit(exit_code)
       elsif KnapsackPro::Config::Env.ci_node_retry_count > 0
         message = "knapsack_pro gem could not connect to Knapsack Pro API and the Fallback Mode cannot be used this time. Running tests in Fallback Mode are not allowed for retried parallel CI node to avoid running the wrong set of tests. Please manually retry this parallel job on your CI server then knapsack_pro gem will try to connect to Knapsack Pro API again and will run a correct set of tests for this CI node. Learn more #{KnapsackPro::Urls::REGULAR_MODE__CONNECTION_ERROR_WITH_FALLBACK_ENABLED_TRUE_AND_POSITIVE_RETRY_COUNT}"
         unless KnapsackPro::Config::Env.fixed_test_suite_split?
           message += " Please ensure you have set KNAPSACK_PRO_FIXED_TEST_SUITE_SPLIT=true to allow Knapsack Pro API remember the recorded CI node tests so when you retry failed tests on the CI node then the same set of tests will be executed. See more #{KnapsackPro::Urls::FIXED_TEST_SUITE_SPLIT}"
         end
         KnapsackPro.logger.error(message)
-        raise message
+        exit_code = KnapsackPro::Config::Env.fallback_mode_error_exit_code
+        Kernel.exit(exit_code)
       else
         KnapsackPro.logger.warn("Fallback mode started. We could not connect with Knapsack Pro API. Your tests will be executed based on directory names. Read more about fallback mode at #{KnapsackPro::Urls::FALLBACK_MODE}")
         fallback_test_files

data/lib/knapsack_pro/config/env.rb

@@ -284,6 +284,10 @@ module KnapsackPro
             detected_ci != KnapsackPro::Config::CI::Base
         end
 
+        def fallback_mode_error_exit_code
+          ENV.fetch('KNAPSACK_PRO_FALLBACK_MODE_ERROR_EXIT_CODE', 1).to_i
+        end
+
         private
 
         def required_env(env_name)

data/lib/knapsack_pro/queue_allocator.rb

@@ -2,6 +2,8 @@
 
 module KnapsackPro
   class QueueAllocator
+    FallbackModeError = Class.new(StandardError)
+
     def initialize(args)
       @fast_and_slow_test_files_to_run = args.fetch(:fast_and_slow_test_files_to_run)
       @fallback_mode_test_files = args.fetch(:fallback_mode_test_files)
@@ -31,14 +33,14 @@ module KnapsackPro
       elsif !KnapsackPro::Config::Env.fallback_mode_enabled?
         message = "Fallback Mode was disabled with KNAPSACK_PRO_FALLBACK_MODE_ENABLED=false. Please restart this CI node to retry tests. Most likely Fallback Mode was disabled due to #{KnapsackPro::Urls::QUEUE_MODE__CONNECTION_ERROR_WITH_FALLBACK_ENABLED_FALSE}"
         KnapsackPro.logger.error(message)
-        raise message
+        raise FallbackModeError.new(message)
       elsif KnapsackPro::Config::Env.ci_node_retry_count > 0
         message = "knapsack_pro gem could not connect to Knapsack Pro API and the Fallback Mode cannot be used this time. Running tests in Fallback Mode are not allowed for retried parallel CI node to avoid running the wrong set of tests. Please manually retry this parallel job on your CI server then knapsack_pro gem will try to connect to Knapsack Pro API again and will run a correct set of tests for this CI node. Learn more #{KnapsackPro::Urls::QUEUE_MODE__CONNECTION_ERROR_WITH_FALLBACK_ENABLED_TRUE_AND_POSITIVE_RETRY_COUNT}"
         unless KnapsackPro::Config::Env.fixed_queue_split?
           message += " Please ensure you have set KNAPSACK_PRO_FIXED_QUEUE_SPLIT=true to allow Knapsack Pro API remember the recorded CI node tests so when you retry failed tests on the CI node then the same set of tests will be executed. See more #{KnapsackPro::Urls::FIXED_QUEUE_SPLIT}"
         end
         KnapsackPro.logger.error(message)
-        raise message
+        raise FallbackModeError.new(message)
       else
         @fallback_activated = true
         KnapsackPro.logger.warn("Fallback mode started. We could not connect with Knapsack Pro API. Your tests will be executed based on directory names. If other CI nodes were able to connect with Knapsack Pro API then you may notice that some of the test files will be executed twice across CI nodes. The most important thing is to guarantee each of test files is run at least once! Read more about fallback mode at #{KnapsackPro::Urls::FALLBACK_MODE}")

data/lib/knapsack_pro/runners/queue/cucumber_runner.rb

@@ -29,6 +29,9 @@ module KnapsackPro
           end
 
           Kernel.exit(accumulator[:exitstatus])
+        rescue KnapsackPro::QueueAllocator::FallbackModeError
+          exit_code = KnapsackPro::Config::Env.fallback_mode_error_exit_code
+          Kernel.exit(exit_code)
         end
 
         def self.run_tests(accumulator)

data/lib/knapsack_pro/runners/queue/minitest_runner.rb

@@ -40,6 +40,9 @@ module KnapsackPro
           end
 
           Kernel.exit(accumulator[:exitstatus])
+        rescue KnapsackPro::QueueAllocator::FallbackModeError
+          exit_code = KnapsackPro::Config::Env.fallback_mode_error_exit_code
+          Kernel.exit(exit_code)
         end
 
         def self.run_tests(accumulator)

data/lib/knapsack_pro/runners/queue/rspec_runner.rb

@@ -55,6 +55,9 @@ module KnapsackPro
           rescue KnapsackPro::Runners::Queue::BaseRunner::TerminationError
             exit_code = @rspec_pure.error_exit_code(@rspec_runner.knapsack__error_exit_code)
             Kernel.exit(exit_code)
+          rescue KnapsackPro::QueueAllocator::FallbackModeError
+            exit_code = KnapsackPro::Config::Env.fallback_mode_error_exit_code
+            Kernel.exit(exit_code)
           rescue Exception => exception
             KnapsackPro.logger.error("An unexpected exception happened. RSpec cannot handle it. The exception: #{exception.inspect}")
             KnapsackPro.logger.error("Exception message: #{exception.message}")

data/lib/knapsack_pro/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module KnapsackPro
-  VERSION = '7.12.1'
+  VERSION = '7.13.0'
 end

data/spec/integration/runners/fallback_spec.rb

@@ -0,0 +1,131 @@
+require 'open3'
+require 'ostruct'
+require 'tempfile'
+
+['RSpec', 'Minitest', 'Cucumber'].each do |runner|
+  describe "#{Object.const_get("KnapsackPro::Runners::#{runner}Runner")} Fallback - Integration tests" do
+    def log(stdout, stderr, status)
+      puts '='*50
+      puts 'STDOUT:'
+      puts stdout
+      puts
+
+      puts '='*50
+      puts 'STDERR:'
+      puts stderr
+      puts
+
+      puts '='*50
+      puts 'Exit status code:'
+      puts status
+      puts
+    end
+
+    subject do
+      stdout, stderr, status = Open3.capture3("ruby #{@task.path}")
+      log(stdout, stderr, status) if ENV['TEST__SHOW_DEBUG_LOG'] == 'true'
+      OpenStruct.new(stdout: stdout, stderr: stderr, exit_code: status.exitstatus)
+    end
+
+    before(:each) do
+      ENV['KNAPSACK_PRO_ENDPOINT'] = 'https://fail.knapsackpro.com' # ensure the API is not reachable
+      ENV['KNAPSACK_PRO_MAX_REQUEST_RETRIES'] = '1' # only try once to reach the API
+    end
+
+    around(:each) do |example|
+      Tempfile.create do |file|
+        file.write(<<~CONTENT)
+          require 'knapsack_pro'
+
+          ENV['KNAPSACK_PRO_TEST_SUITE_TOKEN_#{runner.upcase}'] = SecureRandom.hex
+          ENV['KNAPSACK_PRO_CI_NODE_BUILD_ID'] = SecureRandom.uuid
+
+          #{Object.const_get("KnapsackPro::Runners::#{runner}Runner")}.run('')
+        CONTENT
+        file.rewind
+
+        @task = file
+        example.run
+      end
+    end
+
+    after(:each) do
+      ENV.delete('KNAPSACK_PRO_ENDPOINT')
+      ENV.delete('KNAPSACK_PRO_MAX_REQUEST_RETRIES')
+    end
+
+    context 'with fallback mode disabled' do
+      before(:each) do
+        ENV['KNAPSACK_PRO_FALLBACK_MODE_ENABLED'] = 'false'
+      end
+
+      after(:each) do
+        ENV.delete('KNAPSACK_PRO_FALLBACK_MODE_ENABLED')
+      end
+
+      it 'exits with 1 and logs the error' do
+        actual = subject
+
+        expect(actual.exit_code).to eq 1
+
+        expect(actual.stdout).to include('ERROR -- : [knapsack_pro] Fallback Mode was disabled with KNAPSACK_PRO_FALLBACK_MODE_ENABLED=false. Please restart this CI node to retry tests. Most likely Fallback Mode was disabled due to https://knapsackpro.com/perma/ruby/regular-mode-connection-error-with-fallback-enabled-false')
+      end
+
+      context 'with a user defined exit code' do
+        before do
+          ENV['KNAPSACK_PRO_FALLBACK_MODE_ERROR_EXIT_CODE'] = '123'
+        end
+
+        after do
+          ENV.delete('KNAPSACK_PRO_FALLBACK_MODE_ERROR_EXIT_CODE')
+        end
+
+        it 'exits with the passed exit code and logs the error' do
+          actual = subject
+
+          expect(actual.exit_code).to eq 123
+
+          expect(actual.stdout).to include('ERROR -- : [knapsack_pro] Fallback Mode was disabled with KNAPSACK_PRO_FALLBACK_MODE_ENABLED=false. Please restart this CI node to retry tests. Most likely Fallback Mode was disabled due to https://knapsackpro.com/perma/ruby/regular-mode-connection-error-with-fallback-enabled-false')
+        end
+      end
+    end
+
+    context 'with fallback mode enabled and positive node retry count' do
+      before(:each) do
+        ENV['KNAPSACK_PRO_FALLBACK_MODE_ENABLED'] = 'true'
+        ENV['KNAPSACK_PRO_CI_NODE_RETRY_COUNT'] = '1'
+      end
+
+      after(:each) do
+        ENV.delete('KNAPSACK_PRO_FALLBACK_MODE_ENABLED')
+        ENV.delete('KNAPSACK_PRO_CI_NODE_RETRY_COUNT')
+      end
+
+      it 'exits with 1 and logs the error' do
+        actual = subject
+
+        expect(actual.exit_code).to eq 1
+
+        expect(actual.stdout).to include('ERROR -- : [knapsack_pro] knapsack_pro gem could not connect to Knapsack Pro API and the Fallback Mode cannot be used this time. Running tests in Fallback Mode are not allowed for retried parallel CI node to avoid running the wrong set of tests. Please manually retry this parallel job on your CI server then knapsack_pro gem will try to connect to Knapsack Pro API again and will run a correct set of tests for this CI node. Learn more https://knapsackpro.com/perma/ruby/regular-mode-connection-error-with-fallback-enabled-true-and-positive-retry-count')
+      end
+
+      context 'with a user defined exit code' do
+        before do
+          ENV['KNAPSACK_PRO_FALLBACK_MODE_ERROR_EXIT_CODE'] = '123'
+        end
+
+        after do
+          ENV.delete('KNAPSACK_PRO_FALLBACK_MODE_ERROR_EXIT_CODE')
+        end
+
+        it 'exits with the passed exit code and logs the error' do
+          actual = subject
+
+          expect(actual.exit_code).to eq 123
+
+          expect(actual.stdout).to include('ERROR -- : [knapsack_pro] knapsack_pro gem could not connect to Knapsack Pro API and the Fallback Mode cannot be used this time. Running tests in Fallback Mode are not allowed for retried parallel CI node to avoid running the wrong set of tests. Please manually retry this parallel job on your CI server then knapsack_pro gem will try to connect to Knapsack Pro API again and will run a correct set of tests for this CI node. Learn more https://knapsackpro.com/perma/ruby/regular-mode-connection-error-with-fallback-enabled-true-and-positive-retry-count')
+        end
+      end
+    end
+  end
+end

data/spec/integration/runners/queue/cucumber_runner_fallback_spec.rb

@@ -0,0 +1,129 @@
+require 'open3'
+require 'ostruct'
+require 'tempfile'
+
+describe "#{KnapsackPro::Runners::Queue::CucumberRunner} Fallback - Integration tests" do
+  def log(stdout, stderr, status)
+    puts '='*50
+    puts 'STDOUT:'
+    puts stdout
+    puts
+
+    puts '='*50
+    puts 'STDERR:'
+    puts stderr
+    puts
+
+    puts '='*50
+    puts 'Exit status code:'
+    puts status
+    puts
+  end
+
+  subject do
+    stdout, stderr, status = Open3.capture3("ruby #{@task.path}")
+    log(stdout, stderr, status) if ENV['TEST__SHOW_DEBUG_LOG'] == 'true'
+    OpenStruct.new(stdout: stdout, stderr: stderr, exit_code: status.exitstatus)
+  end
+
+  before(:each) do
+    ENV['KNAPSACK_PRO_ENDPOINT'] = 'https://fail.knapsackpro.com' # ensure the API is not reachable
+    ENV['KNAPSACK_PRO_MAX_REQUEST_RETRIES'] = '1' # only try once to reach the API
+  end
+
+  around(:each) do |example|
+    Tempfile.create do |file|
+      file.write(<<~CONTENT)
+        require 'knapsack_pro'
+
+        ENV['KNAPSACK_PRO_TEST_SUITE_TOKEN_CUCUMBER'] = SecureRandom.hex
+        ENV['KNAPSACK_PRO_CI_NODE_BUILD_ID'] = SecureRandom.uuid
+
+        #{KnapsackPro::Runners::Queue::CucumberRunner}.run('')
+      CONTENT
+      file.rewind
+
+      @task = file
+      example.run
+    end
+  end
+
+  after(:each) do
+    ENV.delete('KNAPSACK_PRO_ENDPOINT')
+    ENV.delete('KNAPSACK_PRO_MAX_REQUEST_RETRIES')
+  end
+
+  context 'with fallback mode disabled' do
+    before(:each) do
+      ENV['KNAPSACK_PRO_FALLBACK_MODE_ENABLED'] = 'false'
+    end
+
+    after(:each) do
+      ENV.delete('KNAPSACK_PRO_FALLBACK_MODE_ENABLED')
+    end
+
+    it 'exits with 1 and logs the error' do
+      actual = subject
+
+      expect(actual.exit_code).to eq 1
+
+      expect(actual.stdout).to include('ERROR -- : [knapsack_pro] Fallback Mode was disabled with KNAPSACK_PRO_FALLBACK_MODE_ENABLED=false. Please restart this CI node to retry tests. Most likely Fallback Mode was disabled due to https://knapsackpro.com/perma/ruby/queue-mode-connection-error-with-fallback-enabled-false')
+    end
+
+    context 'with a user defined exit code' do
+      before do
+        ENV['KNAPSACK_PRO_FALLBACK_MODE_ERROR_EXIT_CODE'] = '123'
+      end
+
+      after do
+        ENV.delete('KNAPSACK_PRO_FALLBACK_MODE_ERROR_EXIT_CODE')
+      end
+
+      it 'exits with the passed exit code and logs the error' do
+        actual = subject
+
+        expect(actual.exit_code).to eq 123
+
+        expect(actual.stdout).to include('ERROR -- : [knapsack_pro] Fallback Mode was disabled with KNAPSACK_PRO_FALLBACK_MODE_ENABLED=false. Please restart this CI node to retry tests. Most likely Fallback Mode was disabled due to https://knapsackpro.com/perma/ruby/queue-mode-connection-error-with-fallback-enabled-false')
+      end
+    end
+  end
+
+  context 'with fallback mode enabled and positive node retry count' do
+    before(:each) do
+      ENV['KNAPSACK_PRO_FALLBACK_MODE_ENABLED'] = 'true'
+      ENV['KNAPSACK_PRO_CI_NODE_RETRY_COUNT'] = '1'
+    end
+
+    after(:each) do
+      ENV.delete('KNAPSACK_PRO_FALLBACK_MODE_ENABLED')
+      ENV.delete('KNAPSACK_PRO_CI_NODE_RETRY_COUNT')
+    end
+
+    it 'exits with 1 and logs the error' do
+      actual = subject
+
+      expect(actual.exit_code).to eq 1
+
+      expect(actual.stdout).to include('ERROR -- : [knapsack_pro] knapsack_pro gem could not connect to Knapsack Pro API and the Fallback Mode cannot be used this time. Running tests in Fallback Mode are not allowed for retried parallel CI node to avoid running the wrong set of tests. Please manually retry this parallel job on your CI server then knapsack_pro gem will try to connect to Knapsack Pro API again and will run a correct set of tests for this CI node. Learn more https://knapsackpro.com/perma/ruby/queue-mode-connection-error-with-fallback-enabled-true-and-positive-retry-count')
+    end
+
+    context 'with a user defined exit code' do
+      before do
+        ENV['KNAPSACK_PRO_FALLBACK_MODE_ERROR_EXIT_CODE'] = '123'
+      end
+
+      after do
+        ENV.delete('KNAPSACK_PRO_FALLBACK_MODE_ERROR_EXIT_CODE')
+      end
+
+      it 'exits with the passed exit code and logs the error' do
+        actual = subject
+
+        expect(actual.exit_code).to eq 123
+
+        expect(actual.stdout).to include('ERROR -- : [knapsack_pro] knapsack_pro gem could not connect to Knapsack Pro API and the Fallback Mode cannot be used this time. Running tests in Fallback Mode are not allowed for retried parallel CI node to avoid running the wrong set of tests. Please manually retry this parallel job on your CI server then knapsack_pro gem will try to connect to Knapsack Pro API again and will run a correct set of tests for this CI node. Learn more https://knapsackpro.com/perma/ruby/queue-mode-connection-error-with-fallback-enabled-true-and-positive-retry-count')
+      end
+    end
+  end
+end

data/spec/integration/runners/queue/minitest_runner_fallback_spec.rb

@@ -0,0 +1,129 @@
+require 'open3'
+require 'ostruct'
+require 'tempfile'
+
+describe "#{KnapsackPro::Runners::Queue::MinitestRunner} Fallback - Integration tests" do
+  def log(stdout, stderr, status)
+    puts '='*50
+    puts 'STDOUT:'
+    puts stdout
+    puts
+
+    puts '='*50
+    puts 'STDERR:'
+    puts stderr
+    puts
+
+    puts '='*50
+    puts 'Exit status code:'
+    puts status
+    puts
+  end
+
+  subject do
+    stdout, stderr, status = Open3.capture3("ruby #{@task.path}")
+    log(stdout, stderr, status) if ENV['TEST__SHOW_DEBUG_LOG'] == 'true'
+    OpenStruct.new(stdout: stdout, stderr: stderr, exit_code: status.exitstatus)
+  end
+
+  before(:each) do
+    ENV['KNAPSACK_PRO_ENDPOINT'] = 'https://fail.knapsackpro.com' # ensure the API is not reachable
+    ENV['KNAPSACK_PRO_MAX_REQUEST_RETRIES'] = '1' # only try once to reach the API
+  end
+
+  around(:each) do |example|
+    Tempfile.create do |file|
+      file.write(<<~CONTENT)
+        require 'knapsack_pro'
+
+        ENV['KNAPSACK_PRO_TEST_SUITE_TOKEN_MINITEST'] = SecureRandom.hex
+        ENV['KNAPSACK_PRO_CI_NODE_BUILD_ID'] = SecureRandom.uuid
+
+        #{KnapsackPro::Runners::Queue::MinitestRunner}.run('')
+      CONTENT
+      file.rewind
+
+      @task = file
+      example.run
+    end
+  end
+
+  after(:each) do
+    ENV.delete('KNAPSACK_PRO_ENDPOINT')
+    ENV.delete('KNAPSACK_PRO_MAX_REQUEST_RETRIES')
+  end
+
+  context 'with fallback mode disabled' do
+    before(:each) do
+      ENV['KNAPSACK_PRO_FALLBACK_MODE_ENABLED'] = 'false'
+    end
+
+    after(:each) do
+      ENV.delete('KNAPSACK_PRO_FALLBACK_MODE_ENABLED')
+    end
+
+    it 'exits with 1 and logs the error' do
+      actual = subject
+
+      expect(actual.exit_code).to eq 1
+
+      expect(actual.stdout).to include('ERROR -- : [knapsack_pro] Fallback Mode was disabled with KNAPSACK_PRO_FALLBACK_MODE_ENABLED=false. Please restart this CI node to retry tests. Most likely Fallback Mode was disabled due to https://knapsackpro.com/perma/ruby/queue-mode-connection-error-with-fallback-enabled-false')
+    end
+
+    context 'with a user defined exit code' do
+      before do
+        ENV['KNAPSACK_PRO_FALLBACK_MODE_ERROR_EXIT_CODE'] = '123'
+      end
+
+      after do
+        ENV.delete('KNAPSACK_PRO_FALLBACK_MODE_ERROR_EXIT_CODE')
+      end
+
+      it 'exits with the passed exit code and logs the error' do
+        actual = subject
+
+        expect(actual.exit_code).to eq 123
+
+        expect(actual.stdout).to include('ERROR -- : [knapsack_pro] Fallback Mode was disabled with KNAPSACK_PRO_FALLBACK_MODE_ENABLED=false. Please restart this CI node to retry tests. Most likely Fallback Mode was disabled due to https://knapsackpro.com/perma/ruby/queue-mode-connection-error-with-fallback-enabled-false')
+      end
+    end
+  end
+
+  context 'with fallback mode enabled and positive node retry count' do
+    before(:each) do
+      ENV['KNAPSACK_PRO_FALLBACK_MODE_ENABLED'] = 'true'
+      ENV['KNAPSACK_PRO_CI_NODE_RETRY_COUNT'] = '1'
+    end
+
+    after(:each) do
+      ENV.delete('KNAPSACK_PRO_FALLBACK_MODE_ENABLED')
+      ENV.delete('KNAPSACK_PRO_CI_NODE_RETRY_COUNT')
+    end
+
+    it 'exits with 1 and logs the error' do
+      actual = subject
+
+      expect(actual.exit_code).to eq 1
+
+      expect(actual.stdout).to include('ERROR -- : [knapsack_pro] knapsack_pro gem could not connect to Knapsack Pro API and the Fallback Mode cannot be used this time. Running tests in Fallback Mode are not allowed for retried parallel CI node to avoid running the wrong set of tests. Please manually retry this parallel job on your CI server then knapsack_pro gem will try to connect to Knapsack Pro API again and will run a correct set of tests for this CI node. Learn more https://knapsackpro.com/perma/ruby/queue-mode-connection-error-with-fallback-enabled-true-and-positive-retry-count')
+    end
+
+    context 'with a user defined exit code' do
+      before do
+        ENV['KNAPSACK_PRO_FALLBACK_MODE_ERROR_EXIT_CODE'] = '123'
+      end
+
+      after do
+        ENV.delete('KNAPSACK_PRO_FALLBACK_MODE_ERROR_EXIT_CODE')
+      end
+
+      it 'exits with the passed exit code and logs the error' do
+        actual = subject
+
+        expect(actual.exit_code).to eq 123
+
+        expect(actual.stdout).to include('ERROR -- : [knapsack_pro] knapsack_pro gem could not connect to Knapsack Pro API and the Fallback Mode cannot be used this time. Running tests in Fallback Mode are not allowed for retried parallel CI node to avoid running the wrong set of tests. Please manually retry this parallel job on your CI server then knapsack_pro gem will try to connect to Knapsack Pro API again and will run a correct set of tests for this CI node. Learn more https://knapsackpro.com/perma/ruby/queue-mode-connection-error-with-fallback-enabled-true-and-positive-retry-count')
+      end
+    end
+  end
+end

data/spec/integration/runners/queue/rspec_runner_fallback_spec.rb

@@ -0,0 +1,137 @@
+require 'open3'
+require 'ostruct'
+require 'tempfile'
+
+describe "#{KnapsackPro::Runners::Queue::RSpecRunner} Fallback - Integration tests" do
+  def log(stdout, stderr, status)
+    puts '='*50
+    puts 'STDOUT:'
+    puts stdout
+    puts
+
+    puts '='*50
+    puts 'STDERR:'
+    puts stderr
+    puts
+
+    puts '='*50
+    puts 'Exit status code:'
+    puts status
+    puts
+  end
+
+  subject do
+    stdout, stderr, status = Open3.capture3("ruby #{@task.path}")
+    log(stdout, stderr, status) if ENV['TEST__SHOW_DEBUG_LOG'] == 'true'
+    OpenStruct.new(stdout: stdout, stderr: stderr, exit_code: status.exitstatus)
+  end
+
+  before(:each) do
+    ENV['KNAPSACK_PRO_ENDPOINT'] = 'https://fail.knapsackpro.com' # ensure the API is not reachable
+    ENV['KNAPSACK_PRO_MAX_REQUEST_RETRIES'] = '1' # only try once to reach the API
+  end
+
+  around(:each) do |example|
+    Tempfile.create do |file|
+      file.write(<<~CONTENT)
+        require 'knapsack_pro'
+
+        ENV['KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC'] = SecureRandom.hex
+        ENV['KNAPSACK_PRO_CI_NODE_BUILD_ID'] = SecureRandom.uuid
+
+        #{KnapsackPro::Runners::Queue::RSpecRunner}.run('')
+      CONTENT
+      file.rewind
+
+      @task = file
+      example.run
+    end
+  end
+
+  after(:each) do
+    ENV.delete('KNAPSACK_PRO_ENDPOINT')
+    ENV.delete('KNAPSACK_PRO_MAX_REQUEST_RETRIES')
+  end
+
+  context 'with fallback mode disabled' do
+    before(:each) do
+      ENV['KNAPSACK_PRO_FALLBACK_MODE_ENABLED'] = 'false'
+    end
+
+    after(:each) do
+      ENV.delete('KNAPSACK_PRO_FALLBACK_MODE_ENABLED')
+    end
+
+    it 'exits with 1 and logs error and summary' do
+      actual = subject
+
+      expect(actual.exit_code).to eq 1
+
+      expect(actual.stdout).to include('ERROR -- : [knapsack_pro] Fallback Mode was disabled with KNAPSACK_PRO_FALLBACK_MODE_ENABLED=false. Please restart this CI node to retry tests. Most likely Fallback Mode was disabled due to https://knapsackpro.com/perma/ruby/queue-mode-connection-error-with-fallback-enabled-false')
+      expect(actual.stdout).to include(/Finished in .* seconds \(files took .* seconds to load\)/)
+      expect(actual.stdout).to include('0 examples, 0 failures')
+    end
+
+    context 'with a user defined exit code' do
+      before do
+        ENV['KNAPSACK_PRO_FALLBACK_MODE_ERROR_EXIT_CODE'] = '123'
+      end
+
+      after do
+        ENV.delete('KNAPSACK_PRO_FALLBACK_MODE_ERROR_EXIT_CODE')
+      end
+
+      it 'exits with the passed exit code and logs error and summary' do
+        actual = subject
+
+        expect(actual.exit_code).to eq 123
+
+        expect(actual.stdout).to include('ERROR -- : [knapsack_pro] Fallback Mode was disabled with KNAPSACK_PRO_FALLBACK_MODE_ENABLED=false. Please restart this CI node to retry tests. Most likely Fallback Mode was disabled due to https://knapsackpro.com/perma/ruby/queue-mode-connection-error-with-fallback-enabled-false')
+        expect(actual.stdout).to include(/Finished in .* seconds \(files took .* seconds to load\)/)
+        expect(actual.stdout).to include('0 examples, 0 failures')
+      end
+    end
+  end
+
+  context 'with fallback mode enabled and positive node retry count' do
+    before(:each) do
+      ENV['KNAPSACK_PRO_FALLBACK_MODE_ENABLED'] = 'true'
+      ENV['KNAPSACK_PRO_CI_NODE_RETRY_COUNT'] = '1'
+    end
+
+    after(:each) do
+      ENV.delete('KNAPSACK_PRO_FALLBACK_MODE_ENABLED')
+      ENV.delete('KNAPSACK_PRO_CI_NODE_RETRY_COUNT')
+    end
+
+    it 'exits with 1 and logs error and summary' do
+      actual = subject
+
+      expect(actual.exit_code).to eq 1
+
+      expect(actual.stdout).to include('ERROR -- : [knapsack_pro] knapsack_pro gem could not connect to Knapsack Pro API and the Fallback Mode cannot be used this time. Running tests in Fallback Mode are not allowed for retried parallel CI node to avoid running the wrong set of tests. Please manually retry this parallel job on your CI server then knapsack_pro gem will try to connect to Knapsack Pro API again and will run a correct set of tests for this CI node. Learn more https://knapsackpro.com/perma/ruby/queue-mode-connection-error-with-fallback-enabled-true-and-positive-retry-count')
+      expect(actual.stdout).to include(/Finished in .* seconds \(files took .* seconds to load\)/)
+      expect(actual.stdout).to include('0 examples, 0 failures')
+    end
+
+    context 'with a user defined exit code' do
+      before do
+        ENV['KNAPSACK_PRO_FALLBACK_MODE_ERROR_EXIT_CODE'] = '123'
+      end
+
+      after do
+        ENV.delete('KNAPSACK_PRO_FALLBACK_MODE_ERROR_EXIT_CODE')
+      end
+
+      it 'exits with the passed exit code and logs error and summary' do
+        actual = subject
+
+        expect(actual.exit_code).to eq 123
+
+        expect(actual.stdout).to include('ERROR -- : [knapsack_pro] knapsack_pro gem could not connect to Knapsack Pro API and the Fallback Mode cannot be used this time. Running tests in Fallback Mode are not allowed for retried parallel CI node to avoid running the wrong set of tests. Please manually retry this parallel job on your CI server then knapsack_pro gem will try to connect to Knapsack Pro API again and will run a correct set of tests for this CI node. Learn more https://knapsackpro.com/perma/ruby/queue-mode-connection-error-with-fallback-enabled-true-and-positive-retry-count')
+        expect(actual.stdout).to include(/Finished in .* seconds \(files took .* seconds to load\)/)
+        expect(actual.stdout).to include('0 examples, 0 failures')
+      end
+    end
+  end
+end

data/spec/integration/runners/queue/rspec_runner_spec.rb

@@ -53,9 +53,7 @@ describe "#{KnapsackPro::Runners::Queue::RSpecRunner} - Integration tests", :cle
     ENV['TEST__TEST_FILE_CASES_FOR_SLOW_TEST_FILES'] = test_file_paths.to_json
   end
 
-  def log_command_result(stdout, stderr, status)
-    return if ENV['TEST__SHOW_DEBUG_LOG'] != 'true'
-
+  def log(stdout, stderr, status)
     puts '='*50
     puts 'STDOUT:'
     puts stdout
@@ -83,7 +81,7 @@ describe "#{KnapsackPro::Runners::Queue::RSpecRunner} - Integration tests", :cle
 
   subject do
     stdout, stderr, status = Open3.capture3(command)
-    log_command_result(stdout, stderr, status)
+    log(stdout, stderr, status) if ENV['TEST__SHOW_DEBUG_LOG'] == 'true'
     OpenStruct.new(stdout: stdout, stderr: stderr, exit_code: status.exitstatus)
   end
 

data/spec/knapsack_pro/allocator_spec.rb

@@ -23,43 +23,63 @@ describe KnapsackPro::Allocator do
 
     shared_examples_for 'when connection to API failed (fallback mode)' do
       context 'when fallback mode is disabled' do
-        before do
+        before(:each) do
           expect(KnapsackPro::Config::Env).to receive(:fallback_mode_enabled?).and_return(false)
         end
 
-        it do
-          expect { subject }.to raise_error(RuntimeError, 'Fallback Mode was disabled with KNAPSACK_PRO_FALLBACK_MODE_ENABLED=false. Please restart this CI node to retry tests. Most likely Fallback Mode was disabled due to https://knapsackpro.com/perma/ruby/regular-mode-connection-error-with-fallback-enabled-false')
+        it 'logs the error and exits' do
+          logger = instance_double(Logger)
+          allow(KnapsackPro).to receive(:logger).and_return(logger)
+          expect(logger).to receive(:error).with(
+            'Fallback Mode was disabled with KNAPSACK_PRO_FALLBACK_MODE_ENABLED=false. Please restart this CI node to retry tests. Most likely Fallback Mode was disabled due to https://knapsackpro.com/perma/ruby/regular-mode-connection-error-with-fallback-enabled-false'
+          )
+
+          expect { subject }.to raise_error(SystemExit)
         end
       end
 
       context 'when CI node retry count > 0' do
-        before do
+        before(:each) do
           expect(KnapsackPro::Config::Env).to receive(:ci_node_retry_count).and_return(1)
         end
 
         context 'when fixed_test_suite_split=true' do
-          before do
+          before(:each) do
             expect(KnapsackPro::Config::Env).to receive(:fixed_test_suite_split).and_return(true)
           end
 
-          it do
-            expect { subject }.to raise_error(RuntimeError, 'knapsack_pro gem could not connect to Knapsack Pro API and the Fallback Mode cannot be used this time. Running tests in Fallback Mode are not allowed for retried parallel CI node to avoid running the wrong set of tests. Please manually retry this parallel job on your CI server then knapsack_pro gem will try to connect to Knapsack Pro API again and will run a correct set of tests for this CI node. Learn more https://knapsackpro.com/perma/ruby/regular-mode-connection-error-with-fallback-enabled-true-and-positive-retry-count')
+          it 'logs the error and exits' do
+            logger = instance_double(Logger)
+            allow(KnapsackPro).to receive(:logger).and_return(logger)
+
+            expect(logger).to receive(:error).with(
+              'knapsack_pro gem could not connect to Knapsack Pro API and the Fallback Mode cannot be used this time. Running tests in Fallback Mode are not allowed for retried parallel CI node to avoid running the wrong set of tests. Please manually retry this parallel job on your CI server then knapsack_pro gem will try to connect to Knapsack Pro API again and will run a correct set of tests for this CI node. Learn more https://knapsackpro.com/perma/ruby/regular-mode-connection-error-with-fallback-enabled-true-and-positive-retry-count'
+            )
+
+            expect { subject }.to raise_error(SystemExit)
           end
         end
 
         context 'when fixed_test_suite_split=false' do
-          before do
+          before(:each) do
             expect(KnapsackPro::Config::Env).to receive(:fixed_test_suite_split).and_return(false)
           end
 
-          it do
-            expect { subject }.to raise_error(RuntimeError, 'knapsack_pro gem could not connect to Knapsack Pro API and the Fallback Mode cannot be used this time. Running tests in Fallback Mode are not allowed for retried parallel CI node to avoid running the wrong set of tests. Please manually retry this parallel job on your CI server then knapsack_pro gem will try to connect to Knapsack Pro API again and will run a correct set of tests for this CI node. Learn more https://knapsackpro.com/perma/ruby/regular-mode-connection-error-with-fallback-enabled-true-and-positive-retry-count Please ensure you have set KNAPSACK_PRO_FIXED_TEST_SUITE_SPLIT=true to allow Knapsack Pro API remember the recorded CI node tests so when you retry failed tests on the CI node then the same set of tests will be executed. See more https://knapsackpro.com/perma/ruby/fixed-test-suite-split')
+          it 'logs the error and exits' do
+            logger = instance_double(Logger)
+            allow(KnapsackPro).to receive(:logger).and_return(logger)
+
+            expect(logger).to receive(:error).with(
+              'knapsack_pro gem could not connect to Knapsack Pro API and the Fallback Mode cannot be used this time. Running tests in Fallback Mode are not allowed for retried parallel CI node to avoid running the wrong set of tests. Please manually retry this parallel job on your CI server then knapsack_pro gem will try to connect to Knapsack Pro API again and will run a correct set of tests for this CI node. Learn more https://knapsackpro.com/perma/ruby/regular-mode-connection-error-with-fallback-enabled-true-and-positive-retry-count Please ensure you have set KNAPSACK_PRO_FIXED_TEST_SUITE_SPLIT=true to allow Knapsack Pro API remember the recorded CI node tests so when you retry failed tests on the CI node then the same set of tests will be executed. See more https://knapsackpro.com/perma/ruby/fixed-test-suite-split'
+            )
+
+            expect { subject }.to raise_error(SystemExit)
           end
         end
       end
 
       context 'when fallback mode started' do
-        before do
+        before(:each) do
           test_flat_distributor = instance_double(KnapsackPro::TestFlatDistributor)
           expect(KnapsackPro::TestFlatDistributor).to receive(:new).with(fallback_mode_test_files, ci_node_total).and_return(test_flat_distributor)
           expect(test_flat_distributor).to receive(:test_files_for_node).with(ci_node_index).and_return([
@@ -72,7 +92,7 @@ describe KnapsackPro::Allocator do
       end
     end
 
-    before do
+    before(:each) do
       encrypted_branch = double
       expect(KnapsackPro::Crypto::BranchEncryptor).to receive(:call).with(repository_adapter.branch).and_return(encrypted_branch)
 
@@ -118,7 +138,7 @@ describe KnapsackPro::Allocator do
             }
           end
 
-          before do
+          before(:each) do
             expect(KnapsackPro::Crypto::Decryptor).to receive(:call).with(fast_and_slow_test_files_to_run, response['test_files']).and_call_original
           end
 
@@ -131,7 +151,7 @@ describe KnapsackPro::Allocator do
           end
           let(:api_code) { 'TEST_SUITE_SPLIT_CACHE_MISS' }
 
-          before do
+          before(:each) do
             encrypted_branch = double
             expect(KnapsackPro::Crypto::BranchEncryptor).to receive(:call).with(repository_adapter.branch).and_return(encrypted_branch)
 
@@ -183,7 +203,7 @@ describe KnapsackPro::Allocator do
                   }
                 end
 
-                before do
+                before(:each) do
                   expect(KnapsackPro::Crypto::Decryptor).to receive(:call).with(fast_and_slow_test_files_to_run, response2['test_files']).and_call_original
                 end
 

data/spec/knapsack_pro/config/env_spec.rb

@@ -1016,9 +1016,8 @@ describe KnapsackPro::Config::Env do
   describe '.rspec_split_by_test_examples?' do
     subject { described_class.rspec_split_by_test_examples? }
 
-    before do
+    after(:each) do
       described_class.remove_instance_variable(:@rspec_split_by_test_examples)
-    rescue NameError
     end
 
     context 'when KNAPSACK_PRO_RSPEC_SPLIT_BY_TEST_EXAMPLES=true AND KNAPSACK_PRO_CI_NODE_TOTAL >= 2' do

data/spec/knapsack_pro/queue_allocator_spec.rb

@@ -31,7 +31,7 @@ describe KnapsackPro::QueueAllocator do
         end
 
         it do
-          expect { subject }.to raise_error(RuntimeError, 'Fallback Mode was disabled with KNAPSACK_PRO_FALLBACK_MODE_ENABLED=false. Please restart this CI node to retry tests. Most likely Fallback Mode was disabled due to https://knapsackpro.com/perma/ruby/queue-mode-connection-error-with-fallback-enabled-false')
+          expect { subject }.to raise_error(KnapsackPro::QueueAllocator::FallbackModeError, 'Fallback Mode was disabled with KNAPSACK_PRO_FALLBACK_MODE_ENABLED=false. Please restart this CI node to retry tests. Most likely Fallback Mode was disabled due to https://knapsackpro.com/perma/ruby/queue-mode-connection-error-with-fallback-enabled-false')
         end
       end
 
@@ -46,7 +46,7 @@ describe KnapsackPro::QueueAllocator do
           end
 
           it do
-            expect { subject }.to raise_error(RuntimeError, 'knapsack_pro gem could not connect to Knapsack Pro API and the Fallback Mode cannot be used this time. Running tests in Fallback Mode are not allowed for retried parallel CI node to avoid running the wrong set of tests. Please manually retry this parallel job on your CI server then knapsack_pro gem will try to connect to Knapsack Pro API again and will run a correct set of tests for this CI node. Learn more https://knapsackpro.com/perma/ruby/queue-mode-connection-error-with-fallback-enabled-true-and-positive-retry-count')
+            expect { subject }.to raise_error(KnapsackPro::QueueAllocator::FallbackModeError, 'knapsack_pro gem could not connect to Knapsack Pro API and the Fallback Mode cannot be used this time. Running tests in Fallback Mode are not allowed for retried parallel CI node to avoid running the wrong set of tests. Please manually retry this parallel job on your CI server then knapsack_pro gem will try to connect to Knapsack Pro API again and will run a correct set of tests for this CI node. Learn more https://knapsackpro.com/perma/ruby/queue-mode-connection-error-with-fallback-enabled-true-and-positive-retry-count')
           end
         end
 
@@ -56,7 +56,7 @@ describe KnapsackPro::QueueAllocator do
           end
 
           it do
-            expect { subject }.to raise_error(RuntimeError, 'knapsack_pro gem could not connect to Knapsack Pro API and the Fallback Mode cannot be used this time. Running tests in Fallback Mode are not allowed for retried parallel CI node to avoid running the wrong set of tests. Please manually retry this parallel job on your CI server then knapsack_pro gem will try to connect to Knapsack Pro API again and will run a correct set of tests for this CI node. Learn more https://knapsackpro.com/perma/ruby/queue-mode-connection-error-with-fallback-enabled-true-and-positive-retry-count Please ensure you have set KNAPSACK_PRO_FIXED_QUEUE_SPLIT=true to allow Knapsack Pro API remember the recorded CI node tests so when you retry failed tests on the CI node then the same set of tests will be executed. See more https://knapsackpro.com/perma/ruby/fixed-queue-split')
+            expect { subject }.to raise_error(KnapsackPro::QueueAllocator::FallbackModeError, 'knapsack_pro gem could not connect to Knapsack Pro API and the Fallback Mode cannot be used this time. Running tests in Fallback Mode are not allowed for retried parallel CI node to avoid running the wrong set of tests. Please manually retry this parallel job on your CI server then knapsack_pro gem will try to connect to Knapsack Pro API again and will run a correct set of tests for this CI node. Learn more https://knapsackpro.com/perma/ruby/queue-mode-connection-error-with-fallback-enabled-true-and-positive-retry-count Please ensure you have set KNAPSACK_PRO_FIXED_QUEUE_SPLIT=true to allow Knapsack Pro API remember the recorded CI node tests so when you retry failed tests on the CI node then the same set of tests will be executed. See more https://knapsackpro.com/perma/ruby/fixed-queue-split')
           end
         end
       end

data/spec/knapsack_pro/utils_spec.rb

@@ -23,7 +23,7 @@ describe KnapsackPro::Utils do
     context 'when Timecop does not mock the time' do
       it do
         now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-        expect(subject).to be_within(0.001).of(now)
+        expect(subject).to be_within(0.01).of(now)
       end
     end
 
@@ -47,7 +47,7 @@ describe KnapsackPro::Utils do
 
         time = Time.local(2020, 1, 31)
         Timecop.travel(time) do
-          expect(subject).to be_within(0.001).of(now)
+          expect(subject).to be_within(0.01).of(now)
         end
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: knapsack_pro
 version: !ruby/object:Gem::Version
-  version: 7.12.1
+  version: 7.13.0
 platform: ruby
 authors:
 - ArturT
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-11-01 00:00:00.000000000 Z
+date: 2024-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -318,7 +318,11 @@ files:
 - spec/fixtures/vcr_cassettes/api/v1/build_subsets/create/success.yml
 - spec/integration/api/build_distributions_subset_spec.rb
 - spec/integration/api/build_subsets_create_spec.rb
+- spec/integration/runners/fallback_spec.rb
+- spec/integration/runners/queue/cucumber_runner_fallback_spec.rb
+- spec/integration/runners/queue/minitest_runner_fallback_spec.rb
 - spec/integration/runners/queue/rspec_runner.rb
+- spec/integration/runners/queue/rspec_runner_fallback_spec.rb
 - spec/integration/runners/queue/rspec_runner_spec.rb
 - spec/knapsack_pro/adapters/base_adapter_spec.rb
 - spec/knapsack_pro/adapters/cucumber_adapter_spec.rb
@@ -429,7 +433,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.11
+rubygems_version: 3.5.16
 signing_key:
 specification_version: 4
 summary: Knapsack Pro splits tests across parallel CI nodes and ensures each parallel
@@ -442,7 +446,11 @@ test_files:
 - spec/fixtures/vcr_cassettes/api/v1/build_subsets/create/success.yml
 - spec/integration/api/build_distributions_subset_spec.rb
 - spec/integration/api/build_subsets_create_spec.rb
+- spec/integration/runners/fallback_spec.rb
+- spec/integration/runners/queue/cucumber_runner_fallback_spec.rb
+- spec/integration/runners/queue/minitest_runner_fallback_spec.rb
 - spec/integration/runners/queue/rspec_runner.rb
+- spec/integration/runners/queue/rspec_runner_fallback_spec.rb
 - spec/integration/runners/queue/rspec_runner_spec.rb
 - spec/knapsack_pro/adapters/base_adapter_spec.rb
 - spec/knapsack_pro/adapters/cucumber_adapter_spec.rb