knapsack_pro 1.17.0 → 1.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 86141c7add4d4159437103ebcb72460961d1a2b7f69493f6c517bcd3160785a1
-  data.tar.gz: 257c9d52f6d1a43c1f92ad0e48eda4d3ec5753d684068a63ea19dcb46484f792
+  metadata.gz: 030ea46e8f4a421824a682b4286f7e362d0eec9a3bc50cd113e703391a51e40f
+  data.tar.gz: 47ab9de263fce5a5576ac2d1054fc66ea15b7982f01595b3ac237534deede7cf
 SHA512:
-  metadata.gz: 2ebc1c14cb430ca87497d946bfeb20d4a2c47f9c1f563663aeb0062192c453de84ab1455a6e109f870eed4a44e8b83a2757391e48c3338feebeccb01cce6ef3e
-  data.tar.gz: f407c9d6566e5cfe73a0198519e513a4d7aa1e55b2371bc3ca1aad6a5818526d16f41b8a1a69a50b3c4277d0272862ec5e59a9e8d5577597222c9d048ae6bb02
+  metadata.gz: c94a8c86cd5114919737a385c792e564d373d82d0f9e195d18c45c043cf28d7ffc8e90bdab81d8d02ffd896aa1761b895c06d60d0dc8b232daffa347e73e2c06
+  data.tar.gz: 2f920d5b94aa829b975d3385e70df37a0817bc1b88962fc34b017cb3095d3b6cecd6e6c4cb0e2c69bb9643962c40b340ffbc8ba780558012949608296a17c3c0

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Change Log
 
+### 1.18.0
+
+* __IMPORTANT__ Do not allow Fallback Mode when the CI node was retried to avoid running the wrong set of tests
+
+    Please read the PR description if you are using retry failed CI node feature on your CI (for instance you use Buildkite).
+
+    https://github.com/KnapsackPro/knapsack_pro-ruby/pull/100
+
+* Increase delay between request retry to Knapsack Pro API from 2s to 4s for 2nd request and from 4s to 8s for 3rd request
+
+    https://github.com/KnapsackPro/knapsack_pro-ruby/pull/99
+
+https://github.com/KnapsackPro/knapsack_pro-ruby/compare/v1.17.0...v1.18.0
+
 ### 1.17.0
 
 * Add `KNAPSACK_PRO_CUCUMBER_QUEUE_PREFIX` to allow run Cucumber with spring gem in Queue Mode

data/README.md

@@ -84,8 +84,9 @@ We keep this old FAQ in README to not break old links spread across the web. You
   - [Supported test runners in queue mode](#supported-test-runners-in-queue-mode)
 - [Extra configuration for CI server](#extra-configuration-for-ci-server)
   - [Info about ENV variables](#info-about-env-variables)
-    - [KNAPSACK_PRO_FIXED_TEST_SUITE_SPLITE (test suite split based on seed)](#knapsack_pro_fixed_test_suite_splite-test-suite-split-based-on-seed)
+    - [KNAPSACK_PRO_FIXED_TEST_SUITE_SPLIT (test suite split based on seed)](#knapsack_pro_fixed_test_suite_split-test-suite-split-based-on-seed)
     - [Environment variables for debugging gem](#environment-variables-for-debugging-gem)
+  - [Required CI configuration if you use retry single failed CI node feature on your CI server when KNAPSACK_PRO_FIXED_QUEUE_SPLIT=true (in Queue Mode) or KNAPSACK_PRO_FIXED_TEST_SUITE_SPLIT=true (in Regular Mode)](#required-ci-configuration-if-you-use-retry-single-failed-ci-node-feature-on-your-ci-server-when-knapsack_pro_fixed_queue_splittrue-in-queue-mode-or-knapsack_pro_fixed_test_suite_splittrue-in-regular-mode)
   - [Passing arguments to rake task](#passing-arguments-to-rake-task)
     - [Passing arguments to rspec](#passing-arguments-to-rspec)
     - [Passing arguments to cucumber](#passing-arguments-to-cucumber)
@@ -551,6 +552,10 @@ There might be some cached test suite splits for git commits you have run in pas
 
   Thanks to that when tests on one of your node failed you can retry the node with exactly the same subset of tests that were run on the node in the first place.
 
+  __IMPORTANT__: [Required CI configuration if you use retry single failed CI node feature on your CI server when KNAPSACK_PRO_FIXED_QUEUE_SPLIT=true (in Queue Mode) or KNAPSACK_PRO_FIXED_TEST_SUITE_SPLIT=true (in Regular Mode)](#required-ci-configuration-if-you-use-retry-single-failed-ci-node-feature-on-your-ci-server-when-knapsack_pro_fixed_queue_splittrue-in-queue-mode-or-knapsack_pro_fixed_test_suite_splittrue-in-regular-mode)
+
+  Other useful info:
+
   * Note when fixed queue split is enabled then you can run tests in a dynamic way only once for particular commit hash and a total number of nodes and for the same branch.
 
   * When Knapsack Pro API server has already information about previous queue split then the information will be used. You will see at the beginning of the knapsack command the log with info that queue name is nil because it was not generated this time. You will get the list of all test files that were executed on the particular CI node in the past.
@@ -598,7 +603,7 @@ In case when you use other CI provider for instance [Jenkins](https://jenkins-ci
 
 `KNAPSACK_PRO_CI_NODE_INDEX` - index of current CI node starts from 0. Second CI node should have `KNAPSACK_PRO_CI_NODE_INDEX=1`.
 
-#### KNAPSACK_PRO_FIXED_TEST_SUITE_SPLITE (test suite split based on seed)
+#### KNAPSACK_PRO_FIXED_TEST_SUITE_SPLIT (test suite split based on seed)
 
 Note this is for knapsack_pro regular mode only.
 
@@ -609,13 +614,19 @@ Note this is for knapsack_pro regular mode only.
 
     Thanks to that when tests on one of your node failed you can retry the node with exactly the same subset of tests that were run on the node in the first place.
 
-    There is one edge case. When you run tests for the first time and there is no data collected about time execution of your tests then
-    we need to collect data to prepare the first test suite split. The second run of your tests will have fixed test suite split.
+    __IMPORTANT__: [Required CI configuration if you use retry single failed CI node feature on your CI server when KNAPSACK_PRO_FIXED_QUEUE_SPLIT=true (in Queue Mode) or KNAPSACK_PRO_FIXED_TEST_SUITE_SPLIT=true (in Regular Mode)](#required-ci-configuration-if-you-use-retry-single-failed-ci-node-feature-on-your-ci-server-when-knapsack_pro_fixed_queue_splittrue-in-queue-mode-or-knapsack_pro_fixed_test_suite_splittrue-in-regular-mode)
+
+    Other useful info:
 
-    To compare if all your CI nodes are running based on the same test suite split seed you can check the value for seed in knapsack logging message
-    before your test starts. The message looks like:
+    * There is one edge case. When you run tests for the first time and there is no data collected about time execution of your tests then
+      we need to collect data to prepare the first test suite split. The second run of your tests will have fixed test suite split.
 
-        [knapsack_pro] Test suite split seed: 8a606431-02a1-4766-9878-0ea42a07ad21
+      To compare if all your CI nodes are running based on the same test suite split seed you can check the value for seed in knapsack logging message
+      before your test starts. The message looks like:
+
+      ```
+      [knapsack_pro] Test suite split seed: 8a606431-02a1-4766-9878-0ea42a07ad21
+      ```
 
 * `KNAPSACK_PRO_FIXED_TEST_SUITE_SPLIT=false`
 
@@ -642,6 +653,22 @@ This is only for maintainer of knapsack_pro gem. Not for the end users.
   * When mode is `development` then endpoint is `http://api.knapsackpro.test:3000`.
   * When mode is `test` then endpoint is `https://api-staging.knapsackpro.com`.
 
+### Required CI configuration if you use retry single failed CI node feature on your CI server when KNAPSACK_PRO_FIXED_QUEUE_SPLIT=true (in Queue Mode) or KNAPSACK_PRO_FIXED_TEST_SUITE_SPLIT=true (in Regular Mode)
+
+Read below required configuration step if you use Queue Mode and you set [`KNAPSACK_PRO_FIXED_QUEUE_SPLIT=true`](#knapsack_pro_fixed_queue_split-remember-queue-split-on-retry-ci-node) or you use Regular Mode which has by default [`KNAPSACK_PRO_FIXED_TEST_SUITE_SPLIT=true`](#knapsack_pro_fixed_test_suite_splite-test-suite-split-based-on-seed).
+
+* __IMPORTANT:__ If you use __the feature to retry only a single failed CI node__ on your CI server (for instance you use Buildkite and you use [auto-retry](https://buildkite.com/docs/pipelines/command-step#retry-attributes) for the failed job) then you need to be aware of [a race condition that could happen](https://github.com/KnapsackPro/knapsack_pro-ruby/pull/100). knapsack_pro should not allow running tests in Fallback Mode in the case when the failed CI node was retried to prevent running the wrong set of tests.
+
+  knapsack_pro has built-in support for retries of failed parallel CI nodes for listed CI servers:
+
+  * Buildkite (knapsack_pro reads `BUILDKITE_RETRY_COUNT`)
+
+  knapsack_pro reads ENV vars for above CI servers and it disables Fallback Mode when failed parallel CI node can't connect with Knapsack Pro API. This way we prevent running the wrong set of tests by Fallback Mode on retried CI node.
+
+  If you use other CI server you need to manually configure your CI server to set `KNAPSACK_PRO_CI_NODE_RETRY_COUNT=1` only during retry CI node attempt. If `KNAPSACK_PRO_CI_NODE_RETRY_COUNT > 0` then knapsack_pro won't allow starting running tests in Fallback Mode and instead will raise error so a user can manually retry CI node later when a connection to Knapsack Pro API can be established.
+
+  If you cannot set `KNAPSACK_PRO_CI_NODE_RETRY_COUNT` only for retried CI node or it is not possible for your CI server then you can disable Fallback Mode completely `KNAPSACK_PRO_FALLBACK_MODE_ENABLED=false`.
+
 ### Passing arguments to rake task
 
 #### Passing arguments to rspec

data/lib/knapsack_pro/allocator.rb

@@ -13,6 +13,17 @@ module KnapsackPro
       if connection.success?
         raise ArgumentError.new(response) if connection.errors?
         prepare_test_files(response)
+      elsif !KnapsackPro::Config::Env.fallback_mode_enabled?
+        message = 'Fallback Mode was disabled with KNAPSACK_PRO_FALLBACK_MODE_ENABLED=false. Please restart this CI node to retry tests. Most likely Fallback Mode was disabled due to https://github.com/KnapsackPro/knapsack_pro-ruby#required-ci-configuration-if-you-use-retry-single-failed-ci-node-feature-on-your-ci-server-when-knapsack_pro_fixed_queue_splittrue-in-queue-mode-or-knapsack_pro_fixed_test_suite_splittrue-in-regular-mode'
+        KnapsackPro.logger.error(message)
+        raise message
+      elsif KnapsackPro::Config::Env.ci_node_retry_count > 0
+        message = 'knapsack_pro gem could not connect to Knapsack Pro API and the Fallback Mode cannot be used this time. Running tests in Fallback Mode are not allowed for retried parallel CI node to avoid running the wrong set of tests. Please manually retry this parallel job on your CI server then knapsack_pro gem will try to connect to Knapsack Pro API again and will run a correct set of tests for this CI node. Learn more https://github.com/KnapsackPro/knapsack_pro-ruby#required-ci-configuration-if-you-use-retry-single-failed-ci-node-feature-on-your-ci-server-when-knapsack_pro_fixed_queue_splittrue-in-queue-mode-or-knapsack_pro_fixed_test_suite_splittrue-in-regular-mode'
+        unless KnapsackPro::Config::Env.fixed_test_suite_split?
+          message += ' Please ensure you have set KNAPSACK_PRO_FIXED_TEST_SUITE_SPLIT=true to allow Knapsack Pro API remember the recorded CI node tests so when you retry failed tests on the CI node then the same set of tests will be executed. See more https://github.com/KnapsackPro/knapsack_pro-ruby#knapsack_pro_fixed_test_suite_split-test-suite-split-based-on-seed'
+        end
+        KnapsackPro.logger.error(message)
+        raise message
       else
         KnapsackPro.logger.warn("Fallback mode started. We could not connect with Knapsack Pro API. Your tests will be executed based on directory names. Read more about fallback mode at https://github.com/KnapsackPro/knapsack_pro-ruby#what-happens-when-knapsack-pro-api-is-not-availablenot-reachable-temporarily")
         fallback_test_files

data/lib/knapsack_pro/client/connection.rb

@@ -5,7 +5,7 @@ module KnapsackPro
 
       TIMEOUT = 15
       MAX_RETRY = 3
-      REQUEST_RETRY_TIMEBOX = 2
+      REQUEST_RETRY_TIMEBOX = 4
 
       def initialize(action)
         @action = action

data/lib/knapsack_pro/config/ci/base.rb

@@ -11,6 +11,9 @@ module KnapsackPro
         def node_build_id
         end
 
+        def node_retry_count
+        end
+
         def commit_hash
         end
 

data/lib/knapsack_pro/config/ci/buildkite.rb

@@ -14,6 +14,10 @@ module KnapsackPro
           ENV['BUILDKITE_BUILD_NUMBER']
         end
 
+        def node_retry_count
+          ENV['BUILDKITE_RETRY_COUNT']
+        end
+
         def commit_hash
           ENV['BUILDKITE_COMMIT']
         end

data/lib/knapsack_pro/config/env.rb

@@ -28,6 +28,14 @@ module KnapsackPro
             'missing-build-id'
         end
 
+        def ci_node_retry_count
+          (
+            ENV['KNAPSACK_PRO_CI_NODE_RETRY_COUNT'] ||
+            ci_env_for(:node_retry_count) ||
+            0
+          ).to_i
+        end
+
         def commit_hash
           ENV['KNAPSACK_PRO_COMMIT_HASH'] ||
             ci_env_for(:commit_hash)
@@ -87,6 +95,14 @@ module KnapsackPro
           ENV['KNAPSACK_PRO_SUBSET_QUEUE_ID'] || raise('Missing Subset Queue ID')
         end
 
+        def fallback_mode_enabled
+          ENV.fetch('KNAPSACK_PRO_FALLBACK_MODE_ENABLED', true)
+        end
+
+        def fallback_mode_enabled?
+          fallback_mode_enabled.to_s == 'true'
+        end
+
         def test_files_encrypted
           ENV['KNAPSACK_PRO_TEST_FILES_ENCRYPTED']
         end
@@ -135,10 +151,18 @@ module KnapsackPro
           ENV.fetch('KNAPSACK_PRO_FIXED_TEST_SUITE_SPLIT', true)
         end
 
+        def fixed_test_suite_split?
+          fixed_test_suite_split.to_s == 'true'
+        end
+
         def fixed_queue_split
           ENV.fetch('KNAPSACK_PRO_FIXED_QUEUE_SPLIT', false)
         end
 
+        def fixed_queue_split?
+          fixed_queue_split.to_s == 'true'
+        end
+
         def cucumber_queue_prefix
           ENV.fetch('KNAPSACK_PRO_CUCUMBER_QUEUE_PREFIX', 'bundle exec')
         end

data/lib/knapsack_pro/queue_allocator.rb

@@ -16,6 +16,17 @@ module KnapsackPro
       if connection.success?
         raise ArgumentError.new(response) if connection.errors?
         prepare_test_files(response)
+      elsif !KnapsackPro::Config::Env.fallback_mode_enabled?
+        message = 'Fallback Mode was disabled with KNAPSACK_PRO_FALLBACK_MODE_ENABLED=false. Please restart this CI node to retry tests. Most likely Fallback Mode was disabled due to https://github.com/KnapsackPro/knapsack_pro-ruby#required-ci-configuration-if-you-use-retry-single-failed-ci-node-feature-on-your-ci-server-when-knapsack_pro_fixed_queue_splittrue-in-queue-mode-or-knapsack_pro_fixed_test_suite_splittrue-in-regular-mode'
+        KnapsackPro.logger.error(message)
+        raise message
+      elsif KnapsackPro::Config::Env.ci_node_retry_count > 0
+        message = 'knapsack_pro gem could not connect to Knapsack Pro API and the Fallback Mode cannot be used this time. Running tests in Fallback Mode are not allowed for retried parallel CI node to avoid running the wrong set of tests. Please manually retry this parallel job on your CI server then knapsack_pro gem will try to connect to Knapsack Pro API again and will run a correct set of tests for this CI node. Learn more https://github.com/KnapsackPro/knapsack_pro-ruby#required-ci-configuration-if-you-use-retry-single-failed-ci-node-feature-on-your-ci-server-when-knapsack_pro_fixed_queue_splittrue-in-queue-mode-or-knapsack_pro_fixed_test_suite_splittrue-in-regular-mode'
+        unless KnapsackPro::Config::Env.fixed_queue_split?
+          message += ' Please ensure you have set KNAPSACK_PRO_FIXED_QUEUE_SPLIT=true to allow Knapsack Pro API remember the recorded CI node tests so when you retry failed tests on the CI node then the same set of tests will be executed. See more https://github.com/KnapsackPro/knapsack_pro-ruby#knapsack_pro_fixed_queue_split-remember-queue-split-on-retry-ci-node'
+        end
+        KnapsackPro.logger.error(message)
+        raise message
       else
         @fallback_activated = true
         KnapsackPro.logger.warn("Fallback mode started. We could not connect with Knapsack Pro API. Your tests will be executed based on directory names. If other CI nodes were able to connect with Knapsack Pro API then you may notice that some of the test files will be executed twice across CI nodes. The most important thing is to guarantee each of test files is run at least once! Read more about fallback mode at https://github.com/KnapsackPro/knapsack_pro-ruby#what-happens-when-knapsack-pro-api-is-not-availablenot-reachable-temporarily")

data/lib/knapsack_pro/version.rb

@@ -1,3 +1,3 @@
 module KnapsackPro
-  VERSION = '1.17.0'
+  VERSION = '1.18.0'
 end

data/spec/knapsack_pro/allocator_spec.rb

@@ -75,16 +75,54 @@ describe KnapsackPro::Allocator do
       let(:success?) { false }
       let(:errors?) { false }
 
-      before do
-        test_flat_distributor = instance_double(KnapsackPro::TestFlatDistributor)
-        expect(KnapsackPro::TestFlatDistributor).to receive(:new).with(test_files, ci_node_total).and_return(test_flat_distributor)
-        expect(test_flat_distributor).to receive(:test_files_for_node).with(ci_node_index).and_return([
-          { 'path' => 'c_spec.rb' },
-          { 'path' => 'd_spec.rb' },
-        ])
+      context 'when fallback mode is disabled' do
+        before do
+          expect(KnapsackPro::Config::Env).to receive(:fallback_mode_enabled?).and_return(false)
+        end
+
+        it do
+          expect { subject }.to raise_error(RuntimeError, 'Fallback Mode was disabled with KNAPSACK_PRO_FALLBACK_MODE_ENABLED=false. Please restart this CI node to retry tests. Most likely Fallback Mode was disabled due to https://github.com/KnapsackPro/knapsack_pro-ruby#required-ci-configuration-if-you-use-retry-single-failed-ci-node-feature-on-your-ci-server-when-knapsack_pro_fixed_queue_splittrue-in-queue-mode-or-knapsack_pro_fixed_test_suite_splittrue-in-regular-mode')
+        end
       end
 
-      it { should eq ['c_spec.rb', 'd_spec.rb'] }
+      context 'when CI node retry count > 0' do
+        before do
+          expect(KnapsackPro::Config::Env).to receive(:ci_node_retry_count).and_return(1)
+        end
+
+        context 'when fixed_test_suite_split=true' do
+          before do
+            expect(KnapsackPro::Config::Env).to receive(:fixed_test_suite_split).and_return(true)
+          end
+
+          it do
+            expect { subject }.to raise_error(RuntimeError, 'knapsack_pro gem could not connect to Knapsack Pro API and the Fallback Mode cannot be used this time. Running tests in Fallback Mode are not allowed for retried parallel CI node to avoid running the wrong set of tests. Please manually retry this parallel job on your CI server then knapsack_pro gem will try to connect to Knapsack Pro API again and will run a correct set of tests for this CI node. Learn more https://github.com/KnapsackPro/knapsack_pro-ruby#required-ci-configuration-if-you-use-retry-single-failed-ci-node-feature-on-your-ci-server-when-knapsack_pro_fixed_queue_splittrue-in-queue-mode-or-knapsack_pro_fixed_test_suite_splittrue-in-regular-mode')
+          end
+        end
+
+        context 'when fixed_test_suite_split=false' do
+          before do
+            expect(KnapsackPro::Config::Env).to receive(:fixed_test_suite_split).and_return(false)
+          end
+
+          it do
+            expect { subject }.to raise_error(RuntimeError, 'knapsack_pro gem could not connect to Knapsack Pro API and the Fallback Mode cannot be used this time. Running tests in Fallback Mode are not allowed for retried parallel CI node to avoid running the wrong set of tests. Please manually retry this parallel job on your CI server then knapsack_pro gem will try to connect to Knapsack Pro API again and will run a correct set of tests for this CI node. Learn more https://github.com/KnapsackPro/knapsack_pro-ruby#required-ci-configuration-if-you-use-retry-single-failed-ci-node-feature-on-your-ci-server-when-knapsack_pro_fixed_queue_splittrue-in-queue-mode-or-knapsack_pro_fixed_test_suite_splittrue-in-regular-mode Please ensure you have set KNAPSACK_PRO_FIXED_TEST_SUITE_SPLIT=true to allow Knapsack Pro API remember the recorded CI node tests so when you retry failed tests on the CI node then the same set of tests will be executed. See more https://github.com/KnapsackPro/knapsack_pro-ruby#knapsack_pro_fixed_test_suite_split-test-suite-split-based-on-seed')
+          end
+        end
+      end
+
+      context 'when fallback mode started' do
+        before do
+          test_flat_distributor = instance_double(KnapsackPro::TestFlatDistributor)
+          expect(KnapsackPro::TestFlatDistributor).to receive(:new).with(test_files, ci_node_total).and_return(test_flat_distributor)
+          expect(test_flat_distributor).to receive(:test_files_for_node).with(ci_node_index).and_return([
+            { 'path' => 'c_spec.rb' },
+            { 'path' => 'd_spec.rb' },
+          ])
+        end
+
+        it { should eq ['c_spec.rb', 'd_spec.rb'] }
+      end
     end
   end
 end

data/spec/knapsack_pro/client/connection_spec.rb

@@ -152,10 +152,10 @@ describe KnapsackPro::Client::Connection do
           server_error = described_class::ServerError.new(parsed_response)
           expect(logger).to receive(:warn).exactly(3).with(server_error.inspect)
 
-          expect(logger).to receive(:warn).with("Wait 2s and retry request to Knapsack Pro API.")
           expect(logger).to receive(:warn).with("Wait 4s and retry request to Knapsack Pro API.")
-          expect(Kernel).to receive(:sleep).with(2)
+          expect(logger).to receive(:warn).with("Wait 8s and retry request to Knapsack Pro API.")
           expect(Kernel).to receive(:sleep).with(4)
+          expect(Kernel).to receive(:sleep).with(8)
 
           expect(subject).to eq(parsed_response)
 

data/spec/knapsack_pro/config/ci/base_spec.rb

@@ -2,6 +2,7 @@ describe KnapsackPro::Config::CI::Base do
   its(:node_total) { should be nil }
   its(:node_index) { should be nil }
   its(:node_build_id) { should be nil }
+  its(:node_retry_count) { should be nil }
   its(:commit_hash) { should be nil }
   its(:branch) { should be nil }
   its(:project_dir) { should be nil }

data/spec/knapsack_pro/config/ci/buildkite_spec.rb

@@ -46,6 +46,19 @@ describe KnapsackPro::Config::CI::Buildkite do
     end
   end
 
+  describe '#node_retry_count' do
+    subject { described_class.new.node_retry_count }
+
+    context 'when environment exists' do
+      let(:env) { { 'BUILDKITE_RETRY_COUNT' => '1' } }
+      it { should eql '1' }
+    end
+
+    context "when environment doesn't exist" do
+      it { should be nil }
+    end
+  end
+
   describe '#commit_hash' do
     subject { described_class.new.commit_hash }
 

data/spec/knapsack_pro/config/env_spec.rb

@@ -77,6 +77,29 @@ describe KnapsackPro::Config::Env do
     end
   end
 
+  describe '.ci_node_retry_count' do
+    subject { described_class.ci_node_retry_count }
+
+    context 'when ENV exists' do
+      context 'when KNAPSACK_PRO_CI_NODE_RETRY_COUNT has value' do
+        before { stub_const("ENV", { 'KNAPSACK_PRO_CI_NODE_RETRY_COUNT' => '1' }) }
+        it { should eq 1 }
+      end
+
+      context 'when CI environment has value' do
+        before do
+          expect(described_class).to receive(:ci_env_for).with(:node_retry_count).and_return('2')
+        end
+
+        it { should eq 2 }
+      end
+    end
+
+    context "when ENV doesn't exist" do
+      it { should eq 0 }
+    end
+  end
+
   describe '.commit_hash' do
     subject { described_class.commit_hash }
 
@@ -317,6 +340,32 @@ describe KnapsackPro::Config::Env do
     end
   end
 
+  describe '.fallback_mode_enabled' do
+    subject { described_class.fallback_mode_enabled }
+
+    context 'when ENV exists' do
+      before { stub_const("ENV", { 'KNAPSACK_PRO_FALLBACK_MODE_ENABLED' => 'false' }) }
+      it { should eq 'false' }
+    end
+
+    context "when ENV doesn't exist" do
+      it { should be true }
+    end
+  end
+
+  describe '.fallback_mode_enabled?' do
+    subject { described_class.fallback_mode_enabled? }
+
+    context 'when ENV exists' do
+      before { stub_const("ENV", { 'KNAPSACK_PRO_FALLBACK_MODE_ENABLED' => 'false' }) }
+      it { should be false }
+    end
+
+    context "when ENV doesn't exist" do
+      it { should be true }
+    end
+  end
+
   describe '.test_files_encrypted' do
     subject { described_class.test_files_encrypted }
 
@@ -490,6 +539,27 @@ describe KnapsackPro::Config::Env do
     end
   end
 
+  describe '.fixed_test_suite_split?' do
+    subject { described_class.fixed_test_suite_split? }
+
+    context 'when ENV exists' do
+      context 'when KNAPSACK_PRO_FIXED_TEST_SUITE_SPLIT=true' do
+        before { stub_const("ENV", { 'KNAPSACK_PRO_FIXED_TEST_SUITE_SPLIT' => 'true' }) }
+        it { should be true }
+      end
+
+      context 'when KNAPSACK_PRO_FIXED_TEST_SUITE_SPLIT=false' do
+        before { stub_const("ENV", { 'KNAPSACK_PRO_FIXED_TEST_SUITE_SPLIT' => 'false' }) }
+        it { should be false }
+      end
+    end
+
+    context "when ENV doesn't exist" do
+      before { stub_const("ENV", {}) }
+      it { should be true }
+    end
+  end
+
   describe '.fixed_queue_split' do
     subject { described_class.fixed_queue_split }
 
@@ -504,6 +574,27 @@ describe KnapsackPro::Config::Env do
     end
   end
 
+  describe '.fixed_queue_split?' do
+    subject { described_class.fixed_queue_split? }
+
+    context 'when ENV exists' do
+      context 'when KNAPSACK_PRO_FIXED_QUEUE_SPLIT=true' do
+        before { stub_const("ENV", { 'KNAPSACK_PRO_FIXED_QUEUE_SPLIT' => 'true' }) }
+        it { should be true }
+      end
+
+      context 'when KNAPSACK_PRO_FIXED_QUEUE_SPLIT=false' do
+        before { stub_const("ENV", { 'KNAPSACK_PRO_FIXED_QUEUE_SPLIT' => 'false' }) }
+        it { should be false }
+      end
+    end
+
+    context "when ENV doesn't exist" do
+      before { stub_const("ENV", {}) }
+      it { should be false }
+    end
+  end
+
   describe '.cucumber_queue_prefix' do
     subject { described_class.cucumber_queue_prefix }
 

data/spec/knapsack_pro/queue_allocator_spec.rb

@@ -81,29 +81,66 @@ describe KnapsackPro::QueueAllocator do
       let(:success?) { false }
       let(:errors?) { false }
 
+      context 'when fallback mode is disabled' do
+        before do
+          expect(KnapsackPro::Config::Env).to receive(:fallback_mode_enabled?).and_return(false)
+        end
 
-      before do
-        test_flat_distributor = instance_double(KnapsackPro::TestFlatDistributor)
-        expect(KnapsackPro::TestFlatDistributor).to receive(:new).with(test_files, ci_node_total).and_return(test_flat_distributor)
-        expect(test_flat_distributor).to receive(:test_files_for_node).with(ci_node_index).and_return([
-          { 'path' => 'c_spec.rb' },
-          { 'path' => 'd_spec.rb' },
-        ])
+        it do
+          expect { subject }.to raise_error(RuntimeError, 'Fallback Mode was disabled with KNAPSACK_PRO_FALLBACK_MODE_ENABLED=false. Please restart this CI node to retry tests. Most likely Fallback Mode was disabled due to https://github.com/KnapsackPro/knapsack_pro-ruby#required-ci-configuration-if-you-use-retry-single-failed-ci-node-feature-on-your-ci-server-when-knapsack_pro_fixed_queue_splittrue-in-queue-mode-or-knapsack_pro_fixed_test_suite_splittrue-in-regular-mode')
+        end
       end
 
-      context 'when no test files were executed yet' do
-        let(:executed_test_files) { [] }
+      context 'when CI node retry count > 0' do
+        before do
+          expect(KnapsackPro::Config::Env).to receive(:ci_node_retry_count).and_return(1)
+        end
+
+        context 'when fixed_queue_split=true' do
+          before do
+            expect(KnapsackPro::Config::Env).to receive(:fixed_queue_split).and_return(true)
+          end
+
+          it do
+            expect { subject }.to raise_error(RuntimeError, 'knapsack_pro gem could not connect to Knapsack Pro API and the Fallback Mode cannot be used this time. Running tests in Fallback Mode are not allowed for retried parallel CI node to avoid running the wrong set of tests. Please manually retry this parallel job on your CI server then knapsack_pro gem will try to connect to Knapsack Pro API again and will run a correct set of tests for this CI node. Learn more https://github.com/KnapsackPro/knapsack_pro-ruby#required-ci-configuration-if-you-use-retry-single-failed-ci-node-feature-on-your-ci-server-when-knapsack_pro_fixed_queue_splittrue-in-queue-mode-or-knapsack_pro_fixed_test_suite_splittrue-in-regular-mode')
+          end
+        end
+
+        context 'when fixed_queue_split=false' do
+          before do
+            expect(KnapsackPro::Config::Env).to receive(:fixed_queue_split).and_return(false)
+          end
 
-        it 'enables fallback mode and returns fallback test files' do
-          expect(subject).to eq ['c_spec.rb', 'd_spec.rb']
+          it do
+            expect { subject }.to raise_error(RuntimeError, 'knapsack_pro gem could not connect to Knapsack Pro API and the Fallback Mode cannot be used this time. Running tests in Fallback Mode are not allowed for retried parallel CI node to avoid running the wrong set of tests. Please manually retry this parallel job on your CI server then knapsack_pro gem will try to connect to Knapsack Pro API again and will run a correct set of tests for this CI node. Learn more https://github.com/KnapsackPro/knapsack_pro-ruby#required-ci-configuration-if-you-use-retry-single-failed-ci-node-feature-on-your-ci-server-when-knapsack_pro_fixed_queue_splittrue-in-queue-mode-or-knapsack_pro_fixed_test_suite_splittrue-in-regular-mode Please ensure you have set KNAPSACK_PRO_FIXED_QUEUE_SPLIT=true to allow Knapsack Pro API remember the recorded CI node tests so when you retry failed tests on the CI node then the same set of tests will be executed. See more https://github.com/KnapsackPro/knapsack_pro-ruby#knapsack_pro_fixed_queue_split-remember-queue-split-on-retry-ci-node')
+          end
         end
       end
 
-      context 'when test files were already executed' do
-        let(:executed_test_files) { ['c_spec.rb', 'additional_executed_spec.rb'] }
+      context 'when fallback mode started' do
+        before do
+          test_flat_distributor = instance_double(KnapsackPro::TestFlatDistributor)
+          expect(KnapsackPro::TestFlatDistributor).to receive(:new).with(test_files, ci_node_total).and_return(test_flat_distributor)
+          expect(test_flat_distributor).to receive(:test_files_for_node).with(ci_node_index).and_return([
+            { 'path' => 'c_spec.rb' },
+            { 'path' => 'd_spec.rb' },
+          ])
+        end
+
+        context 'when no test files were executed yet' do
+          let(:executed_test_files) { [] }
+
+          it 'enables fallback mode and returns fallback test files' do
+            expect(subject).to eq ['c_spec.rb', 'd_spec.rb']
+          end
+        end
+
+        context 'when test files were already executed' do
+          let(:executed_test_files) { ['c_spec.rb', 'additional_executed_spec.rb'] }
 
-        it 'enables fallback mode and returns fallback test files' do
-          expect(subject).to eq ['d_spec.rb']
+          it 'enables fallback mode and returns fallback test files' do
+            expect(subject).to eq ['d_spec.rb']
+          end
         end
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: knapsack_pro
 version: !ruby/object:Gem::Version
-  version: 1.17.0
+  version: 1.18.0
 platform: ruby
 authors:
 - ArturT
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-06 00:00:00.000000000 Z
+date: 2020-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake