knapsack_pro 0.54.0 → 0.55.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5ca270139eec1a9a068a6038deaa4670812226c1663dd1874d4db0ea74815ce7
-  data.tar.gz: 96e8ed4266180dacd9e0be2c7b5b2f5cd05ec28537d895e7c6d88989fc8b98b0
+  metadata.gz: 0fcf3c3da71b36a383c225ec0eb45a1d9093a2c82389772b908e47d6b176fba4
+  data.tar.gz: 9d45db01a28ca6b54ae76061b0afb3100a9a8e9e70287b299eb5af47aa4a2a8c
 SHA512:
-  metadata.gz: 69b834636b9d84069f44e566fd6f6e4ed80f0a9a68202b57156421fd4d34b06ed6907429a3d59265f6c306bd7c537f3cd57e9d95b2a1e2421444970da7762385
-  data.tar.gz: 350e5c4346a395e00e7e5cc9dcbb0ba9c7f34143b17e4174cffea1b7b1ebde1517d6e02be40dde84103916165b080008df56209623b785e7c3b62317b8646e6b
+  metadata.gz: 4b974565bf8d3dbcce04d9e3f6b00663100b6f5c7012957e9e8030cb20f4e57efb6851a0ce6f3023c85e8727651d95f9a7aa48fb13e6f57797ba1ff2c427bbec
+  data.tar.gz: 7140a623c618602670eb69a04e7802da9ef695d929113786a072e5facf350c6a52fdc7d735be38a6feb7cc4f2374ea1443334f2f113d8dcb38b88448f69ee9de

data/CHANGELOG.md

@@ -2,6 +2,14 @@
 
 * TODO
 
+### 0.55.0
+
+* Fix to record proper time for around(:each) in RSpec
+
+    https://github.com/KnapsackPro/knapsack_pro-ruby/pull/62
+
+https://github.com/KnapsackPro/knapsack_pro-ruby/compare/v0.54.0...v0.55.0
+
 ### 0.54.0
 
 * Add Queue Mode for Minitest

data/README.md

@@ -112,6 +112,7 @@ The knapsack_pro has also [queue mode](#queue-mode) to get an optimal test suite
     - [Why I see API error commit_hash parameter is required?](#why-i-see-api-error-commit_hash-parameter-is-required)
     - [Why I see `LoadError: cannot load such file -- spec_helper`?](#why-i-see-loaderror-cannot-load-such-file----spec_helper)
     - [Why my CI build fails when I use Test::Unit even when all tests passed?](#why-my-ci-build-fails-when-i-use-testunit-even-when-all-tests-passed)
+    - [Why I see HEAD as branch name in user dashboard for Build metrics for my API token?](#why-i-see-head-as-branch-name-in-user-dashboard-for-build-metrics-for-my-api-token)
     - [Queue Mode problems](#queue-mode-problems)
       - [Why when I use Queue Mode for RSpec then my tests fail?](#why-when-i-use-queue-mode-for-rspec-then-my-tests-fail)
       - [Why when I use Queue Mode for RSpec then FactoryBot/FactoryGirl tests fail?](#why-when-i-use-queue-mode-for-rspec-then-factorybotfactorygirl-tests-fail)
@@ -122,6 +123,8 @@ The knapsack_pro has also [queue mode](#queue-mode) to get an optimal test suite
       - [Why my tests are executed twice in queue mode? Why CI node runs whole test suite again?](#why-my-tests-are-executed-twice-in-queue-mode-why-ci-node-runs-whole-test-suite-again)
       - [How to fix capybara-screenshot fail with `SystemStackError: stack level too deep` when using Queue Mode for RSpec?](#how-to-fix-capybara-screenshot-fail-with-systemstackerror-stack-level-too-deep-when-using-queue-mode-for-rspec)
       - [Parallel tests Cucumber and RSpec with Cucumber failures exit CI node early leaving fewer CI nodes to finish RSpec Queue.](#parallel-tests-cucumber-and-rspec-with-cucumber-failures-exit-ci-node-early-leaving-fewer-ci-nodes-to-finish-rspec-queue)
+      - [Why when I reran the same build (same commit hash, etc) on Codeship then no tests would get executed in Queue Mode?](#why-when-i-reran-the-same-build-same-commit-hash-etc-on-codeship-then-no-tests-would-get-executed-in-queue-mode)
+      - [How to find seed in RSpec output when I use Queue Mode for RSpec?](#how-to-find-seed-in-rspec-output-when-i-use-queue-mode-for-rspec)
   - [General questions](#general-questions)
     - [How to run tests for particular CI node in your development environment](#how-to-run-tests-for-particular-ci-node-in-your-development-environment)
       - [for knapack_pro regular mode](#for-knapack_pro-regular-mode)
@@ -149,6 +152,8 @@ The knapsack_pro has also [queue mode](#queue-mode) to get an optimal test suite
     - [How to run only RSpec feature tests or non feature tests?](#how-to-run-only-rspec-feature-tests-or-non-feature-tests)
     - [How to use CodeClimate with knapsack_pro?](#how-to-use-codeclimate-with-knapsack_pro)
     - [How to run knapsack_pro only on a few parallel CI nodes instead of all?](#how-to-run-knapsack_pro-only-on-a-few-parallel-ci-nodes-instead-of-all)
+    - [How to use simplecov in Queue Mode?](#how-to-use-simplecov-in-queue-mode)
+    - [Do I need to use separate API token for Queue Mode and Regular Mode?](#do-i-need-to-use-separate-api-token-for-queue-mode-and-regular-mode)
   - [Questions around data usage and security](#questions-around-data-usage-and-security)
     - [What data is sent to your servers?](#what-data-is-sent-to-your-servers)
     - [How is that data secured?](#how-is-that-data-secured)
@@ -883,7 +888,7 @@ test_ci_node_0:
     # Cucumber tests in Knapsack Pro Regular Mode (deterministic test suite split)
     - bundle exec rake knapsack_pro:cucumber
     # RSpec tests in Knapsack Pro Queue Mode (dynamic test suite split)
-    # It will autobalance bulid because it is executed after Cucumber tests.
+    # It will autobalance build because it is executed after Cucumber tests.
     - bundle exec rake knapsack_pro:queue:rspec
 
 # second CI node running in parallel
@@ -910,7 +915,7 @@ Configure test pipelines (1/2 used)
 KNAPSACK_PRO_CI_NODE_TOTAL=2 KNAPSACK_PRO_CI_NODE_INDEX=0 bundle exec rake knapsack_pro:cucumber
 
 # RSpec tests in Knapsack Pro Queue Mode (dynamic test suite split)
-# It will autobalance bulid because it is executed after Cucumber tests.
+# It will autobalance build because it is executed after Cucumber tests.
 KNAPSACK_PRO_CI_NODE_TOTAL=2 KNAPSACK_PRO_CI_NODE_INDEX=0 bundle exec rake knapsack_pro:queue:rspec
 ```
 
@@ -923,7 +928,7 @@ Configure test pipelines (2/2 used)
 KNAPSACK_PRO_CI_NODE_TOTAL=2 KNAPSACK_PRO_CI_NODE_INDEX=1 bundle exec rake knapsack_pro:cucumber
 
 # RSpec tests in Knapsack Pro Queue Mode (dynamic test suite split)
-# It will autobalance bulid because it is executed after Cucumber tests.
+# It will autobalance build because it is executed after Cucumber tests.
 KNAPSACK_PRO_CI_NODE_TOTAL=2 KNAPSACK_PRO_CI_NODE_INDEX=1 bundle exec rake knapsack_pro:queue:rspec
 ```
 
@@ -1079,6 +1084,16 @@ then problem might be related to the fact you specified complex `KNAPSACK_PRO_TE
 
 Please ensure you are actually using only Test::Unit runner. You may use some hybrid of Test::Unit and Minitest. Ensure you are not loading Minitest.
 
+#### Why I see HEAD as branch name in user dashboard for Build metrics for my API token?
+
+knapack_pro detects your branch name from environment variables of [supported CI providers](#supported-ci-providers). Sometimes the CI provider may expose the `HEAD` instead of branch name (for instance for pull request merge commits).
+
+The same can happen for CI provider not supported by default by knapsack_pro when you use [KNAPSACK_PRO_REPOSITORY_ADAPTER=git](#when-should-you-set-global-variable-knapsack_pro_repository_adaptergit-required-when-ci-provider-is-not-supported) to use local git installed on CI node to detect the branch name and git commit.
+
+knapack_pro uses git command `git -C /home/user/project_dir rev-parse --abbrev-ref HEAD` to detect branch name. See [source of knapack_pro](https://github.com/KnapsackPro/knapsack_pro-ruby/blob/master/lib/knapsack_pro/repository_adapters/git_adapter.rb). In most of cases it's good way to detect branch name. But if your CI provider during CI build checkouts to specific git commit then git cannot provide the name of the branch. In such scenario you would see `HEAD` as your branch name. It is good enough situation and knapack_pro will work correctly. The benefit of knowing exactly the branch name allows KnapsackPro API to better track history of test files timing changes across branches in order to better do split of test suite. The difference should be rather very small so it's not a problem that you have `HEAD` as branch name.
+
+If you would like to see exact branch name instead of `HEAD` in your `build metrics` history in [user dashboard](https://knapsackpro.com/dashboard) then you can explicitly provide the branch name with `KNAPSACK_PRO_BRANCH` for each CI build.
+
 #### Queue Mode problems
 
 ##### Why when I use Queue Mode for RSpec then my tests fail?
@@ -1190,6 +1205,45 @@ CI providers tips:
 
 * If you use CircleCI 2.0 you can use `when=always` flag. Read more [here](https://discuss.circleci.com/t/parallel-tests-cuc-rspec-w-failures-exit-early-leaving-less-workers-to-finish/18081).
 
+##### Why when I reran the same build (same commit hash, etc) on Codeship then no tests would get executed in Queue Mode?
+
+Codeship uses the same build ID ([`CI_BUILD_NUMBER`](https://documentation.codeship.com/basic/builds-and-configuration/set-environment-variables/#default-environment-variables)) if you re-run a build, so Codeship is not giving enough information to knapsack_pro gem that this is an independent build. Knapsack Pro API assumes you already ran tests for that build ID hence no tests were executed for reran CI build.
+
+To fix problem you can set `KNAPSACK_PRO_CI_NODE_BUILD_ID=missing-build-id` as empty string.
+This way knapsack_pro won’t use build ID provided by Codeship and each build will be treated as a unique. This should be good enough solution for most users.
+
+There is one edge case with that solution. Please note that the knapsack_pro gem doesn't have a CI build ID in order to generate a queue for each particular CI build. This may result in two different CI builds taking tests from the same queue when CI builds are running at the same time against the same git commit.
+
+To avoid this you should specify a unique `KNAPSACK_PRO_CI_NODE_BUILD_ID` environment variable for each CI build. This mean that each CI node that is part of particular CI build should have the same value for `KNAPSACK_PRO_CI_NODE_BUILD_ID`.
+
+##### How to find seed in RSpec output when I use Queue Mode for RSpec?
+
+In output for RSpec in knapack_pro Queue Mode you may see message:
+
+> INFO -- : [knapsack_pro] To retry in development the subset of tests fetched from API queue please run below command on your machine. If you use --order random then remember to add proper --seed 123 that you will find at the end of rspec command.
+>
+> INFO -- : [knapsack_pro] bundle exec rspec --default-path spec "spec/a_spec.rb" "spec/b_spec.rb"
+
+The seed number is used by RSpec only when you tell it, you need to provide argument `--order random`:
+
+```
+bundle exec rake "knapsack_pro:queue:rspec[--order random]"
+```
+
+then in RSpec output you will see something like:
+
+```
+Randomized with seed 11055
+```
+
+You can use the seed number to run tests in development:
+
+```
+bundle exec rspec --seed 11055 --default-path spec "spec/a_spec.rb" "spec/b_spec.rb"
+```
+
+If you don't use RSpec argument `--order random` then you don't need to provide `--seed` number when you want to reproduce tests in development.
+
 ### General questions
 
 #### How to run tests for particular CI node in your development environment
@@ -1400,6 +1454,35 @@ The xml report will contain all tests executed across intermediate test subset r
       end
     end
 
+Note if you use a CI provider or your own CI solution that uses common local drive for all parallel CI nodes then above solution needs to be adjusted to produce report file with CI node index number in the file name to avoid file conflicts. Example file name with CI node index number: `tmp/rspec_final_results_N.xml`.
+
+```
+# Queue Mode
+
+# must be exported to read correctly the value in below knapsack_pro command
+export KNAPSACK_PRO_CI_NODE_INDEX=0
+# if your CI provider exposes CI node index under other environment variable name then you could use it instead
+
+bundle exec rake "knapsack_pro:queue:rspec[--format documentation --format RspecJunitFormatter --out tmp/rspec_$KNAPSACK_PRO_CI_NODE_INDEX.xml]"
+```
+
+In below code we use CI node index number in `TMP_RSPEC_XML_REPORT` and `FINAL_RSPEC_XML_REPORT`:
+
+```
+# spec_helper.rb or rails_helper.rb
+
+# TODO This must be the same path as value for rspec --out argument
+TMP_RSPEC_XML_REPORT = "tmp/rspec_#{ENV['KNAPSACK_PRO_CI_NODE_INDEX']}.xml"
+# move results to FINAL_RSPEC_XML_REPORT so the results won't accumulate with duplicated xml tags in TMP_RSPEC_XML_REPORT
+FINAL_RSPEC_XML_REPORT = "tmp/rspec_final_results_#{ENV['KNAPSACK_PRO_CI_NODE_INDEX']}.xml"
+
+KnapsackPro::Hooks::Queue.after_subset_queue do |queue_id, subset_queue_id|
+  if File.exist?(TMP_RSPEC_XML_REPORT)
+    FileUtils.mv(TMP_RSPEC_XML_REPORT, FINAL_RSPEC_XML_REPORT)
+  end
+end
+```
+
 #### How many API keys I need?
 
 Basically you need as many API keys as you have steps in your build.
@@ -1697,6 +1780,30 @@ On the 3rd CI node, you can run other things like linters etc.
 If you would like to check what is the CI node total ENV variable name exposed by your CI provider you can check that in your CI provider environment variables docs
 or preview the [ENV variables that knapack_pro can read](https://github.com/KnapsackPro/knapsack_pro-ruby/tree/master/lib/knapsack_pro/config/ci) for supported CI providers.
 
+#### How to use simplecov in Queue Mode?
+
+If you would like to make [simplecov](https://github.com/colszowka/simplecov) gem work with knapack_pro Queue Mode to correctly track code coverage for parallel CI nodes please do:
+
+```
+# spec_helper.rb or rails_helper.rb
+require 'knapsack_pro'
+
+require 'simplecov'
+SimpleCov.start
+
+KnapsackPro::Hooks::Queue.before_queue do |queue_id|
+  SimpleCov.command_name("rspec_ci_node_#{KnapsackPro::Config::Env.ci_node_index}")
+end
+```
+
+This way there should be no conflict between code coverage reports generated per CI node index even when you use the same local drive (for instance you use Jenkins as your CI provider). The simplecov will generate single report at `coverage/index.html` with merged data from parallel CI nodes.
+
+#### Do I need to use separate API token for Queue Mode and Regular Mode?
+
+I recommend to record timing of a new test suite with `API token A` and knapsack_pro Regular Mode. After you recorded test suite timing then you should use the `API token A` to run your tests in knapack_pro Queue Mode. This way Queue Mode will leverage test suite timing recorded in a fast way with Regular Mode so the first run in Queue Mode won't be slow due to recording test files timing for the first time.
+
+When you want to go back from Queue Mode to Regular Mode then the fact of using the same API token could cause edge cases that some builds might not be well balanced in Regular Mode. That is why I recommend using separate API token for Regular Mode and Queue Mode. If you plan to use only Queue Mode then no worry.
+
 ### Questions around data usage and security
 
 #### What data is sent to your servers?

data/lib/knapsack_pro/adapters/rspec_adapter.rb

@@ -21,18 +21,19 @@ module KnapsackPro
 
       def bind_time_tracker
         ::RSpec.configure do |config|
-          config.prepend_before(:each) do
+          config.around(:each) do |example|
             current_example_group =
               if ::RSpec.respond_to?(:current_example)
                 ::RSpec.current_example.metadata[:example_group]
               else
                 example.metadata
               end
+
             KnapsackPro.tracker.current_test_path = KnapsackPro::Adapters::RSpecAdapter.test_path(current_example_group)
             KnapsackPro.tracker.start_timer
-          end
 
-          config.append_after(:each) do
+            example.run
+
             KnapsackPro.tracker.stop_timer
           end
 

data/lib/knapsack_pro/version.rb

@@ -1,3 +1,3 @@
 module KnapsackPro
-  VERSION = '0.54.0'
+  VERSION = '0.55.0'
 end

data/spec/knapsack_pro/adapters/rspec_adapter_spec.rb

@@ -78,10 +78,10 @@ describe KnapsackPro::Adapters::RSpecAdapter do
           example_group: example_group
         })
       end
+      let(:example) { double }
 
       it do
-        expect(config).to receive(:prepend_before).with(:each).and_yield
-        expect(config).to receive(:append_after).with(:each).and_yield
+        expect(config).to receive(:around).with(:each).and_yield(example)
         expect(config).to receive(:after).with(:suite).and_yield
         expect(::RSpec).to receive(:configure).and_yield(config)
 
@@ -92,6 +92,8 @@ describe KnapsackPro::Adapters::RSpecAdapter do
         expect(tracker).to receive(:current_test_path=).with(test_path)
         expect(tracker).to receive(:start_timer)
 
+        expect(example).to receive(:run)
+
         expect(tracker).to receive(:stop_timer)
 
         expect(KnapsackPro::Presenter).to receive(:global_time).and_return(global_time)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: knapsack_pro
 version: !ruby/object:Gem::Version
-  version: 0.54.0
+  version: 0.55.0
 platform: ruby
 authors:
 - ArturT
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-11 00:00:00.000000000 Z
+date: 2018-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake