knapsack_pro 0.32.0 → 0.33.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f61f4cbac8cf4737c994517d3d2b720b43e31dc0
-  data.tar.gz: 8e78bb42b3cf42aae6ff29cbaac9bbadc059c005
+  metadata.gz: d67c800e334256a4ec337d39b062f884dd104c66
+  data.tar.gz: 6abba16d16344dc3646fe33b1f72a7940790736f
 SHA512:
-  metadata.gz: b03f01778fa3c20735b54c71631b0beeb74658fdef67ab8247aa325aef55de0268ef6ede238a6d75733f3f4ccd74df734769b974c2dca2616a412830f015165e
-  data.tar.gz: 30fcb16188e7e2c72fcfeaa1038775fa9f54f5ffb84ba8ababed6883637ce9b8e4e04f4f2c91f7015e94629437fd4b167f075f3fd026edb834b1f019a7e6664b
+  metadata.gz: f901514b6e6ee5ecc22cc72f957bf78845196b53a3aadba5dabadf2cb37b84ea5369f91deee8eb220238c24a7736540067ef4deb9f4c2899020141abba1868bd
+  data.tar.gz: a2f8feb04f69e5fcb5f3a702a239a45238ffdf92c15a2c66ffb6e5618c2f64527722a5bb5037e72d396defec1e4ceb45eebab4aa6f153cf22139c3bfcc270c2a

data/CHANGELOG.md

@@ -2,6 +2,16 @@
 
 * TODO
 
+### 0.33.0
+
+* Add RSpec Queue Formatter to hide duplicated pending and failed tests in Queue Mode
+
+  You can keep duplicated pending/failed summary with flag `KNAPSACK_PRO_MODIFY_DEFAULT_RSPEC_FORMATTERS`. More can be found in read me.
+
+    https://github.com/KnapsackPro/knapsack_pro-ruby/pull/31
+
+https://github.com/KnapsackPro/knapsack_pro-ruby/compare/v0.32.0...v0.33.0
+
 ### 0.32.0
 
 * Add encryption for branch names

data/README.md

@@ -74,6 +74,7 @@ The knapsack_pro has also [queue mode](#queue-mode) to get most optimal test sui
   - [Additional info about queue mode](#additional-info-about-queue-mode)
   - [Extra configuration for Queue Mode](#extra-configuration-for-queue-mode)
     - [KNAPSACK_PRO_FIXED_QUEUE_SPLIT (remember queue split on retry CI node)](#knapsack_pro_fixed_queue_split-remember-queue-split-on-retry-ci-node)
+    - [KNAPSACK_PRO_MODIFY_DEFAULT_RSPEC_FORMATTERS (hide duplicated summary of pending and failed tests)](#knapsack_pro_modify_default_rspec_formatters-hide-duplicated-summary-of-pending-and-failed-tests)
   - [Supported test runners in queue mode](#supported-test-runners-in-queue-mode)
 - [Extra configuration for CI server](#extra-configuration-for-ci-server)
   - [Info about ENV variables](#info-about-env-variables)
@@ -98,33 +99,36 @@ The knapsack_pro has also [queue mode](#queue-mode) to get most optimal test sui
     - [Info for snap-ci.com users](#info-for-snap-cicom-users)
     - [Info for Jenkins users](#info-for-jenkins-users)
 - [FAQ](#faq)
-  - [How to run tests for particular CI node in your development environment](#how-to-run-tests-for-particular-ci-node-in-your-development-environment)
-    - [for knapack_pro regular mode](#for-knapack_pro-regular-mode)
-    - [for knapsack_pro queue mode](#for-knapsack_pro-queue-mode)
-  - [What happens when Knapsack Pro API is not available/not reachable temporarily?](#what-happens-when-knapsack-pro-api-is-not-availablenot-reachable-temporarily)
-    - [for knapack_pro regular mode](#for-knapack_pro-regular-mode-1)
-    - [for knapsack_pro queue mode](#for-knapsack_pro-queue-mode-1)
-  - [How can I change log level?](#how-can-i-change-log-level)
-  - [How to split tests based on test level instead of test file level?](#how-to-split-tests-based-on-test-level-instead-of-test-file-level)
-    - [A. Create multiple small test files](#a-create-multiple-small-test-files)
-    - [B. Use tags to mark set of tests in particular test file](#b-use-tags-to-mark-set-of-tests-in-particular-test-file)
-  - [How to make knapsack_pro works for forked repositories of my project?](#how-to-make-knapsack_pro-works-for-forked-repositories-of-my-project)
+  - [Common problems](#common-problems)
+    - [Why I see API error commit_hash parameter is required?](#why-i-see-api-error-commit_hash-parameter-is-required)
+    - [Queue Mode problems](#queue-mode-problems)
+      - [Why when I use Queue Mode for RSpec and test fails then I see multiple times info about failed test in RSpec result?](#why-when-i-use-queue-mode-for-rspec-and-test-fails-then-i-see-multiple-times-info-about-failed-test-in-rspec-result)
+      - [Why when I use Queue Mode for RSpec then I see multiple times the same pending tests?](#why-when-i-use-queue-mode-for-rspec-then-i-see-multiple-times-the-same-pending-tests)
+      - [Does in Queue Mode the RSpec is initialized many times that causes Rails load over and over again?](#does-in-queue-mode-the-rspec-is-initialized-many-times-that-causes-rails-load-over-and-over-again)
+      - [Why my tests are executed twice in queue mode? Why CI node runs whole test suite again?](#why-my-tests-are-executed-twice-in-queue-mode-why-ci-node-runs-whole-test-suite-again)
+      - [How to fix capybara-screenshot fail with `SystemStackError: stack level too deep` when using Queue Mode for RSpec?](#how-to-fix-capybara-screenshot-fail-with-systemstackerror-stack-level-too-deep-when-using-queue-mode-for-rspec)
+  - [General questions](#general-questions)
+    - [How to run tests for particular CI node in your development environment](#how-to-run-tests-for-particular-ci-node-in-your-development-environment)
+      - [for knapack_pro regular mode](#for-knapack_pro-regular-mode)
+      - [for knapsack_pro queue mode](#for-knapsack_pro-queue-mode)
+    - [What happens when Knapsack Pro API is not available/not reachable temporarily?](#what-happens-when-knapsack-pro-api-is-not-availablenot-reachable-temporarily)
+      - [for knapack_pro regular mode](#for-knapack_pro-regular-mode-1)
+      - [for knapsack_pro queue mode](#for-knapsack_pro-queue-mode-1)
+    - [How can I change log level?](#how-can-i-change-log-level)
+    - [How to split tests based on test level instead of test file level?](#how-to-split-tests-based-on-test-level-instead-of-test-file-level)
+      - [A. Create multiple small test files](#a-create-multiple-small-test-files)
+      - [B. Use tags to mark set of tests in particular test file](#b-use-tags-to-mark-set-of-tests-in-particular-test-file)
+    - [How to make knapsack_pro works for forked repositories of my project?](#how-to-make-knapsack_pro-works-for-forked-repositories-of-my-project)
+    - [How to use junit formatter?](#how-to-use-junit-formatter)
+    - [How many API keys I need?](#how-many-api-keys-i-need)
+    - [What is optimal order of test commands?](#what-is-optimal-order-of-test-commands)
+    - [How to set `before(:suite)` and `after(:suite)` RSpec hooks in Queue Mode (Percy.io example)?](#how-to-set-beforesuite-and-aftersuite-rspec-hooks-in-queue-mode-percyio-example)
+    - [How to call `before(:suite)` and `after(:suite)` RSpec hooks only once in Queue Mode?](#how-to-call-beforesuite-and-aftersuite-rspec-hooks-only-once-in-queue-mode)
+    - [How to run knapsack_pro with parallel_tests gem?](#how-to-run-knapsack_pro-with-parallel_tests-gem)
   - [Questions around data usage and security](#questions-around-data-usage-and-security)
     - [What data is sent to your servers?](#what-data-is-sent-to-your-servers)
     - [How is that data secured?](#how-is-that-data-secured)
     - [Who has access to the data?](#who-has-access-to-the-data)
-  - [Why when I use Queue Mode for RSpec and test fails then I see multiple times info about failed test in RSpec result?](#why-when-i-use-queue-mode-for-rspec-and-test-fails-then-i-see-multiple-times-info-about-failed-test-in-rspec-result)
-  - [Why when I use Queue Mode for RSpec then I see multiple times the same pending tests?](#why-when-i-use-queue-mode-for-rspec-then-i-see-multiple-times-the-same-pending-tests)
-  - [Does in Queue Mode the RSpec is initialized many times that causes Rails load over and over again?](#does-in-queue-mode-the-rspec-is-initialized-many-times-that-causes-rails-load-over-and-over-again)
-  - [How to use junit formatter?](#how-to-use-junit-formatter)
-  - [Why I see API error commit_hash parameter is required?](#why-i-see-api-error-commit_hash-parameter-is-required)
-  - [How many API keys I need?](#how-many-api-keys-i-need)
-  - [What is optimal order of test commands?](#what-is-optimal-order-of-test-commands)
-  - [Why my tests are executed twice in queue mode? Why CI node runs whole test suite again?](#why-my-tests-are-executed-twice-in-queue-mode-why-ci-node-runs-whole-test-suite-again)
-  - [How to set `before(:suite)` and `after(:suite)` RSpec hooks in Queue Mode (Percy.io example)?](#how-to-set-beforesuite-and-aftersuite-rspec-hooks-in-queue-mode-percyio-example)
-  - [How to call `before(:suite)` and `after(:suite)` RSpec hooks only once in Queue Mode?](#how-to-call-beforesuite-and-aftersuite-rspec-hooks-only-once-in-queue-mode)
-  - [How to fix capybara-screenshot fail with `SystemStackError: stack level too deep` when using Queue Mode for RSpec?](#how-to-fix-capybara-screenshot-fail-with-systemstackerror-stack-level-too-deep-when-using-queue-mode-for-rspec)
-  - [How to run knapsack_pro with parallel_tests gem?](#how-to-run-knapsack_pro-with-parallel_tests-gem)
 - [Gem tests](#gem-tests)
   - [Spec](#spec)
 - [Contributing](#contributing)
@@ -392,7 +396,7 @@ There might be some cached test suite splits for git commits you run in past for
 
 * `KNAPSACK_PRO_FIXED_QUEUE_SPLIT=false` (default)
 
-  By default, the fixed queue split is off. It means when you will run tests for the same commit hash and a total number of nodes and for the same branch then each time the queue will be generated dynamically and CI nodes will fetch from Knapsack Pro API the test files in a dynamic way. This is default because it gives the most optimal test suite split for the whole test build across all CI nodes even when you retry the same build on CI many times.
+  By default, the fixed queue split is off. It means when you will run tests for the same commit hash and a total number of nodes and for the same branch, and the CI build ID is different with second tests run then the queue will be generated dynamically and CI nodes will fetch from Knapsack Pro API the test files in a dynamic way. This is default because it gives the most optimal test suite split for the whole test build across all CI nodes.
 
 * `KNAPSACK_PRO_FIXED_QUEUE_SPLIT=true`
 
@@ -411,6 +415,18 @@ There might be some cached test suite splits for git commits you run in past for
 
   To [reproduce tests executed on CI node](#for-knapsack_pro-queue-mode) in development environment please see FAQ.
 
+#### KNAPSACK_PRO_MODIFY_DEFAULT_RSPEC_FORMATTERS (hide duplicated summary of pending and failed tests)
+
+* `KNAPSACK_PRO_MODIFY_DEFAULT_RSPEC_FORMATTERS=true` (default)
+
+  By default, the knapack_pro will monkey patch [RSpec Formatters](https://www.relishapp.com/rspec/rspec-core/v/2-6/docs/command-line/format-option) in order to
+  hide the summary of pending and failed tests after each intermediate run of tests fetched from the work queue on Knapsack Pro API.
+  knapack_pro shows summary of all pending and failed tests at the very end when work queue ended. If you use your custom formatter and you have problem with it then you can disable `KNAPSACK_PRO_MODIFY_DEFAULT_RSPEC_FORMATTERS=false` monkey patching.
+
+* `KNAPSACK_PRO_MODIFY_DEFAULT_RSPEC_FORMATTERS=false`
+
+  It causes to show summary of pending and failed tests after each intermediate tests run from the work queue. The summary will grown cumulatively after each intermediate tests run so it means you will see multiple times summary of the same pending/failed tests. It doesn't mean the test files are executed twice. Test files are executed only once. Only summary report grows cumulatively.
+
 ### Supported test runners in queue mode
 
 At this moment the queue mode works for:
@@ -791,9 +807,66 @@ If you are going to relay on rspec to autobalance build when cucumber tests were
 
 ## FAQ
 
-### How to run tests for particular CI node in your development environment
+### Common problems
+
+#### Why I see API error commit_hash parameter is required?
+
+    ERROR -- : [knapsack_pro] {"errors"=>[{"commit_hash"=>["parameter is required"]}]}
+
+When Knapsack Pro API returns error like above the problem is because you use CI provider not supported by knapack_pro which means
+knapack_pro gem cannot determine the git commit hash and branch name. To fix this problem you can do:
+
+* if you have git installed on CI node then you can use it to determine git commit hash and branch name. [See this](#when-you-set-global-variable-knapsack_pro_repository_adaptergit-required-when-ci-provider-is-not-supported)
+* if you have no git installed on CI node then you should manually set `KNAPSACK_PRO_BRANCH` and `KNAPSACK_PRO_COMMIT_HASH`. For instance this might be useful when you use Jenkins. [See this](#when-you-not-set-global-variable-knapsack_pro_repository_adapter-default)
+
+#### Queue Mode problems
+
+##### Why when I use Queue Mode for RSpec and test fails then I see multiple times info about failed test in RSpec result?
+
+The problem may happen when you use old knapsack_pro `< 0.33.0` or if you use custom rspec formatter, or when you set flag [KNAPSACK_PRO_MODIFY_DEFAULT_RSPEC_FORMATTERS=false](#knapsack_pro_modify_default_rspec_formatters-hide-duplicated-summary-of-pending-and-failed-tests).
+
+When you use Queue Mode then knapack_pro does multiple requests to Knapsack Pro API and fetches a few test files to execute.
+This means RSpec will remember failed tests so far and it will present them at the end of each executed test subset if flag `KNAPSACK_PRO_MODIFY_DEFAULT_RSPEC_FORMATTERS=false`.
+You can see the list of all failed test files at the end of knapack_pro queue mode command.
+
+##### Why when I use Queue Mode for RSpec then I see multiple times the same pending tests?
+
+The problem may happen when you use old knapsack_pro `< 0.33.0` or if you use custom rspec formatter, or when you set flag [KNAPSACK_PRO_MODIFY_DEFAULT_RSPEC_FORMATTERS=false](#knapsack_pro_modify_default_rspec_formatters-hide-duplicated-summary-of-pending-and-failed-tests).
+
+When you use Queue Mode then knapack_pro does multiple requests to Knapsack Pro API and fetches a few test files to execute.
+This means RSpec will remember pending tests so far and it will present them at the end of each executed test subset if flag `KNAPSACK_PRO_MODIFY_DEFAULT_RSPEC_FORMATTERS=false`.
+You can see the list of all pending test files at the end of knapack_pro queue mode command.
+
+##### Does in Queue Mode the RSpec is initialized many times that causes Rails load over and over again?
+
+No. In Queue Mode the RSpec configuration is updated every time when knapsack_pro gem gets a new set of test files from the Knapsack Pro API and it looks in knapsack_pro output like RSpec was loaded many times but in fact, it loads your project environment only once.
+
+##### Why my tests are executed twice in queue mode? Why CI node runs whole test suite again?
+
+This may happen when you use not supported CI provider by knapack_pro. It's because of missing value of CI build ID. You can set unique `KNAPSACK_PRO_CI_NODE_BUILD_ID` for each CI build. The problem with test suite run again happens when one of your CI node started work later when all other CI nodes already executed whole test suite.
+The slow CI node that started work late will initialize a new queue hence the tests executed twice.
+
+To solve this problem you can set `KNAPSACK_PRO_CI_NODE_BUILD_ID` as mentioned above or you can set `KNAPSACK_PRO_FIXED_QUEUE_SPLIT=true`.
+Please [read this](#knapsack_pro_fixed_queue_split-remember-queue-split-on-retry-ci-node).
 
-#### for knapack_pro regular mode
+##### How to fix capybara-screenshot fail with `SystemStackError: stack level too deep` when using Queue Mode for RSpec?
+
+Please use fixed version of capybara-screenshot.
+
+```
+# Gemfile
+group :test do
+  gem 'capybara-screenshot', github: 'ArturT/capybara-screenshot', branch: 'fix-reporter_module-loaded-twice'
+end
+```
+
+Here is [fix PR](https://github.com/mattheworiordan/capybara-screenshot/pull/205) to official capybara-screenshot repository and the explanation of the problem.
+
+### General questions
+
+#### How to run tests for particular CI node in your development environment
+
+##### for knapack_pro regular mode
 
 In your development environment you can debug tests that were run on the particular CI node.
 For instance to run subset of tests for the first CI node with specified seed you can do.
@@ -821,7 +894,7 @@ Command similar to above can be executed in your development this way:
 
 If you were running your tests with `--order random` on your CI then you can additionaly pass seed param with proper value in above command (`--seed 123`).
 
-#### for knapsack_pro queue mode
+##### for knapsack_pro queue mode
 
 There are a few ways to reproduce tests executed on CI node in your development environment.
 
@@ -864,17 +937,17 @@ There are a few ways to reproduce tests executed on CI node in your development
       KNAPSACK_PRO_FIXED_QUEUE_SPLIT=true \
       bundle exec rake "knapsack_pro:queue:rspec[--seed 123]"
 
-### What happens when Knapsack Pro API is not available/not reachable temporarily?
+#### What happens when Knapsack Pro API is not available/not reachable temporarily?
 
-#### for knapack_pro regular mode
+##### for knapack_pro regular mode
 
 knapack_pro gem will retry requests to Knapsack Pro API multiple times every few seconds til it switch to fallback behaviour and it will split test files across CI nodes based on popular test directory names.
 
-#### for knapsack_pro queue mode
+##### for knapsack_pro queue mode
 
 knapack_pro gem will retry requests to Knapsack Pro API multiple times every few seconds til it fails.
 
-### How can I change log level?
+#### How can I change log level?
 
 You can change log level by specifying the `KNAPSACK_PRO_LOG_LEVEL` environment variable.
 
@@ -887,16 +960,16 @@ Recommended log levels you can use:
 * `debug` is default log level and it is recommended to log details about requests to Knapsack Pro API. Thanks to that you can debug things or ensure everything works. For instance in [user dashboard](https://knapsackpro.com/dashboard) you can find tips referring to debug logs.
 * `info` level shows message like how to retry tests in development or info why something works this way or the other (for instance why tests were not executed on the CI node). You can use `info` level when you really don't want to see all debug messages from default log level.
 
-### How to split tests based on test level instead of test file level?
+#### How to split tests based on test level instead of test file level?
 
 If you want to split one big test file (test file with long time execution) across multiple CI nodes then you can:
 
-#### A. Create multiple small test files
+##### A. Create multiple small test files
 
 Create multiple small test files instead of one long running test file with many test cases.
 A lot of small test files will give you better test suite split results.
 
-#### B. Use tags to mark set of tests in particular test file
+##### B. Use tags to mark set of tests in particular test file
 
 Another way is to use tags to mark subset of tests in particular test file and then split tests based on tags.
 
@@ -933,7 +1006,7 @@ You need to run below commands for each CI node.
     # run other tests without tag A & B
     KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC=api_key_for_tests_without_tags_A_and_B bundle exec rake "knapsack_pro:rspec[--tag ~tagA --tag ~tagB]"
 
-### How to make knapsack_pro works for forked repositories of my project?
+#### How to make knapsack_pro works for forked repositories of my project?
 
 Imagine one of the scenarios, for this example I use the Travis-CI.
 
@@ -961,49 +1034,7 @@ fi
 Now you can use `bin/knapsack_pro_rspec` command instead of `bundle exec rake knapsack_pro:rspec`.
 Remember to follow other steps required for your CI provider.
 
-### Questions around data usage and security
-
-#### What data is sent to your servers?
-
-The knapsack_pro gem sends branch name, commit hash, CI total node number, CI index node number, the test file paths like `spec/models/user_spec.rb` and the time execution of each test file path as a float.
-
-Here is the [full specification of the API](http://docs.knapsackpro.com/api/v1/) used by knapsack_pro gem.
-
-#### How is that data secured?
-
-The test file paths can be [encrypted](#test-file-names-encryption) on your CI node with your salt and later send to knapsackpro.com API.
-You generate salt locally and only you can decrypt the test file paths.
-
-Connection with knapsackpro.com server is via https.
-
-Regarding payments we use the BraintreePayments.com and they store credit cards and your private information.
-
-#### Who has access to the data?
-
-I’m the only admin so I can preview data in case you need help with debugging some problem etc. I’m not able to decrypt them without knowing the salt.
-
-When you sign in to your user dashboard then you can preview data for recent 100 builds on CI. If the test file paths are encrypted then you only see hashes for test file paths.
-You need to [decrypt](#how-to-debug-test-file-names) them locally on your machine to find out what each test file hash is.
-
-### Why when I use Queue Mode for RSpec and test fails then I see multiple times info about failed test in RSpec result?
-
-RSpec collects information about failed tests and presents it at the end of RSpec result.
-When you use Queue Mode then knapack_pro does multiple requests to Knapsack Pro API and fetches a few test files to execute.
-This means RSpec will remember failed tests so far and it will present them at the end of each executed test subset.
-You can see the list of all failed test files at the end of knapack_pro queue mode command.
-
-### Why when I use Queue Mode for RSpec then I see multiple times the same pending tests?
-
-RSpec collects information about pending tests and presents it at the end of RSpec result.
-When you use Queue Mode then knapack_pro does multiple requests to Knapsack Pro API and fetches a few test files to execute.
-This means RSpec will remember pending tests so far and it will present them at the end of each executed test subset.
-You can see the list of all pending test files at the end of knapack_pro queue mode command.
-
-### Does in Queue Mode the RSpec is initialized many times that causes Rails load over and over again?
-
-No. In Queue Mode the RSpec configuration is updated every time when knapsack_pro gem gets a new set of test files from the Knapsack Pro API and it looks in knapsack_pro output like RSpec was loaded many times but in fact, it loads your project environment only once.
-
-### How to use junit formatter?
+#### How to use junit formatter?
 
 You can use junit formatter for rspec thanks to gem [rspec_junit_formatter](https://github.com/sj26/rspec_junit_formatter).
 Here you can find example how to generate `rspec.xml` file with junit format and at the same time show normal documentation format output for RSpec.
@@ -1015,17 +1046,7 @@ Here you can find example how to generate `rspec.xml` file with junit format and
     # The xml report will contain all tests executed across intermediate test subset runs based on queue
     bundle exec rake "knapsack_pro:queue:rspec[--format documentation --format RspecJunitFormatter --out tmp/rspec.xml]"
 
-### Why I see API error commit_hash parameter is required?
-
-    ERROR -- : [knapsack_pro] {"errors"=>[{"commit_hash"=>["parameter is required"]}]}
-
-When Knapsack Pro API returns error like above the problem is because you use CI provider not supported by knapack_pro which means
-knapack_pro gem cannot determine the git commit hash and branch name. To fix this problem you can do:
-
-* if you have git installed on CI node then you can use it to determine git commit hash and branch name. [See this](#when-you-set-global-variable-knapsack_pro_repository_adaptergit-required-when-ci-provider-is-not-supported)
-* if you have no git installed on CI node then you should manually set `KNAPSACK_PRO_BRANCH` and `KNAPSACK_PRO_COMMIT_HASH`. For instance this might be useful when you use Jenkins. [See this](#when-you-not-set-global-variable-knapsack_pro_repository_adapter-default)
-
-### How many API keys I need?
+#### How many API keys I need?
 
 Basically you need as many API keys as you have steps in your build.
 
@@ -1044,7 +1065,7 @@ Note:
 * If you pass `--tag tagA` or `--tag ~tagA` then you run subset of your whole test suite hence you need separate API key.
 * If you use regular or queue mode then you need separate API key for each mode.
 
-### What is optimal order of test commands?
+#### What is optimal order of test commands?
 
 __Tip 1:__
 
@@ -1068,13 +1089,7 @@ When you have short test suite, for instance in javascript then you could distri
 
 You will run your javascript tests on single CI node and the knapack_pro will auto-balance CI build with Queue Mode. Thanks to that CI build time execution will be flat and optimal (as fast as possible).
 
-### Why my tests are executed twice in queue mode? Why CI node runs whole test suite again?
-
-This may happen when one of your CI node started work later when all other CI nodes already executed whole test suite.
-The slow CI node will initialize a new queue hence the tests executed twice. To solve this problem you should set `KNAPSACK_PRO_FIXED_QUEUE_SPLIT=true`.
-Please [read this](#knapsack_pro_fixed_queue_split-remember-queue-split-on-retry-ci-node).
-
-### How to set `before(:suite)` and `after(:suite)` RSpec hooks in Queue Mode (Percy.io example)?
+#### How to set `before(:suite)` and `after(:suite)` RSpec hooks in Queue Mode (Percy.io example)?
 
 Some tools like [Percy.io](https://percy.io/docs/clients/ruby/capybara-rails) requires to set hooks for RSpec `before(:suite)` and `after(:suite)`.
 Knapsack Pro Queue Mode runs subset of test files from the work queue many times. This means the RSpec hooks `before(:suite)` and `after(:suite)` will execute multiple times. If you want to run some code only once before Queue Mode starts work and after it finishes then you should do it this way:
@@ -1089,7 +1104,7 @@ Percy::Capybara.initialize_build
 at_exit { Percy::Capybara.finalize_build }
 ```
 
-### How to call `before(:suite)` and `after(:suite)` RSpec hooks only once in Queue Mode?
+#### How to call `before(:suite)` and `after(:suite)` RSpec hooks only once in Queue Mode?
 
 Knapsack Pro Queue Mode runs subset of test files from the work queue many times. This means the RSpec hooks `before(:suite)` and `after(:suite)` will be executed multiple times. If you want to run some code only once before Queue Mode starts work and after it finishes then you should do it this way:
 
@@ -1111,20 +1126,7 @@ RSpec.configure do |config|
 end
 ```
 
-### How to fix capybara-screenshot fail with `SystemStackError: stack level too deep` when using Queue Mode for RSpec?
-
-Please use fixed version of capybara-screenshot.
-
-```
-# Gemfile
-group :test do
-  gem 'capybara-screenshot', github: 'ArturT/capybara-screenshot', branch: 'fix-reporter_module-loaded-twice'
-end
-```
-
-Here is [fix PR](https://github.com/mattheworiordan/capybara-screenshot/pull/205) to official capybara-screenshot repository and the explanation of the problem.
-
-### How to run knapsack_pro with parallel_tests gem?
+#### How to run knapsack_pro with parallel_tests gem?
 
 You can run knapsack_pro with [parallel_tests](https://github.com/grosser/parallel_tests) gem to run multiple concurrent knapsack_pro commands per CI node.
 
@@ -1207,6 +1209,30 @@ To ensure everything works you can check output for each CI node.
     (tests output here)
     ```
 
+### Questions around data usage and security
+
+#### What data is sent to your servers?
+
+The knapsack_pro gem sends branch name, commit hash, CI total node number, CI index node number, the test file paths like `spec/models/user_spec.rb` and the time execution of each test file path as a float.
+
+Here is the [full specification of the API](http://docs.knapsackpro.com/api/v1/) used by knapsack_pro gem.
+
+#### How is that data secured?
+
+The test file paths and/or branch names can be [encrypted](#test-file-names-encryption) on your CI node with a salt and later send to knapsackpro.com API.
+You generate the salt locally and only you can decrypt the test file paths or branch names.
+
+Connection with knapsackpro.com server is via https.
+
+Regarding payments we use the BraintreePayments.com and they store credit cards and your private information.
+
+#### Who has access to the data?
+
+I’m the only admin so I can preview data in case you need help with debugging some problem etc. I’m not able to decrypt them without knowing the salt.
+
+When you sign in to your user dashboard then you can preview data for recent 100 builds on CI. If the test file paths are encrypted then you only see hashes for test file paths.
+You need to [decrypt](#how-to-debug-test-file-names) them locally on your machine to find out what each test file hash is.
+
 ## Gem tests
 
 ### Spec

data/lib/knapsack_pro/config/env.rb

@@ -75,6 +75,14 @@ module KnapsackPro
           test_files_encrypted == 'true'
         end
 
+        def modify_default_rspec_formatters
+          ENV.fetch('KNAPSACK_PRO_MODIFY_DEFAULT_RSPEC_FORMATTERS', true)
+        end
+
+        def modify_default_rspec_formatters?
+          modify_default_rspec_formatters.to_s == 'true'
+        end
+
         def branch_encrypted
           ENV['KNAPSACK_PRO_BRANCH_ENCRYPTED']
         end

data/lib/knapsack_pro/formatters/rspec_queue_summary_formatter.rb

@@ -0,0 +1,97 @@
+RSpec::Support.require_rspec_core('formatters/base_formatter')
+RSpec::Support.require_rspec_core('formatters/base_text_formatter')
+
+module KnapsackPro
+  module Formatters
+    module RSpecHideFailuresAndPendingExtension
+      def dump_failures(notification); end
+      def dump_pending(notification); end
+      def dump_summary(summary); end
+    end
+
+    class RSpecQueueSummaryFormatter < RSpec::Core::Formatters::BaseFormatter
+      RSpec::Core::Formatters.register self, :dump_failures, :dump_pending
+
+      def self.registered_output=(output)
+        @registered_output = {
+          ENV['KNAPSACK_PRO_QUEUE_ID'] => output
+        }
+      end
+
+      def self.registered_output
+        @registered_output[ENV['KNAPSACK_PRO_QUEUE_ID']]
+      end
+
+      def self.most_recent_failures_summary=(fully_formatted_failed_examples)
+        @most_recent_failures_summary = {
+          ENV['KNAPSACK_PRO_QUEUE_ID'] => fully_formatted_failed_examples
+        }
+      end
+
+      def self.most_recent_failures_summary
+        @most_recent_failures_summary ||= {}
+        @most_recent_failures_summary[ENV['KNAPSACK_PRO_QUEUE_ID']] || []
+      end
+
+      def self.most_recent_pending=(fully_formatted_pending_examples)
+        @most_recent_pending = {
+          ENV['KNAPSACK_PRO_QUEUE_ID'] => fully_formatted_pending_examples
+        }
+      end
+
+      def self.most_recent_pending
+        @most_recent_pending ||= {}
+        @most_recent_pending[ENV['KNAPSACK_PRO_QUEUE_ID']] || []
+      end
+
+      def self.most_recent_summary=(fully_formatted)
+        @most_recent_summary = {
+          ENV['KNAPSACK_PRO_QUEUE_ID'] => fully_formatted
+        }
+      end
+
+      def self.most_recent_summary
+        @most_recent_summary ||= {}
+        @most_recent_summary[ENV['KNAPSACK_PRO_QUEUE_ID']] || []
+      end
+
+      def self.print_summary
+        registered_output.puts('Knapsack Pro Queue finished!')
+        registered_output.puts('')
+
+        unless most_recent_pending.empty?
+          registered_output.puts('All pending tests on this CI node:')
+          registered_output.puts(most_recent_pending)
+        end
+
+        unless most_recent_failures_summary.empty?
+          registered_output.puts('All failed tests on this CI node:')
+          registered_output.puts(most_recent_failures_summary)
+        end
+
+        registered_output.puts(most_recent_summary)
+      end
+
+      def initialize(output)
+        super
+        self.class.registered_output = output
+      end
+
+      def dump_failures(notification)
+        return if notification.failure_notifications.empty?
+        self.class.most_recent_failures_summary = notification.fully_formatted_failed_examples
+      end
+
+      def dump_pending(notification)
+        return if notification.pending_examples.empty?
+        self.class.most_recent_pending = notification.fully_formatted_pending_examples
+      end
+    end
+  end
+end
+
+if KnapsackPro::Config::Env.modify_default_rspec_formatters?
+  class RSpec::Core::Formatters::BaseTextFormatter
+    prepend KnapsackPro::Formatters::RSpecHideFailuresAndPendingExtension
+  end
+end

data/lib/knapsack_pro/runners/queue/rspec_runner.rb

@@ -4,6 +4,7 @@ module KnapsackPro
       class RSpecRunner < BaseRunner
         def self.run(args)
           require 'rspec/core'
+          require_relative '../../formatters/rspec_queue_summary_formatter'
 
           ENV['KNAPSACK_PRO_TEST_SUITE_TOKEN'] = KnapsackPro::Config::Env.test_suite_token_rspec
           ENV['KNAPSACK_PRO_QUEUE_RECORDING_ENABLED'] = 'true'
@@ -11,7 +12,12 @@ module KnapsackPro
 
           runner = new(KnapsackPro::Adapters::RSpecAdapter)
 
-          cli_args = (args || '').split + [
+          cli_args = (args || '').split
+          # if user didn't provide the format then use explicitly default progress formatter
+          # in order to avoid KnapsackPro::Formatters::RSpecQueueSummaryFormatter being the only default formatter
+          cli_args += ['--format', 'progress'] unless cli_args.include?('--format')
+          cli_args += [
+            '--format', KnapsackPro::Formatters::RSpecQueueSummaryFormatter.to_s,
             '--default-path', runner.test_dir,
           ]
           run_tests(runner, true, cli_args, 0, [])
@@ -22,8 +28,9 @@ module KnapsackPro
 
           if test_file_paths.empty?
             unless all_test_file_paths.empty?
-              cli_args = args + all_test_file_paths
+              KnapsackPro::Formatters::RSpecQueueSummaryFormatter.print_summary
 
+              cli_args = args + all_test_file_paths
               log_rspec_command(cli_args, :end_of_queue)
             end
 

data/lib/knapsack_pro/version.rb

@@ -1,3 +1,3 @@
 module KnapsackPro
-  VERSION = '0.32.0'
+  VERSION = '0.33.0'
 end

data/spec/knapsack_pro/config/env_spec.rb

@@ -302,6 +302,40 @@ describe KnapsackPro::Config::Env do
     end
   end
 
+  describe '.modify_default_rspec_formatters' do
+    subject { described_class.modify_default_rspec_formatters }
+
+    context 'when ENV exists' do
+      let(:modify_default_rspec_formatters) { 'false' }
+      before { stub_const("ENV", { 'KNAPSACK_PRO_MODIFY_DEFAULT_RSPEC_FORMATTERS' => modify_default_rspec_formatters }) }
+      it { should eq modify_default_rspec_formatters }
+    end
+
+    context "when ENV doesn't exist" do
+      it { should be true }
+    end
+  end
+
+  describe '.modify_default_rspec_formatters?' do
+    subject { described_class.modify_default_rspec_formatters? }
+
+    before do
+      expect(described_class).to receive(:modify_default_rspec_formatters).and_return(modify_default_rspec_formatters)
+    end
+
+    context 'when enabled' do
+      let(:modify_default_rspec_formatters) { true }
+
+      it { should be true }
+    end
+
+    context 'when disabled' do
+      let(:modify_default_rspec_formatters) { false }
+
+      it { should be false }
+    end
+  end
+
   describe '.branch_encrypted' do
     subject { described_class.branch_encrypted }
 

data/spec/knapsack_pro/runners/queue/rspec_runner_spec.rb

@@ -1,4 +1,13 @@
+
 describe KnapsackPro::Runners::Queue::RSpecRunner do
+  before do
+    # we don't want to modify rspec formatters because we want to see tests summary at the end
+    # when you run this test file or whole test suite for the knapsack_pro gem
+    stub_const('ENV', { 'KNAPSACK_PRO_MODIFY_DEFAULT_RSPEC_FORMATTERS' => false })
+
+    require KnapsackPro.root + '/lib/knapsack_pro/formatters/rspec_queue_summary_formatter'
+  end
+
   describe '.run' do
     let(:test_suite_token_rspec) { 'fake-token' }
     let(:queue_id) { 'fake-queue-id' }
@@ -21,13 +30,26 @@ describe KnapsackPro::Runners::Queue::RSpecRunner do
     end
 
     context 'when args provided' do
-      let(:args) { '--example-arg example-value' }
+      context 'when format param is not provided' do
+        let(:args) { '--example-arg example-value' }
 
-      it do
-        result = double
-        expect(described_class).to receive(:run_tests).with(runner, true, ['--example-arg', 'example-value', '--default-path', 'fake-test-dir'], 0, []).and_return(result)
+        it 'uses default formatter progress' do
+          result = double
+          expect(described_class).to receive(:run_tests).with(runner, true, ['--example-arg', 'example-value', '--format', 'progress', '--format', 'KnapsackPro::Formatters::RSpecQueueSummaryFormatter', '--default-path', 'fake-test-dir'], 0, []).and_return(result)
 
-        expect(subject).to eq result
+          expect(subject).to eq result
+        end
+      end
+
+      context 'when format param is provided' do
+        let(:args) { '--format documentation' }
+
+        it 'uses provided format param instead of default formatter progress' do
+          result = double
+          expect(described_class).to receive(:run_tests).with(runner, true, ['--format', 'documentation', '--format', 'KnapsackPro::Formatters::RSpecQueueSummaryFormatter', '--default-path', 'fake-test-dir'], 0, []).and_return(result)
+
+          expect(subject).to eq result
+        end
       end
     end
 
@@ -36,7 +58,7 @@ describe KnapsackPro::Runners::Queue::RSpecRunner do
 
       it do
         result = double
-        expect(described_class).to receive(:run_tests).with(runner, true, ['--default-path', 'fake-test-dir'], 0, []).and_return(result)
+        expect(described_class).to receive(:run_tests).with(runner, true, ['--format', 'progress', '--format', 'KnapsackPro::Formatters::RSpecQueueSummaryFormatter', '--default-path', 'fake-test-dir'], 0, []).and_return(result)
 
         expect(subject).to eq result
       end
@@ -85,6 +107,7 @@ describe KnapsackPro::Runners::Queue::RSpecRunner do
         let(:exit_code) { 0 }
 
         it do
+          expect(KnapsackPro::Formatters::RSpecQueueSummaryFormatter).to receive(:print_summary)
           expect(KnapsackPro::Report).to receive(:save_node_queue_to_api)
           expect(described_class).to receive(:exit).with(exitstatus)
 
@@ -96,6 +119,7 @@ describe KnapsackPro::Runners::Queue::RSpecRunner do
         let(:exit_code) { double }
 
         it do
+          expect(KnapsackPro::Formatters::RSpecQueueSummaryFormatter).to receive(:print_summary)
           expect(KnapsackPro::Report).to receive(:save_node_queue_to_api)
           expect(described_class).to receive(:exit).with(exit_code)
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: knapsack_pro
 version: !ruby/object:Gem::Version
-  version: 0.32.0
+  version: 0.33.0
 platform: ruby
 authors:
 - ArturT
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-15 00:00:00.000000000 Z
+date: 2017-04-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -244,6 +244,7 @@ files:
 - lib/knapsack_pro/crypto/decryptor.rb
 - lib/knapsack_pro/crypto/digestor.rb
 - lib/knapsack_pro/crypto/encryptor.rb
+- lib/knapsack_pro/formatters/rspec_queue_summary_formatter.rb
 - lib/knapsack_pro/logger_wrapper.rb
 - lib/knapsack_pro/presenter.rb
 - lib/knapsack_pro/queue_allocator.rb