ndr_dev_support 5.5.0 → 5.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 266a4e4e0fe8d5aaf551ce205b4d699aec39922d634c34a922cdf825b2e38f0d
-  data.tar.gz: 1f07112de5ce4ecb8e18aa1b9a2013bd71d6b99e4442aeb27b8a239928242d0b
+  metadata.gz: e945839e116f3076fe950ec01ddeb9a9442bfc1bf2132460bf5d2a15d983d362
+  data.tar.gz: a39d1a0b73260aa8c45ccd48e6f3c1478ac1db86a18952084b0d35bfa2b7c8a3
 SHA512:
-  metadata.gz: d458bf7e248657f21ae473e6809288f91fd0b71dfa660b8706a199c3f4e74f06a02f850d5a3b3008ec5c7ae03545a3720a3a0917ca84f6c69793c629fe1a65bb
-  data.tar.gz: b6d391962ebe0e2c5522c4e60186f8a3c0dce42df2e11c8534db28f8e6838d1390c0910adf67007ffc419dfbbe8ee2f346e02d8c77b49366b950661d571659bb
+  metadata.gz: 2052481ce02c0fcb3947778df3132a2ce76d2848f8748970494a490926fcb8764275e953ab1f3169710325a774087deeb901bd70c676cc0b2bd1380945fb0610
+  data.tar.gz: 0a97dd68a18fbbc247a618f9c4fe23b3d66552c8971dba5dd54380f9c019d466c5cccea02bf5b498577ea8230723d95e7f36753c19c1b42556184c79ab7bc571

data/CHANGELOG.md

@@ -1,7 +1,13 @@
 ## [Unreleased]
 *no unreleased changes*
 
+## 5.6.0 / 2020-02-14
+### Added
+* Add `flakey_test` to the minitest DSL, to allow sporadic failures to be retried
+* CI: include minitest seed in slack output
+
 ## 5.5.0 / 2020-01-27
+### Added
 * bundle master RuboCop config, and allow it to be `required`
 
 ## 5.4.8 / 2020-01-24

data/README.md

@@ -135,6 +135,27 @@ Beyond standard Capybara testing DSL, ndr_dev_support bundles some additional fu
 
 When using a headless browser for integration tests, the test database must be consistent between the test runner and the application being tested. With transactional tests in operation, this means that both must share a connection. It is up to the individual project to provide this facility; as of Rails 5.1, it is built in to the framework directly.
 
+#### Flakey Tests
+
+It is an unfortunate reality that sometimes tests are written that will fail sporadically. Whilst in such cases the test methodology should be addressed, investigations can be time-consuming.
+Therefore, `ndr_dev_support` grudgingly provides "flakey test" support, to minimise disruption to CI pipelines whilst root causes are investigated.
+
+```ruby
+test 'thing that always passes' do
+  # reliable test
+end
+
+flakey_test 'thing that occassionally needs a second or third attempt' do
+  # less reliable test
+end
+
+flakey_test 'thing that often needs multiple attempts', attempts: 10 do
+  # really unreliable test (gulp...)
+end
+```
+
+If tests still fail, they'll fail as normal. If tests pass after flakey failure, they'll be flagged to the RakeCI server, and rendered in purple on Slack.
+
 ### Deployment support
 
 There are various capistrano plugins in the `ndr_dev_support/capistrano` directory - see each one for details.

data/code_safety.yml

@@ -19,7 +19,7 @@ file safety:
   CHANGELOG.md:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: aa0d11190daa8842da406b933d61c4043823ef9a
+    safe_revision: 8995f6ed25fa308e541c3d33396b0a30c4f4db80
   CODE_OF_CONDUCT.md:
     comments: 
     reviewed_by: timgentry
@@ -35,7 +35,7 @@ file safety:
   README.md:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 2cf2b6e3ce2f054289699527c13cd7d795347ce3
+    safe_revision: a2f178853da640112cf3063ca1d640157c9edb9f
   Rakefile:
     comments: 
     reviewed_by: josh.pencheon
@@ -67,7 +67,7 @@ file safety:
   lib/minitest/rake_ci_plugin.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 26efe45f35f211e7cb7a04967c51c0b6d116321e
+    safe_revision: 958ef09c5cf50bb50348e6bff949e3e5989254f9
   lib/ndr_dev_support.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -115,7 +115,7 @@ file safety:
   lib/ndr_dev_support/integration_testing.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 1ecf54219c77969066a142a42b694e805d69f8b4
+    safe_revision: a2f178853da640112cf3063ca1d640157c9edb9f
   lib/ndr_dev_support/integration_testing/drivers/chrome.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -140,6 +140,10 @@ file safety:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: f1a32b1f2d1851b87a883dbf8620aa0e921e436c
+  lib/ndr_dev_support/integration_testing/flakey_tests.rb:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: a2f178853da640112cf3063ca1d640157c9edb9f
   lib/ndr_dev_support/rake_ci/brakeman_helper.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -211,7 +215,7 @@ file safety:
   lib/ndr_dev_support/version.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: aa0d11190daa8842da406b933d61c4043823ef9a
+    safe_revision: 8995f6ed25fa308e541c3d33396b0a30c4f4db80
   lib/tasks/audit_code.rake:
     comments: Identical to the version reviewed by josh.pencheon when contained within
       ndr_support
@@ -256,7 +260,7 @@ file safety:
   lib/tasks/ci/redmine.rake:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 3c24b66ea4beb348de34330984302f1fe25faefd
+    safe_revision: 958ef09c5cf50bb50348e6bff949e3e5989254f9
   lib/tasks/ci/rugged.rake:
     comments: 
     reviewed_by: josh.pencheon
@@ -284,7 +288,7 @@ file safety:
   ndr_dev_support.gemspec:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 83097be1002f924a37fcbd6d9099e08e9ed65621
+    safe_revision: a2f178853da640112cf3063ca1d640157c9edb9f
   test/daemon/ci_server_test.rb:
     comments: 
     reviewed_by: josh.pencheon

data/lib/minitest/rake_ci_plugin.rb

@@ -5,12 +5,35 @@ require 'ndr_dev_support/rake_ci/concerns/commit_metadata_persistable'
 
 # The plugin needs to extend Minitest
 module Minitest
-  def self.plugin_rake_ci_init(_options)
-    reporter << RakeCIReporter.new if RakeCIReporter.enabled?
+  def self.plugin_rake_ci_init(options)
+    reporter << RakeCIReporter.new(options[:io], options) if RakeCIReporter.enabled?
+  end
+
+  # Intermediate Reporter than can also track flakey failures
+  class FlakeyStatisticsReporter < StatisticsReporter
+    attr_accessor :flakey_results
+
+    def initialize(*)
+      super
+
+      self.flakey_results = []
+    end
+
+    def record(result)
+      super
+
+      return unless result.respond_to?(:flakes)
+
+      flakey_results << result if result.flakes.any?
+    end
+
+    def flakes
+      flakey_results.sum { |result| result.flakes.length }
+    end
   end
 
   # RakeCI Minitest Reporter
-  class RakeCIReporter < StatisticsReporter
+  class RakeCIReporter < FlakeyStatisticsReporter
     def self.enable!
       @enabled = true
     end
@@ -55,6 +78,10 @@ module Minitest
       snippets_for results.reject(&:skipped?).reject(&:error?)
     end
 
+    def flake_snippets
+      snippets_for flakey_results
+    end
+
     # Adapted from Rails' TestUnit reporter
     def snippets_for(results, limit = 5)
       executable = defined?(Rails) ? 'bin/rails test ' : 'bundle exec rake test TEST='
@@ -78,7 +105,7 @@ module Minitest
     def current_statistics
       @current_statistics ||= {
         total_time: total_time, runs: count, assertions: assertions, failures: failures,
-        errors: errors, skips: skips
+        errors: errors, skips: skips, flakes: flakes
       }
     end
 
@@ -106,6 +133,7 @@ module Minitest
       @current_attachments << failures_attachment if failures.positive?
       @current_attachments << errors_attachment if errors.positive?
       @current_attachments << pass_attachment if passing?
+      @current_attachments << flakes_attachment if flakes.positive?
       @current_attachments
     end
 
@@ -113,7 +141,15 @@ module Minitest
       {
         color: 'danger',
         text: 'test failure'.pluralize(failures) + failure_snippets,
-        footer: 'bundle exec rake ci:minitest'
+        footer: footer
+      }
+    end
+
+    def flakes_attachment
+      {
+        color: '#bb44ff',
+        text: 'flakey test'.pluralize(flakes) + flake_snippets,
+        footer: footer
       }
     end
 
@@ -121,7 +157,7 @@ module Minitest
       {
         color: 'warning',
         text: 'test error'.pluralize(errors) + error_snippets,
-        footer: 'bundle exec rake ci:minitest'
+        footer: footer
       }
     end
 
@@ -129,7 +165,7 @@ module Minitest
       {
         color: 'good',
         text: newly_passing? ? 'Tests now pass! :tada:' : 'Tests passed',
-        footer: 'bundle exec rake ci:minitest'
+        footer: footer
       }
     end
 
@@ -150,5 +186,9 @@ module Minitest
     def name
       'minitest'
     end
+
+    def footer
+      "bundle exec rake ci:minitest --seed #{options[:seed]}"
+    end
   end
 end

data/lib/ndr_dev_support/integration_testing.rb

@@ -18,6 +18,9 @@ end
 # Include our custom DSL extensions, that also cover screenshotting:
 require 'ndr_dev_support/integration_testing/dsl'
 
+# Include support for retrying tests that sporadically fail:
+require 'ndr_dev_support/integration_testing/flakey_tests'
+
 # Keeps the selenium webdrivers automatically updated:
 require 'webdrivers'
 Webdrivers.cache_time = 24.hours

data/lib/ndr_dev_support/integration_testing/flakey_tests.rb

@@ -0,0 +1,55 @@
+module NdrDevSupport
+  module IntegrationTesting
+    # Grudging handling of flakey integration tests. Allows tests to be declared
+    # with `flakey_test`. Our CI reporter gathers information on flakey failures.
+    module FlakeyTests
+      extend ActiveSupport::Concern
+
+      included do
+        class_attribute :attempts_per_test, default: {}
+      end
+
+      class_methods do
+        def flakey_test(description, attempts: 3, &block)
+          test(description, &block).tap do |test_name|
+            self.attempts_per_test = attempts_per_test.merge(test_name.to_s => attempts)
+          end
+        end
+      end
+
+      def flakes
+        @flakes ||= []
+      end
+
+      def run
+        attempts_remaining = attempts_per_test[name]
+        return super unless attempts_remaining
+
+        previous_failure = failures.last
+        failed_attempts = []
+
+        loop do
+          break if attempts_remaining < 1
+
+          super
+
+          # No failure was added; we passed!
+          break if failures.last == previous_failure
+
+          # Ran out of attempts:
+          break if (attempts_remaining -= 1) < 1
+
+          # Loop round and have another go:
+          failed_attempts << failures.pop
+        end
+
+        # Attempts were only flakey if we eventually passed:
+        flakes.concat(failed_attempts) if failures.last == previous_failure
+
+        self
+      end
+    end
+  end
+end
+
+ActionDispatch::IntegrationTest.include(NdrDevSupport::IntegrationTesting::FlakeyTests)

data/lib/ndr_dev_support/version.rb

@@ -2,5 +2,5 @@
 # This defines the NdrDevSupport version. If you change it, rebuild and commit the gem.
 # Use "rake build" to build the gem, see rake -T for all bundler rake tasks (and our own).
 module NdrDevSupport
-  VERSION = '5.5.0'.freeze
+  VERSION = '5.6.0'.freeze
 end

data/lib/tasks/ci/redmine.rake

@@ -41,7 +41,8 @@ namespace :ci do
       resolved_tickets.map! { |ticket_id| "https://#{hostname}/issues/#{ticket_id}" }
 
       old_attachment = @attachments.detect { |att| att[:text] =~ /Tests( now)? pass/ }
-      basic_text     = old_attachment.try(:[], :text) || 'Tests passed'
+      basic_text = old_attachment.try(:[], :text) || 'Tests passed'
+      footer = old_attachment.try(:[], :footer) || 'bundle exec rake ci:minitest'
 
       @attachments.delete(old_attachment) if old_attachment
 
@@ -51,7 +52,7 @@ namespace :ci do
         title: "#{issue_s.capitalize} Resolved",
         text: "#{basic_text}, so #{issue_s} #{resolved_tickets.join(', ')}" \
               " #{resolved_tickets.count == 1 ? 'has' : 'have'} been resolved",
-        footer: 'bundle exec rake ci:minitest',
+        footer: footer,
         mrkdwn_in: ['text']
       }
       @attachments << attachment

data/ndr_dev_support.gemspec

@@ -36,6 +36,7 @@ Gem::Specification.new do |spec|
   # Integration test dependencies:
   spec.add_dependency 'capybara', '>= 3.20'
   spec.add_dependency 'capybara-screenshot'
+  spec.add_dependency 'minitest', '~> 5.0'
   spec.add_dependency 'poltergeist', '>= 1.8.0'
   spec.add_dependency 'selenium-webdriver'
   spec.add_dependency 'show_me_the_cookies'
@@ -55,7 +56,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'capistrano', '~> 2.15'
 
   spec.add_development_dependency 'bundler'
-  spec.add_development_dependency 'minitest', '~> 5.0'
   spec.add_development_dependency 'mocha'
   spec.add_development_dependency 'rake', '~> 10.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_dev_support
 version: !ruby/object:Gem::Version
-  version: 5.5.0
+  version: 5.6.0
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-01-27 00:00:00.000000000 Z
+date: 2020-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pry
@@ -136,6 +136,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
 - !ruby/object:Gem::Dependency
   name: poltergeist
   requirement: !ruby/object:Gem::Requirement
@@ -332,20 +346,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.0'
 - !ruby/object:Gem::Dependency
   name: mocha
   requirement: !ruby/object:Gem::Requirement
@@ -415,6 +415,7 @@ files:
 - lib/ndr_dev_support/integration_testing/drivers/poltergeist.rb
 - lib/ndr_dev_support/integration_testing/drivers/switchable.rb
 - lib/ndr_dev_support/integration_testing/dsl.rb
+- lib/ndr_dev_support/integration_testing/flakey_tests.rb
 - lib/ndr_dev_support/rake_ci/brakeman_helper.rb
 - lib/ndr_dev_support/rake_ci/commit_cop.rb
 - lib/ndr_dev_support/rake_ci/commit_cop/concerns/deputisable.rb