rspec-rebound 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2e912934fa33c3b36080535612810431e72564925d5fc9448e0a96254dc0e13a
+  data.tar.gz: 9a0b3c4a4f32979120e4da4b7a6c4cfe61cf2a57f3b0ee16da0112f2e4e06bcd
+SHA512:
+  metadata.gz: f5983ecd3470416e8f5407bab05f11b900637b66953d881734095537143a7aea4ba6f2deae807cc10dddd6169989855f3c772ec2c8ece696be58b001586273ec
+  data.tar.gz: 27527c62988cee2af25cfdac853c3456f5e288e5060d59ce131b3476e92160102cf962d5e89c1a6e0acc12013cdabf2e3e6d8a276db23b6e0cea4576b7d43b6c

data/.github/workflows/ci.yml

@@ -0,0 +1,53 @@
+name: Test Rspec Rebound
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        gemfile: [rspec_3.3.gemfile, rspec_3.4.gemfile, rspec_3.5.gemfile, rspec_3.7.gemfile, rspec_3.9.gemfile, rspec_3.10.gemfile, rspec_3.11.gemfile, rspec_3.12.gemfile, rspec_3.13.gemfile]
+        ruby-version: [ '2.7', '3.0', '3.1', '3.3']
+        exclude:
+          - ruby-version: '2.7'
+            gemfile: 'rspec_3.13.gemfile'
+          - ruby-version: '2.7'
+            gemfile: 'rspec_3.12.gemfile'
+          - ruby-version: '2.7'
+            gemfile: 'rspec_3.11.gemfile'
+          - ruby-version: '2.7'
+            gemfile: 'rspec_3.10.gemfile'
+          - ruby-version: '2.7'
+            gemfile: 'rspec_3.9.gemfile'
+          - ruby-version: '2.7'
+            gemfile: 'rspec_3.8.gemfile'
+          - ruby-version: '3.3'
+            gemfile: 'rspec_3.3.gemfile'
+          - ruby-version: '3.3'
+            gemfile: 'rspec_3.4.gemfile'
+          - ruby-version: '3.3'
+            gemfile: 'rspec_3.5.gemfile'
+
+    env:
+      BUNDLE_GEMFILE: spec/gemfiles/${{ matrix.gemfile }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Ruby ${{ matrix.ruby-version }}
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+          bundler-cache: true
+
+      - name: Run tests
+        run: |
+          bundle exec rspec

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.byebug_history

data/.travis.yml

@@ -0,0 +1,15 @@
+language: ruby
+rvm:
+  - 2.1
+  - 2.2
+  - 2.3
+  - 2.4
+  - 2.5
+  - 2.6
+gemfile:
+  - gemfiles/rspec_3.3.gemfile
+  - gemfiles/rspec_3.4.gemfile
+  - gemfiles/rspec_3.5.gemfile
+  - gemfiles/rspec_3.6.gemfile
+  - gemfiles/rspec_3.7.gemfile
+cache: bundler

data/Appraisals

@@ -0,0 +1,19 @@
+appraise 'rspec-3.3' do
+  gem 'rspec', '~> 3.3.0'
+end
+
+appraise 'rspec-3.4' do
+  gem 'rspec', '~> 3.4.0'
+end
+
+appraise 'rspec-3.5' do
+  gem 'rspec', '~> 3.5.0'
+end
+
+appraise 'rspec-3.6' do
+  gem 'rspec', '~> 3.6.0'
+end
+
+appraise 'rspec-3.7' do
+  gem 'rspec', '~> 3.7.0'
+end

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in rspec-retry.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,5 @@
+guard 'rspec', :version => 2, :cli => '-c -f d' do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Yusuke Mito
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,104 @@
+# RSpec::Rebound
+
+RSpec::Rebound adds a ``:retry`` option for intermittently failing rspec examples.
+If an example has the ``:retry`` option, rspec will retry the example the
+specified number of times until the example succeeds.
+
+## Acknowledgments
+
+This gem is derived from the [rspec-retry](https://github.com/NoRedInk/rspec-retry) gem by NoRedInk. We would like to express our sincere gratitude to NoRedInk for their original work. RSpec::Rebound is an updated, upgraded, and maintained version of that gem.
+
+**Important migration note:** When moving from rspec-retry to rspec-rebound, you should subtract 1 from every `retry: X` value in your tests. In rspec-rebound, `retry: 2` means up to 2 retries (3 attempts total), whereas in rspec-retry it might have meant 2 attempts total. This change makes the retry count more intuitive.
+
+### Compatibility
+
+| Rspec Version | Rspec-Rebound Version |
+|---------------|------------------------|
+| > 3.3         | 0.1.0                  |
+| < 3.3         | Use [rspec-retry](https://github.com/NoRedInk/rspec-retry) instead |
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'rspec-rebound', group: :test # Unlike rspec, this doesn't need to be included in development group
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install rspec-rebound
+
+require in ``spec_helper.rb``
+
+```ruby
+# spec/spec_helper.rb
+require 'rspec/rebound'
+
+RSpec.configure do |config|
+  # show retry status in spec process
+  config.verbose_retry = true
+  # show exception that triggers a retry if verbose_retry is set to true
+  config.display_try_failure_messages = true
+
+  # run retry only on features
+  config.around :each, :js do |ex|
+    ex.run_with_retry retry: 3
+  end
+
+  # callback to be run between retries
+  config.retry_callback = proc do |ex|
+    # run some additional clean up task - can be filtered by example metadata
+    if ex.metadata[:js]
+      Capybara.reset!
+    end
+  end
+end
+```
+
+## Usage
+
+```ruby
+it 'should randomly succeed', :retry => 2 do
+  expect(rand(2)).to eq(1)
+end
+
+it 'should succeed after a while', :retry => 2, :retry_wait => 10 do
+  expect(command('service myservice status')).to eq('started')
+end
+# run spec (following log is shown if verbose_retry options is true)
+# RSpec::Rebound: 2nd try ./spec/lib/random_spec.rb:49
+# RSpec::Rebound: 3rd try ./spec/lib/random_spec.rb:49
+```
+
+### Calling `run_with_retry` programmatically
+
+You can call `ex.run_with_retry(opts)` on an individual example.
+
+## Configuration
+
+- __:verbose_retry__(default: *false*) Print retry status
+- __:display_try_failure_messages__ (default: *false*) If verbose retry is enabled, print what reason forced the retry
+- __:default_retry_count__(default: *0*) If retry count is not set in an example, this value is used by default. This is a 'retry' count. If increased from the default of 0 to 1, if they fail all examples will be retried once (and 2 attempts in total).
+- __:default_sleep_interval__(default: *0*) Seconds to wait between retries
+- __:clear_lets_on_failure__(default: *true*) Clear memoized values for ``let``s before retrying
+- __:exceptions_to_hard_fail__(default: *[]*) List of exceptions that will trigger an immediate test failure without retry. Takes precedence over __:exceptions_to_retry__
+- __:exceptions_to_retry__(default: *[]*) List of exceptions that will trigger a retry (when empty, all exceptions will)
+- __:retry_callback__(default: *nil*) Callback function to be called between retries
+
+
+## Environment Variables
+- __RSPEC_RETRY_RETRY_COUNT__ can override the retry counts even if a retry count is set in an example or default_retry_count is set in a configuration.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a pull request

data/Rakefile

@@ -0,0 +1,12 @@
+#!/usr/bin/env rake
+require 'bundler/setup'
+require "bundler/gem_tasks"
+
+begin
+  require 'rspec/core/rake_task'
+
+  RSpec::Core::RakeTask.new(:spec)
+
+  task :default => :spec
+rescue LoadError
+end

data/changelog.md

@@ -0,0 +1,107 @@
+# 0.6.2 - 2019-11-21
+## enhancements
+expose `example.attempts` (in addition to `example.metadata[:attempts]`) (thanks @knu / #103)
+expose `example.metadata[:retry_exceptions]` (thanks @drummond-work / #106)
+
+## bugfixes / cleanup
+ci works again! (#107)
+cleanup travis.yml (thanks @olleolleolle / #105)
+
+
+# 0.6.1 - 2018-06-11
+## enhancements
+add more flexible method of retrying (thanks @varyform / #48)
+store retry attempts in example metadata (thanks @aeberlin / #43)
+
+# 0.6.0 - 2018-05-15
+## enhancements
+add exponential backoff option (thanks @patveith, @thedrow / #91, #94, #95)  
+better documentation (thanks @swrobel / #90)
+
+# 0.5.7 - 2018-03-13
+## enhancements
+remove <3.8 constraint on rspec-core to prevent breaking when rspec-core upgrades (thanks @dthorsen / #89)
+
+# 0.5.6 - 2017-10-17
+## enhancements
+added support for rspec 3.7.0
+
+# 0.5.5 - 2017-08-22
+## enhancements
+added retry_callback to help with cleanup between runs (thanks @abrom / #80)
+
+# 0.5.4 - 2017-05-08
+## enhancements
+added support for rspec 3.6.0 (thanks @dthorsen / #76)
+
+# 0.5.3 - 2017-01-11
+## enhancements
+printing summary of rspec to output not STDOUT (thanks @trevorcreech / #68)  
+removing some development dependencies
+
+# 0.5.2 - 2016-10-03
+## bugfixes
+supports versions > 3.5.0  (thanks @y-yagi / #65)
+
+# 0.5.1 - 2016-9-30
+## enhancements
+better failure message for multiple failures in one test (thanks @JonRowe / #62)
+
+# 0.5.0 - 2016-8-8
+drop support for rspec 3.2, added support for 3.4, 3.5
+
+# 0.4.6 - 2016-8-8
+## bugfixes
+failure message was off by 1 (thanks @anthonywoo, @vgrigoruk / #57)
+
+## enhancements
+add the `exceptions_to_hard_fail` options (thanks @james-dominy, @ShockwaveNN / #59)  
+add retry reporter & api for accessing retry from reporter (thanks @tdeo / #54)
+
+# 0.4.5 - 2015-11-4
+## enhancements
+retry can be called programmatically (thanks, @dwbutler / #45)
+
+# 0.4.4 - 2015-9-9
+## bugfixes
+fix gem permissions to be readable (thanks @jdelStrother / #42)
+
+# 0.4.3 - 2015-8-29
+## bugfixes
+will retry on children of exceptions in the `exceptions_to_retry` list
+(thanks @dwbutler! #40, #41)
+
+## enhancements
+setting `config.display_try_failure_messages` (and `config.verbose_retry`) will
+spit out some debug information about why an exception is being retried
+(thanks @mmorast, #24)
+
+# 0.4.2 - 2015-7-10
+## bugfixes
+setting retry to 0 will still run tests (#34)
+
+## enhancements
+can set env variable RSPEC_RETRY_RETRY_COUNT to override anything specified in
+code (thanks @sunflat, #28, #36)
+
+# 0.4.1 - 2015-7-9
+## bugfixes
+rspec-retry now supports rspec 3.3. (thanks @eitoball, #32)
+
+## dev changes
+include travis configuration for testing rspec 3.2.* and 3.3.*
+(thanks @eitoball, #31)
+
+# Changelog
+
+## 0.1.0 (YYYY-MM-DD)
+
+### Added
+- Initial release
+- [List major features]
+
+### Changed
+- [List changes]
+
+### Fixed
+- [List fixes]

data/lib/rspec/rebound/formatter.rb

@@ -0,0 +1,56 @@
+require 'rspec/core/formatters/base_text_formatter'
+
+class RSpec::Rebound::Formatter < RSpec::Core::Formatters::BaseTextFormatter
+  RSpec::Core::Formatters.register self, :example_passed
+
+  def initialize(output)
+    super(output)
+    @tries = Hash.new { |h, k| h[k] = { successes: 0, tries: 0 } }
+  end
+
+  def seed(_); end
+
+  def message(_message); end
+
+  def close(_); end
+
+  def dump_failures(_); end
+
+  def dump_pending(_); end
+
+  def dump_summary(notification)
+    summary = "\nRSpec Rebound Summary:\n"
+    @tries.each do |key, retry_data|
+      next if retry_data[:successes] < 1 || retry_data[:tries] <= 1
+      summary += "\t#{key.location}: #{key.full_description}: passed at attempt #{retry_data[:tries]}\n"
+    end
+    retried = @tries.count { |_, v| v[:tries] > 1  && v[:successes] > 0 }
+    summary += "\n\t#{retried} of #{notification.example_count} tests passed with retries.\n"
+    summary += "\t#{notification.failure_count} tests failed all retries.\n"
+    output.puts summary
+  end
+
+  def example_passed(notification)
+    increment_success notification.example
+  end
+
+  def retry(example)
+    increment_tries example
+  end
+
+  private
+
+  def increment_success(example)
+    # debugger
+    previous = @tries[example]
+    @tries[example] = {
+      successes: previous[:successes] + 1, tries: previous[:tries] + 1 }
+  end
+
+  def increment_tries(example)
+    # debugger
+    previous = @tries[example]
+    @tries[example] = {
+      successes: previous[:successes], tries: previous[:tries] + 1 }
+  end
+end

data/lib/rspec/rebound/version.rb

@@ -0,0 +1,5 @@
+module RSpec
+  class Rebound
+    VERSION = "0.1.0"
+  end
+end

data/lib/rspec/rebound.rb

@@ -0,0 +1,190 @@
+require 'rspec/core'
+require 'rspec/rebound/version'
+require 'rspec_ext/rspec_ext'
+
+module RSpec
+  class Rebound
+    def self.setup
+      RSpec.configure do |config|
+        config.add_setting :verbose_retry, :default => false
+        config.add_setting :default_retry_count, :default => 1
+        config.add_setting :default_sleep_interval, :default => 0
+        config.add_setting :exponential_backoff, :default => false
+        config.add_setting :clear_lets_on_failure, :default => true
+        config.add_setting :display_try_failure_messages, :default => false
+
+        # retry based on example metadata
+        config.add_setting :retry_count_condition, :default => ->(_) { nil }
+
+        # If a list of exceptions is provided and 'retry' > 1, we only retry if
+        # the exception that was raised by the example is NOT in that list. Otherwise
+        # we ignore the 'retry' value and fail immediately.
+        #
+        # If no list of exceptions is provided and 'retry' > 1, we always retry.
+        config.add_setting :exceptions_to_hard_fail, :default => []
+
+        # If a list of exceptions is provided and 'retry' > 1, we only retry if
+        # the exception that was raised by the example is in that list. Otherwise
+        # we ignore the 'retry' value and fail immediately.
+        #
+        # If no list of exceptions is provided and 'retry' > 1, we always retry.
+        config.add_setting :exceptions_to_retry, :default => []
+
+        # Callback between retries
+        config.add_setting :retry_callback, :default => nil
+
+        config.around(:each) do |ex|
+          ex.run_with_retry
+        end
+      end
+    end
+
+    attr_reader :context, :ex
+
+    def initialize(ex, opts = {})
+      @ex = ex
+      @ex.metadata.merge!(opts)
+      current_example.attempts ||= 0
+    end
+
+    def current_example
+      @current_example ||= RSpec.current_example
+    end
+
+    def retry_count
+      [
+          (
+          ENV['RSPEC_RETRY_RETRY_COUNT'] ||
+              ex.metadata[:retry] ||
+              RSpec.configuration.retry_count_condition.call(ex) ||
+              RSpec.configuration.default_retry_count
+          ).to_i,
+          0
+      ].max
+    end
+
+    def attempts
+      current_example.attempts ||= 0
+    end
+
+    def attempts=(val)
+      current_example.attempts = val
+    end
+
+    def clear_lets
+      !ex.metadata[:clear_lets_on_failure].nil? ?
+          ex.metadata[:clear_lets_on_failure] :
+          RSpec.configuration.clear_lets_on_failure
+    end
+
+    def sleep_interval
+      if ex.metadata[:exponential_backoff]
+          2**(current_example.attempts-1) * ex.metadata[:retry_wait]
+      else
+          ex.metadata[:retry_wait] ||
+              RSpec.configuration.default_sleep_interval
+      end
+    end
+
+    def exceptions_to_hard_fail
+      ex.metadata[:exceptions_to_hard_fail] ||
+          RSpec.configuration.exceptions_to_hard_fail
+    end
+
+    def exceptions_to_retry
+      ex.metadata[:exceptions_to_retry] ||
+          RSpec.configuration.exceptions_to_retry
+    end
+
+    def verbose_retry?
+      RSpec.configuration.verbose_retry?
+    end
+
+    def display_try_failure_messages?
+      RSpec.configuration.display_try_failure_messages?
+    end
+
+    def run
+      example = current_example
+      loop do
+        if attempts > 0
+          RSpec.configuration.formatters.each { |f| f.retry(example) if f.respond_to? :retry }
+          if verbose_retry?
+            message = "RSpec::Rebound: #{ordinalize(attempts + 1)} try #{example.location}"
+            message = "\n" + message if attempts == 1
+            RSpec.configuration.reporter.message(message)
+          end
+        end
+
+        example.metadata[:retry_attempts] = self.attempts
+        example.metadata[:retry_exceptions] ||= []
+
+        example.clear_exception
+        ex.run
+
+        self.attempts += 1
+
+        break if example.exception.nil?
+
+        example.metadata[:retry_exceptions] << example.exception
+
+        break if attempts >= retry_count + 1
+
+        if exceptions_to_hard_fail.any?
+          break if exception_exists_in?(exceptions_to_hard_fail, example.exception)
+        end
+
+        if exceptions_to_retry.any?
+          break unless exception_exists_in?(exceptions_to_retry, example.exception)
+        end
+
+        if verbose_retry? && display_try_failure_messages?
+          if attempts != retry_count + 1
+            exception_strings =
+              if ::RSpec::Core::MultipleExceptionError::InterfaceTag === example.exception
+                example.exception.all_exceptions.map(&:to_s)
+              else
+                [example.exception.to_s]
+              end
+
+            try_message = "\n#{ordinalize(attempts)} Try error in #{example.location}:\n#{exception_strings.join "\n"}\n"
+            RSpec.configuration.reporter.message(try_message)
+          end
+        end
+
+        example.example_group_instance.clear_lets if clear_lets
+
+        # If the callback is defined, let's call it
+        if RSpec.configuration.retry_callback
+          example.example_group_instance.instance_exec(example, &RSpec.configuration.retry_callback)
+        end
+
+        sleep sleep_interval if sleep_interval.to_f > 0
+      end
+    end
+
+    private
+
+    # borrowed from ActiveSupport::Inflector
+    def ordinalize(number)
+      if (11..13).include?(number.to_i % 100)
+        "#{number}th"
+      else
+        case number.to_i % 10
+        when 1; "#{number}st"
+        when 2; "#{number}nd"
+        when 3; "#{number}rd"
+        else    "#{number}th"
+        end
+      end
+    end
+
+    def exception_exists_in?(list, exception)
+      list.any? do |exception_klass|
+        exception.is_a?(exception_klass) || exception_klass === exception
+      end
+    end
+  end
+end
+
+RSpec::Rebound.setup

data/lib/rspec_ext/rspec_ext.rb

@@ -0,0 +1,39 @@
+module RSpec
+  module Core
+    class Example
+      attr_accessor :attempts
+
+      def clear_exception
+        @exception = nil
+      end
+
+      class Procsy
+        def run_with_retry(opts = {})
+          RSpec::Rebound.new(self, opts).run
+        end
+
+        def attempts
+          @example.attempts
+        end
+      end
+    end
+  end
+end
+
+module RSpec
+  module Core
+    class ExampleGroup
+      def clear_memoized
+        if respond_to? :__init_memoized, true
+          __init_memoized
+        else
+          @__memoized = nil
+        end
+      end
+
+      def clear_lets
+        clear_memoized
+      end
+    end
+  end
+end

data/rspec-rebound.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/rspec/rebound/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Federico Aldunate", 'Agustin Fornio']
+  gem.email         = ["tech@windmotion.io"]
+  gem.description   = %q{A RSpec extension that automatically retries intermittently failing examples to reduce test flakiness and improve reliability in your test suite.}
+  gem.summary       = %q{Retry intermittently failing RSpec examples to eliminate flaky tests and increase test suite stability without modifying your existing specs.}
+  gem.homepage      = "https://github.com/windmotion-io/rspec-rebound"
+  gem.license       = "MIT"
+  
+  gem.required_ruby_version = ">= 2.0"
+  
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = []
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "rspec-rebound"
+  gem.require_paths = ["lib"]
+  gem.version       = RSpec::Rebound::VERSION
+  gem.add_runtime_dependency "rspec-core", "~> 3.3"
+  gem.add_development_dependency "rspec", "~> 3.3"
+end