rspec-fortify 1.0.0.pre

data/lib/rspec/fortify.rb

@@ -0,0 +1,255 @@
+# frozen_string_literal: true
+
+require 'rspec/core'
+require 'rspec/fortify/version'
+require 'rspec_ext/rspec_ext'
+
+module RSpec
+  class Fortify
+    def self.setup # rubocop:disable Metrics/AbcSize
+      RSpec.configure do |config|
+        config.add_setting :clear_lets_on_failure, default: true
+        config.add_setting :default_retry_count, default: 1
+        config.add_setting :default_sleep_interval, default: 0
+        config.add_setting :display_try_failure_messages, default: true
+        config.add_setting :exponential_backoff, default: false
+        config.add_setting :retry_on_failure, default: main? || pr?
+        config.add_setting :retry_on_failure_count, default: 2
+        config.add_setting :retry_on_success, default: pr? && changed_specs.size < 30
+        config.add_setting :retry_on_success_count, default: 10
+        config.add_setting :verbose_retry, default: true
+
+        # retry based on example metadata
+        config.add_setting :retry_count_condition, default: ->(_) {}
+
+        # If a list of exceptions is provided and 'retry' > 1, we only retry if
+        # the exception that was raised by the example is NOT in that list. Otherwise
+        # we ignore the 'retry' value and fail immediately.
+        #
+        # If no list of exceptions is provided and 'retry' > 1, we always retry.
+        config.add_setting :exceptions_to_hard_fail, default: []
+
+        # If a list of exceptions is provided and 'retry' > 1, we only retry if
+        # the exception that was raised by the example is in that list. Otherwise
+        # we ignore the 'retry' value and fail immediately.
+        #
+        # If no list of exceptions is provided and 'retry' > 1, we always retry.
+        config.add_setting :exceptions_to_retry, default: []
+
+        # Callback between retries
+        config.add_setting :retry_callback, default: nil
+
+        config.around :each, &:run_with_retry
+      end
+    end
+
+    attr_reader :context, :ex
+
+    def initialize(example, opts = {})
+      @ex = example
+      @ex.metadata.merge!(opts)
+      current_example.attempts ||= 0
+    end
+
+    def current_example
+      @current_example ||= RSpec.current_example
+    end
+
+    def retry_count # rubocop:disable Metrics/AbcSize
+      if retry_on_success?
+        RSpec.configuration.retry_on_success_count
+      elsif retry_on_failure?
+        RSpec.configuration.retry_on_failure_count
+      else
+        [
+          (
+          ENV['RSPEC_FORTIFY_RETRY_COUNT'] ||
+              ex.metadata[:retry] ||
+              RSpec.configuration.retry_count_condition.call(ex) ||
+              RSpec.configuration.default_retry_count
+        ).to_i,
+          1,
+        ].max
+      end
+    end
+
+    def attempts
+      current_example.attempts ||= 0
+    end
+
+    def attempts=(val)
+      current_example.attempts = val
+    end
+
+    def clear_lets
+      if ex.metadata[:clear_lets_on_failure].nil?
+        RSpec.configuration.clear_lets_on_failure
+      else
+        ex.metadata[:clear_lets_on_failure]
+      end
+    end
+
+    def sleep_interval
+      if ex.metadata[:exponential_backoff]
+        (2**(current_example.attempts - 1)) * ex.metadata[:retry_wait]
+      else
+        ex.metadata[:retry_wait] ||
+          RSpec.configuration.default_sleep_interval
+      end
+    end
+
+    def exceptions_to_hard_fail
+      ex.metadata[:exceptions_to_hard_fail] ||
+        RSpec.configuration.exceptions_to_hard_fail
+    end
+
+    def exceptions_to_retry
+      ex.metadata[:exceptions_to_retry] ||
+        RSpec.configuration.exceptions_to_retry
+    end
+
+    def verbose_retry?
+      RSpec.configuration.verbose_retry?
+    end
+
+    def display_try_failure_messages?
+      RSpec.configuration.display_try_failure_messages?
+    end
+
+    def run # rubocop:disable Metrics/PerceivedComplexity, Metrics/AbcSize, Metrics/CyclomaticComplexity
+      loop do
+        RSpec.configuration.formatters.each { |f| f.retry(ex) if f.respond_to? :retry } if attempts.positive?
+
+        if verbose_retry? && (log_first_attempt? || attempts.positive?)
+          message = "RSpec::Fortify: #{ordinalize(attempts + 1)} try #{ex.location}"
+          message = "\n#{message}" if attempts == 1
+          RSpec.configuration.reporter.message(message)
+        end
+
+        ex.metadata[:retry_attempts] = attempts
+        ex.metadata[:retry_exceptions] ||= []
+
+        current_example.clear_exception
+        ex.run
+
+        self.attempts += 1
+
+        break unless should_retry?
+
+        ex.metadata[:retry_exceptions] << ex.exception if ex.exception
+
+        break if attempts >= retry_count
+
+        if exceptions_to_hard_fail.any? && ex.exception && exception_exists_in?(exceptions_to_hard_fail, ex.exception)
+          break
+        end
+
+        break if exceptions_to_retry.any? && ex.exception && !exception_exists_in?(exceptions_to_retry, ex.exception)
+
+        if verbose_retry? && display_try_failure_messages? && retry_on_failure? && (attempts != retry_count)
+          exception_strings =
+            if ex.exception.is_a?(::RSpec::Core::MultipleExceptionError::InterfaceTag)
+              ex.exception.all_exceptions.map(&:to_s)
+            else
+              [ex.exception.to_s]
+            end
+
+          try_message = "\n#{ordinalize(attempts)} Try error in #{ex.location}:\n#{exception_strings.join "\n"}\n"
+          RSpec.configuration.reporter.message(try_message)
+        end
+
+        ex.example_group_instance.clear_lets if clear_lets
+
+        if RSpec.configuration.retry_callback
+          ex.ex_group_instance.instance_exec(ex, &RSpec.configuration.retry_callback)
+        end
+
+        sleep sleep_interval if sleep_interval.to_f.positive?
+      end
+    end
+
+    private
+
+    # borrowed from ActiveSupport::Inflector
+    def ordinalize(number)
+      if (11..13).cover?(number.to_i % 100)
+        "#{number}th"
+      else
+        case number.to_i % 10
+          when 1 then "#{number}st"
+          when 2 then "#{number}nd"
+          when 3 then "#{number}rd"
+          else "#{number}th"
+        end
+      end
+    end
+
+    def exception_exists_in?(list, exception)
+      list.any? do |exception_klass|
+        exception.is_a?(exception_klass) || exception_klass === exception # rubocop:disable Style/CaseEquality
+      end
+    end
+
+    def log_first_attempt?
+      cast_to_boolean(ENV.fetch('RSPEC_FORTIFY_LOG_FIRST_ATTEMPT', 'false'))
+    end
+
+    def retry_on_failure?
+      RSpec.configuration.retry_on_failure
+    end
+
+    def retry_on_success?
+      RSpec.configuration.retry_on_success && current_example_changed?
+    end
+
+    def current_example_changed?
+      # Added or modified specs can be passed in via this
+      # CHANGED_SPECS env var using some version of git diff like:
+      # git diff origin/main...HEAD --name-only --relative --diff-filter=AM | grep '_spec.rb$' | tr '\n' ',' | sed 's/,$//'
+      changed_specs = ENV.fetch('CHANGED_SPECS', nil)&.split(',') || []
+      changed_specs.include?(ex.file_path.sub(%r{^\./}, ''))
+    end
+
+    def current_attempt_failed?
+      !ex.exception.nil?
+    end
+
+    def current_attempt_succeeded?
+      ex.exception.nil?
+    end
+
+    def should_retry?
+      if retry_on_success?
+        current_attempt_succeeded?
+      elsif retry_on_failure?
+        current_attempt_failed?
+      end
+    end
+  end
+end
+
+def cast_to_boolean(value)
+  if value.nil? || %w(false f 0 no n).include?(value.to_s.downcase) # rubocop:disable Style/IfWithBooleanLiteralBranches
+    false
+  else
+    true
+  end
+end
+
+def ci?
+  cast_to_boolean(ENV.fetch('CI', 'false'))
+end
+
+def pr?
+  ci? && cast_to_boolean(ENV.fetch('CIRCLE_PULL_REQUEST', 'false'))
+end
+
+def main?
+  ci? && !pr?
+end
+
+def changed_specs
+  ENV.fetch('CHANGED_SPECS', nil)&.split(',') || []
+end
+
+RSpec::Fortify.setup

data/lib/rspec_ext/rspec_ext.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Core
+    class Example
+      attr_accessor :attempts
+
+      def clear_exception
+        @exception = nil
+      end
+
+      class Procsy
+        def run_with_retry(opts = {})
+          RSpec::Fortify.new(self, opts).run
+        end
+
+        def attempts
+          @example.attempts
+        end
+      end
+    end
+  end
+end
+
+module RSpec
+  module Core
+    class ExampleGroup
+      def clear_memoized
+        if respond_to? :__init_memoized, true
+          __init_memoized
+        else
+          @__memoized = nil
+        end
+      end
+
+      def clear_lets
+        clear_memoized
+      end
+    end
+  end
+end

data/rspec-fortify.gemspec

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'English'
+require File.expand_path('lib/rspec/fortify/version', __dir__)
+
+Gem::Specification.new do |spec|
+  spec.authors       = ['Yusuke Mito', 'Michael Glass', 'Devin Burnette']
+  spec.email         = ['devin@betterment.com']
+  spec.description   = 'retry intermittently failing rspec examples'
+  spec.summary       = 'retry intermittently failing rspec examples'
+  spec.homepage      = 'https://github.com/Betterment/rspec-fortify'
+  spec.license       = 'MIT'
+  spec.metadata['rubygems_mfa_required'] = 'true'
+
+  spec.files         = `git ls-files`.split($OUTPUT_RECORD_SEPARATOR)
+  spec.executables   = []
+  spec.name          = 'rspec-fortify'
+  spec.require_paths = ['lib']
+  spec.version       = RSpec::Fortify::VERSION
+  spec.required_ruby_version = '>= 3.0'
+  spec.add_runtime_dependency 'rspec-core', '>3.9'
+end

data/spec/fixtures/bad_test.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe 'Bad Test Example' do # rubocop:disable RSpec/FilePath
+  let(:foo) { false }
+
+  it 'is a bad test' do
+    expect(foo).to eq true
+  end
+end

data/spec/fixtures/flaky_test.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe 'Flaky Test Example' do # rubocop:disable RSpec/FilePath
+  before(:all) do # rubocop:disable RSpec/BeforeAfterAll
+    $try_counter = 0 # rubocop:disable Style/GlobalVars
+  end
+
+  it 'fails on the third try' do
+    $try_counter += 1 # rubocop:disable Style/GlobalVars
+
+    expect($try_counter).not_to eq(3), 'Intentionally failing on the third try' # rubocop:disable Style/GlobalVars
+  end
+end

data/spec/fixtures/good_test.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe 'Good Test Example' do # rubocop:disable RSpec/FilePath
+  let(:foo) { true }
+
+  it 'is a good test' do
+    expect(foo).to eq true
+  end
+end

data/spec/lib/rspec/fortify_spec.rb

@@ -0,0 +1,212 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'tty-command'
+
+describe RSpec::Fortify do
+  let(:cmd) { TTY::Command.new printer: :null }
+
+  describe '#run' do
+    context 'when main' do
+      context 'when example changed' do
+        let(:env) do
+          {
+            'CHANGED_SPECS' => 'spec/fixtures/flaky_test.rb,spec/fixtures/good_test.rb,spec/fixtures/bad_test.rb',
+            'CI' => 'true',
+            'RSPEC_FORTIFY_LOG_FIRST_ATTEMPT' => 'true',
+          }
+        end
+
+        it 'retries bad test examples the configured failed retry amount' do
+          out, _err = cmd.run!('bundle exec rspec spec/fixtures/bad_test.rb', env: env)
+          expect(out).to include('2nd try')
+          expect(out).not_to include('10th try')
+        end
+
+        it 'runs good test examples once' do
+          out, _err = cmd.run!('bundle exec rspec spec/fixtures/good_test.rb', env: env)
+          expect(out).to include('1st try')
+          expect(out).not_to include('2nd try')
+        end
+
+        it 'runs flaky test examples once' do
+          out, _err = cmd.run!('bundle exec rspec spec/fixtures/flaky_test.rb', env: env)
+          expect(out).to include('1st try')
+          expect(out).not_to include('2nd try')
+        end
+      end
+
+      context 'when example did not change' do
+        let(:env) do
+          {
+            'CI' => 'true',
+            'RSPEC_FORTIFY_LOG_FIRST_ATTEMPT' => 'true',
+          }
+        end
+
+        it 'retries bad test examples the configured failed retry amount' do
+          out, _err = cmd.run!('bundle exec rspec spec/fixtures/bad_test.rb', env: env)
+          expect(out).to include('2nd try')
+          expect(out).not_to include('10th try')
+        end
+
+        it 'runs good test examples once' do
+          out, _err = cmd.run!('bundle exec rspec spec/fixtures/good_test.rb', env: env)
+          expect(out).to include('1st try')
+          expect(out).not_to include('2nd try')
+        end
+
+        it 'runs flaky test examples once' do
+          out, _err = cmd.run!('bundle exec rspec spec/fixtures/flaky_test.rb', env: env)
+          expect(out).to include('1st try')
+          expect(out).not_to include('2nd try')
+        end
+      end
+    end
+
+    context 'when pr' do
+      context 'when example changed' do
+        context 'when too many changed specs' do
+          let(:env) do
+            {
+              'CHANGED_SPECS' => 'spec/fixtures/flaky_test.rb,spec/fixtures/good_test.rb,spec/fixtures/bad_test.rb,spec/fixtures/fake_test1.rb,spec/fixtures/fake_test2.rb,spec/fixtures/fake_test3.rb,spec/fixtures/fake_test4.rb,spec/fixtures/fake_test5.rb,spec/fixtures/fake_test6.rb,spec/fixtures/fake_test7.rb,spec/fixtures/fake_test8.rb,spec/fixtures/fake_test9.rb,spec/fixtures/fake_test10.rb,spec/fixtures/fake_test11.rb,spec/fixtures/fake_test12.rb,spec/fixtures/fake_test13.rb,spec/fixtures/fake_test14.rb,spec/fixtures/fake_test15.rb,spec/fixtures/fake_test16.rb,spec/fixtures/fake_test17.rb,spec/fixtures/fake_test18.rb,spec/fixtures/fake_test19.rb,spec/fixtures/fake_test20.rb,spec/fixtures/fake_test21.rb,spec/fixtures/fake_test22.rb,spec/fixtures/fake_test23.rb,spec/fixtures/fake_test24.rb,spec/fixtures/fake_test25.rb,spec/fixtures/fake_test26.rb,spec/fixtures/fake_test27.rb,spec/fixtures/fake_test28.rb,spec/fixtures/fake_test29.rb,spec/fixtures/fake_test30.rb', # rubocop:disable Layout/LineLength
+              'CI' => 'true',
+              'CIRCLE_PULL_REQUEST' => 'https://github.com/foo/bar/pull/123',
+              'RSPEC_FORTIFY_LOG_FIRST_ATTEMPT' => 'true',
+            }
+          end
+
+          it 'retries bad test examples the configured failed retry amount' do
+            out, _err = cmd.run!('bundle exec rspec spec/fixtures/bad_test.rb', env: env)
+            expect(out).to include('2nd try')
+            expect(out).not_to include('10th try')
+          end
+
+          it 'runs good test examples once' do
+            out, _err = cmd.run!('bundle exec rspec spec/fixtures/good_test.rb', env: env)
+            expect(out).to include('1st try')
+            expect(out).not_to include('2nd try')
+          end
+
+          it 'runs flaky test examples once' do
+            out, _err = cmd.run!('bundle exec rspec spec/fixtures/flaky_test.rb', env: env)
+            expect(out).to include('1st try')
+            expect(out).not_to include('2nd try')
+          end
+        end
+
+        context 'when not too many changed specs' do
+          let(:env) do
+            {
+              'CHANGED_SPECS' => 'spec/fixtures/flaky_test.rb,spec/fixtures/good_test.rb,spec/fixtures/bad_test.rb',
+              'CI' => 'true',
+              'CIRCLE_PULL_REQUEST' => 'https://github.com/foo/bar/pull/123',
+              'RSPEC_FORTIFY_LOG_FIRST_ATTEMPT' => 'true',
+            }
+          end
+
+          it 'retries bad test examples the configured failed retry amount' do
+            out, _err = cmd.run!('bundle exec rspec spec/fixtures/bad_test.rb', env: env)
+            expect(out).to include('1st try')
+            expect(out).not_to include('2nd try')
+          end
+
+          it 'retries good test examples the configured success retry amount' do
+            out, _err = cmd.run!('bundle exec rspec spec/fixtures/good_test.rb', env: env)
+            expect(out).to include('10th try')
+          end
+
+          it 'retries flaky test examples until they flake' do
+            out, _err = cmd.run!('bundle exec rspec spec/fixtures/flaky_test.rb', env: env)
+            expect(out).to include('2nd try')
+          end
+        end
+      end
+
+      context 'when example did not change' do
+        let(:env) do
+          {
+            'CI' => 'true',
+            'CIRCLE_PULL_REQUEST' => 'https://github.com/foo/bar/pull/123',
+            'RSPEC_FORTIFY_LOG_FIRST_ATTEMPT' => 'true',
+          }
+        end
+
+        it 'retries bad test examples the configured failed retry amount' do
+          out, _err = cmd.run!('bundle exec rspec spec/fixtures/bad_test.rb', env: env)
+          expect(out).to include('2nd try')
+          expect(out).not_to include('10th try')
+        end
+
+        it 'runs good test examples once' do
+          out, _err = cmd.run!('bundle exec rspec spec/fixtures/good_test.rb', env: env)
+          expect(out).to include('1st try')
+          expect(out).not_to include('2nd try')
+        end
+
+        it 'runs flaky test examples once' do
+          out, _err = cmd.run!('bundle exec rspec spec/fixtures/flaky_test.rb', env: env)
+          expect(out).to include('1st try')
+          expect(out).not_to include('2nd try')
+        end
+      end
+    end
+
+    context 'when local dev' do
+      context 'when example changed' do
+        let(:env) do
+          {
+            'CHANGED_SPECS' => 'spec/fixtures/flaky_test.rb,spec/fixtures/good_test.rb,spec/fixtures/bad_test.rb',
+            'CI' => 'false',
+            'RSPEC_FORTIFY_LOG_FIRST_ATTEMPT' => 'true',
+          }
+        end
+
+        it 'runs bad test examples once' do
+          out, _err = cmd.run!('bundle exec rspec spec/fixtures/bad_test.rb', env: env)
+          expect(out).to include('1st try')
+          expect(out).not_to include('2nd try')
+        end
+
+        it 'runs good test examples once' do
+          out, _err = cmd.run!('bundle exec rspec spec/fixtures/good_test.rb', env: env)
+          expect(out).to include('1st try')
+          expect(out).not_to include('2nd try')
+        end
+
+        it 'runs flaky test examples once' do
+          out, _err = cmd.run!('bundle exec rspec spec/fixtures/flaky_test.rb', env: env)
+          expect(out).to include('1st try')
+          expect(out).not_to include('2nd try')
+        end
+      end
+
+      context 'when example did not change' do
+        let(:env) do
+          {
+            'CI' => 'false',
+            'RSPEC_FORTIFY_LOG_FIRST_ATTEMPT' => 'true',
+          }
+        end
+
+        it 'runs bad test examples once' do
+          out, _err = cmd.run!('bundle exec rspec spec/fixtures/bad_test.rb', env: env)
+          expect(out).to include('1st try')
+          expect(out).not_to include('2nd try')
+        end
+
+        it 'runs good test examples once' do
+          out, _err = cmd.run!('bundle exec rspec spec/fixtures/good_test.rb', env: env)
+          expect(out).to include('1st try')
+          expect(out).not_to include('2nd try')
+        end
+
+        it 'runs flaky test examples once' do
+          out, _err = cmd.run!('bundle exec rspec spec/fixtures/flaky_test.rb', env: env)
+          expect(out).to include('1st try')
+          expect(out).not_to include('2nd try')
+        end
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require 'rspec/fortify'

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification
+name: rspec-fortify
+version: !ruby/object:Gem::Version
+  version: 1.0.0.pre
+platform: ruby
+authors:
+- Yusuke Mito
+- Michael Glass
+- Devin Burnette
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2025-06-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '3.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '3.9'
+description: retry intermittently failing rspec examples
+email:
+- devin@betterment.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/ci.yml"
+- ".github/workflows/linter.yml"
+- ".gitignore"
+- ".rubocop.yml"
+- ".ruby-version"
+- ".travis.yml"
+- Appraisals
+- Gemfile
+- Guardfile
+- LICENSE
+- README.md
+- Rakefile
+- changelog.md
+- gemfiles/rspec_3.10.gemfile
+- gemfiles/rspec_3.10.gemfile.lock
+- gemfiles/rspec_3.11.gemfile
+- gemfiles/rspec_3.11.gemfile.lock
+- gemfiles/rspec_3.12.gemfile
+- gemfiles/rspec_3.12.gemfile.lock
+- gemfiles/rspec_3.13.gemfile
+- gemfiles/rspec_3.13.gemfile.lock
+- gemfiles/rspec_3.9.gemfile
+- gemfiles/rspec_3.9.gemfile.lock
+- lib/rspec/fortify.rb
+- lib/rspec/fortify/formatter.rb
+- lib/rspec/fortify/version.rb
+- lib/rspec_ext/rspec_ext.rb
+- rspec-fortify.gemspec
+- spec/fixtures/bad_test.rb
+- spec/fixtures/flaky_test.rb
+- spec/fixtures/good_test.rb
+- spec/lib/rspec/fortify_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/Betterment/rspec-fortify
+licenses:
+- MIT
+metadata:
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubygems_version: 3.3.26
+signing_key:
+specification_version: 4
+summary: retry intermittently failing rspec examples
+test_files: []