rspec-rebound 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2e912934fa33c3b36080535612810431e72564925d5fc9448e0a96254dc0e13a
-  data.tar.gz: 9a0b3c4a4f32979120e4da4b7a6c4cfe61cf2a57f3b0ee16da0112f2e4e06bcd
+  metadata.gz: 6321cbe02cc7af139d7eee15311cbe181db50074175a4468ebf02824ff78fc22
+  data.tar.gz: a859bf2ba2cd650d615081bed4cf4cf7562e0d1bc22642f57c460a3e88348b64
 SHA512:
-  metadata.gz: f5983ecd3470416e8f5407bab05f11b900637b66953d881734095537143a7aea4ba6f2deae807cc10dddd6169989855f3c772ec2c8ece696be58b001586273ec
-  data.tar.gz: 27527c62988cee2af25cfdac853c3456f5e288e5060d59ce131b3476e92160102cf962d5e89c1a6e0acc12013cdabf2e3e6d8a276db23b6e0cea4576b7d43b6c
+  metadata.gz: d36794943dec221c8d663e7bd23ac8fe8c306d72a4926b61a55d2a87d589741c9df7b757657083d0ce7b435c2c3dae5a38dd6524fef411de1cbbecd14e328d79
+  data.tar.gz: ea4ae48a1f2a5901b5ae8f935788218d24a0632673c5b86d4e2743b911173f8674d25d4a7c07cea9479e74b975cc156ce8ae3b61ecc7c1fea7252a80ec8d7b67

data/README.md

@@ -51,6 +51,11 @@ RSpec.configure do |config|
     ex.run_with_retry retry: 3
   end
 
+  # callback to be run when a flaky test is detected
+  config.flaky_test_callback = proc do |example|
+    Rspec::Watchdog::Reporter.report(example)
+  end
+
   # callback to be run between retries
   config.retry_callback = proc do |ex|
     # run some additional clean up task - can be filtered by example metadata
@@ -90,10 +95,11 @@ You can call `ex.run_with_retry(opts)` on an individual example.
 - __:exceptions_to_hard_fail__(default: *[]*) List of exceptions that will trigger an immediate test failure without retry. Takes precedence over __:exceptions_to_retry__
 - __:exceptions_to_retry__(default: *[]*) List of exceptions that will trigger a retry (when empty, all exceptions will)
 - __:retry_callback__(default: *nil*) Callback function to be called between retries
-
+- __:flaky_test_callback__(default: *nil*) Callback function to be called when a flaky test is detected (when a test fails but then passes on a subsequent attempt)
+- __:flaky_spec_detection__(default: *false*) If true, flaky tests will be detected and reported, even if the retry count is set to 0. This is useful for detecting flaky tests that are not being retried.
 
 ## Environment Variables
-- __RSPEC_RETRY_RETRY_COUNT__ can override the retry counts even if a retry count is set in an example or default_retry_count is set in a configuration.
+- __RSPEC_REBOUND_RETRY_COUNT__ can override the retry counts even if a retry count is set in an example or default_retry_count is set in a configuration.
 
 ## Contributing
 

data/changelog.md

@@ -92,16 +92,17 @@ rspec-retry now supports rspec 3.3. (thanks @eitoball, #32)
 include travis configuration for testing rspec 3.2.* and 3.3.*
 (thanks @eitoball, #31)
 
-# Changelog
-
-## 0.1.0 (YYYY-MM-DD)
+## 0.1.0 (2025-03-22)
 
 ### Added
 - Initial release
-- [List major features]
+- All respec-retry features
 
-### Changed
-- [List changes]
+## 0.2.0 (2025-03-29)
 
-### Fixed
-- [List fixes]
+### Added
+- Added support for RSpec 3.10
+- Added support for Ruby 3.1
+- Added support for Ruby 3.2
+- Added flaky_test_callback
+- Added flaky_spec_detection

data/lib/rspec/rebound/formatter.rb

@@ -41,14 +41,12 @@ class RSpec::Rebound::Formatter < RSpec::Core::Formatters::BaseTextFormatter
   private
 
   def increment_success(example)
-    # debugger
     previous = @tries[example]
     @tries[example] = {
       successes: previous[:successes] + 1, tries: previous[:tries] + 1 }
   end
 
   def increment_tries(example)
-    # debugger
     previous = @tries[example]
     @tries[example] = {
       successes: previous[:successes], tries: previous[:tries] + 1 }

data/lib/rspec/rebound/version.rb

@@ -1,5 +1,5 @@
 module RSpec
   class Rebound
-    VERSION = "0.1.0"
+    VERSION = "0.2.0"
   end
 end

data/lib/rspec/rebound.rb

@@ -33,6 +33,12 @@ module RSpec
         # Callback between retries
         config.add_setting :retry_callback, :default => nil
 
+        # Callback for flaky tests
+        config.add_setting :flaky_test_callback, :default => nil
+
+        # If true, flaky tests will be detected and reported, even if the retry count is set to 0. This is useful for detecting flaky tests that are not being retried.
+        config.add_setting :flaky_spec_detection, :default => false
+
         config.around(:each) do |ex|
           ex.run_with_retry
         end
@@ -54,7 +60,7 @@ module RSpec
     def retry_count
       [
           (
-          ENV['RSPEC_RETRY_RETRY_COUNT'] ||
+          ENV['RSPEC_REBOUND_RETRY_COUNT'] ||
               ex.metadata[:retry] ||
               RSpec.configuration.retry_count_condition.call(ex) ||
               RSpec.configuration.default_retry_count
@@ -122,13 +128,29 @@ module RSpec
         example.clear_exception
         ex.run
 
+        if example.exception.nil?
+          # If it's a flaky test, call the callback
+          if attempts > 0
+            if RSpec.configuration.flaky_test_callback
+              example.example_group_instance.instance_exec(example, &RSpec.configuration.flaky_test_callback)
+            end
+
+            if flaky_spec_detection?(attempts)
+              if display_try_failure_messages?
+                display_try_failure_message(example, attempts, retry_count)
+              end
+              example.exception = example.metadata[:retry_exceptions].last
+            end
+          end
+        end
+
         self.attempts += 1
 
         break if example.exception.nil?
 
         example.metadata[:retry_exceptions] << example.exception
 
-        break if attempts >= retry_count + 1
+        break if !flaky_spec_detection?(attempts) && attempts >= retry_count + 1
 
         if exceptions_to_hard_fail.any?
           break if exception_exists_in?(exceptions_to_hard_fail, example.exception)
@@ -139,17 +161,7 @@ module RSpec
         end
 
         if verbose_retry? && display_try_failure_messages?
-          if attempts != retry_count + 1
-            exception_strings =
-              if ::RSpec::Core::MultipleExceptionError::InterfaceTag === example.exception
-                example.exception.all_exceptions.map(&:to_s)
-              else
-                [example.exception.to_s]
-              end
-
-            try_message = "\n#{ordinalize(attempts)} Try error in #{example.location}:\n#{exception_strings.join "\n"}\n"
-            RSpec.configuration.reporter.message(try_message)
-          end
+          display_try_failure_message(example, attempts, retry_count)
         end
 
         example.example_group_instance.clear_lets if clear_lets
@@ -159,6 +171,12 @@ module RSpec
           example.example_group_instance.instance_exec(example, &RSpec.configuration.retry_callback)
         end
 
+        if RSpec.configuration.flaky_spec_detection
+          if attempts > 0 && example.exception.nil?
+            example.example_group_instance.instance_exec(example, &RSpec.configuration.flaky_spec_detection)
+          end
+        end
+
         sleep sleep_interval if sleep_interval.to_f > 0
       end
     end
@@ -184,6 +202,24 @@ module RSpec
         exception.is_a?(exception_klass) || exception_klass === exception
       end
     end
+
+    def flaky_spec_detection?(attempts)
+      RSpec.configuration.flaky_spec_detection? && attempts == 1
+    end
+
+    def display_try_failure_message(example, attempts, retry_count)
+      return if attempts == retry_count + 1
+      
+      exception_strings =
+        if ::RSpec::Core::MultipleExceptionError::InterfaceTag === example.exception
+          example.exception.all_exceptions.map(&:to_s)
+        else
+          [example.exception.to_s]
+        end
+
+      try_message = "\n#{ordinalize(attempts)} Try error in #{example.location}:\n#{exception_strings.join "\n"}\n"
+      RSpec.configuration.reporter.message(try_message)
+    end
   end
 end
 

data/rspec-rebound.gemspec

@@ -19,4 +19,5 @@ Gem::Specification.new do |gem|
   gem.version       = RSpec::Rebound::VERSION
   gem.add_runtime_dependency "rspec-core", "~> 3.3"
   gem.add_development_dependency "rspec", "~> 3.3"
+  gem.add_development_dependency "debug", "~> 1.0"
 end

data/spec/lib/rspec/rebound_spec.rb

@@ -26,7 +26,7 @@ describe RSpec::Rebound do
   class OtherError < StandardError; end
   class SharedError < StandardError; end
   before(:all) do
-    ENV.delete('RSPEC_RETRY_RETRY_COUNT')
+    ENV.delete('RSPEC_REBOUND_RETRY_COUNT')
   end
 
   context 'no retry option' do
@@ -81,14 +81,14 @@ describe RSpec::Rebound do
       end
     end
 
-    context 'with the environment variable RSPEC_RETRY_RETRY_COUNT' do
+    context 'with the environment variable RSPEC_REBOUND_RETRY_COUNT' do
       before(:all) do
         set_expectations([false, false, true])
-        ENV['RSPEC_RETRY_RETRY_COUNT'] = '3'
+        ENV['RSPEC_REBOUND_RETRY_COUNT'] = '3'
       end
 
       after(:all) do
-        ENV.delete('RSPEC_RETRY_RETRY_COUNT')
+        ENV.delete('RSPEC_REBOUND_RETRY_COUNT')
       end
 
       it 'should override the retry count set in an example', :retry => 2 do
@@ -378,6 +378,85 @@ describe RSpec::Rebound do
     end
   end
 
+  describe 'Flaky callback detection' do
+    let!(:example_group) do
+      RSpec.describe do
+        class ReboundResults
+          @@results = {}
+          @@flaky_test_callback_called = nil
+
+          def self.results
+            @@results
+          end
+
+          def self.flaky_test_callback_called
+            @@flaky_test_callback_called
+          end
+
+          def add(example)
+            @@results[example.description] = [example.exception.nil?, example.attempts]
+          end
+        end
+
+        def count
+          @count ||= 0
+          @count
+        end
+      
+        def count_up
+          @count ||= 0
+          @count += 1
+        end
+
+        def set_expectations(expectations)
+          @expectations = expectations
+        end
+      
+        def shift_expectation
+          @expectations.shift
+        end
+
+        before(:all) do
+          RSpec.configuration.flaky_test_callback = proc do |example|
+            ReboundResults.class_variable_set(:@@flaky_test_callback_called, example.description)
+          end
+        end
+    
+        after(:all) do
+          RSpec.configuration.flaky_test_callback = nil
+        end
+
+        around do |example|
+          example.run_with_retry
+          results = ReboundResults.results
+          results[example.description] = [example.exception.nil?, example.attempts]
+          ReboundResults.class_variable_set(:@@results, results)
+        end
+
+        before(:all) do
+          set_expectations([false, true])
+        end
+
+        specify 'without retry option', retry: 0 do
+          expect(2).to eq(count)
+        end
+
+        specify 'with retry option', retry: 1 do
+          expect(true).to be(shift_expectation)
+        end
+      end
+    end
+
+    it 'should be exposed' do
+      example_group.run
+      expect(ReboundResults.results).to eq({
+        'without retry option' => [false, 1],
+        'with retry option' => [true, 2]
+      })
+      expect(ReboundResults.flaky_test_callback_called).to eq("with retry option")
+    end
+  end
+
   describe 'output in verbose mode' do
 
     line_1 = __LINE__ + 8

data/spec/spec_helper.rb

@@ -2,10 +2,8 @@ require 'rspec'
 require 'rspec/core/sandbox'
 
 require 'rspec/rebound'
-if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2')
-  require "pry-debugger"
-else
-  require "pry-byebug"
+if Gem::Version.new(RUBY_VERSION) > Gem::Version.new('3')
+  require 'debug'
 end
 
 RSpec.configure do |config|

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rspec-rebound
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Federico Aldunate
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-03-23 00:00:00.000000000 Z
+date: 2025-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec-core
@@ -39,6 +39,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: debug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 description: A RSpec extension that automatically retries intermittently failing examples
   to reduce test flakiness and improve reliability in your test suite.
 email:
@@ -94,7 +108,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.11
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: Retry intermittently failing RSpec examples to eliminate flaky tests and