rspec-rewind 0.1.0 → 1.0.0

data/lib/rspec/rewind/configuration.rb

@@ -4,23 +4,58 @@ module RSpec
   module Rewind
     class Configuration
       include MatcherValidation
+      include ConfigurationValidation
 
       attr_reader :default_retries, :backoff, :retry_on, :skip_retry_on, :retry_if, :retry_callback, :flaky_callback,
-                  :verbose, :display_retry_failure_messages, :clear_lets_on_failure, :retry_budget, :flaky_reporter
+                  :not_retried_callback, :before_retry, :after_retry, :verbose, :display_retry_failure_messages,
+                  :display_retry_backtrace_top, :clear_lets_on_failure, :retry_budget, :flaky_reporter,
+                  :flaky_report_path, :retry_if_mode, :retry_on_default, :report_retry_events,
+                  :strict_callbacks, :strict_callable_arity, :metadata_report_keys, :max_retries,
+                  :max_elapsed_time, :max_total_sleep, :sleeper, :clock, :dry_run,
+                  :strict_matcher_validation, :reset_failure_policy, :retry_summary,
+                  :display_retry_summary, :fail_on_flaky, :max_flaky_examples,
+                  :freeze_configuration_at_suite_start, :warn_on_delay_conflict,
+                  :detect_retry_gem_conflicts
 
       def initialize
         self.default_retries = 0
         self.backoff = Backoff.fixed(0)
+        self.strict_matcher_validation = false
         self.retry_on = []
         self.skip_retry_on = []
         self.retry_if = nil
         self.retry_callback = nil
         self.flaky_callback = nil
+        self.not_retried_callback = nil
+        self.before_retry = nil
+        self.after_retry = nil
+        self.retry_summary = RetrySummary.new
         self.verbose = false
         self.display_retry_failure_messages = false
+        self.display_retry_backtrace_top = false
+        self.display_retry_summary = false
+        self.fail_on_flaky = false
+        self.max_flaky_examples = nil
+        self.freeze_configuration_at_suite_start = false
+        self.warn_on_delay_conflict = true
+        self.detect_retry_gem_conflicts = true
         self.clear_lets_on_failure = true
+        self.reset_failure_policy = :raise
         self.retry_budget = nil
         self.flaky_reporter = FlakyReporter.null
+        self.flaky_report_path = nil
+        self.retry_if_mode = :override
+        self.retry_on_default = :all
+        self.report_retry_events = false
+        self.strict_callbacks = false
+        self.strict_callable_arity = false
+        self.metadata_report_keys = []
+        self.max_retries = nil
+        self.max_elapsed_time = nil
+        self.max_total_sleep = nil
+        self.sleeper = Kernel.method(:sleep)
+        self.clock = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+        self.dry_run = false
       end
 
       def default_retries=(value)
@@ -32,11 +67,19 @@ module RSpec
       end
 
       def retry_on=(values)
-        @retry_on = normalize_matchers(values, field: 'retry_on')
+        @retry_on = normalize_matchers(
+          values,
+          field: 'retry_on',
+          strict_exception_matchers: @strict_matcher_validation
+        )
       end
 
       def skip_retry_on=(values)
-        @skip_retry_on = normalize_matchers(values, field: 'skip_retry_on')
+        @skip_retry_on = normalize_matchers(
+          values,
+          field: 'skip_retry_on',
+          strict_exception_matchers: @strict_matcher_validation
+        )
       end
 
       def retry_if=(callable)
@@ -51,6 +94,26 @@ module RSpec
         @flaky_callback = normalize_callable(callable, field: 'flaky_callback')
       end
 
+      def not_retried_callback=(callable)
+        @not_retried_callback = normalize_callable(callable, field: 'not_retried_callback')
+      end
+
+      def before_retry=(callable)
+        @before_retry = normalize_callable(callable, field: 'before_retry')
+      end
+
+      def after_retry=(callable)
+        @after_retry = normalize_callable(callable, field: 'after_retry')
+      end
+
+      def retry_summary=(summary)
+        unless summary.respond_to?(:record) && summary.respond_to?(:reset!)
+          raise ArgumentError, 'retry_summary must respond to #record and #reset!'
+        end
+
+        @retry_summary = summary
+      end
+
       def verbose=(value)
         @verbose = normalize_boolean(value, field: 'verbose')
       end
@@ -59,13 +122,45 @@ module RSpec
         @display_retry_failure_messages = normalize_boolean(value, field: 'display_retry_failure_messages')
       end
 
+      def display_retry_backtrace_top=(value)
+        @display_retry_backtrace_top = normalize_boolean(value, field: 'display_retry_backtrace_top')
+      end
+
+      def display_retry_summary=(value)
+        @display_retry_summary = normalize_boolean(value, field: 'display_retry_summary')
+      end
+
+      def fail_on_flaky=(value)
+        @fail_on_flaky = normalize_boolean(value, field: 'fail_on_flaky')
+      end
+
+      def max_flaky_examples=(value)
+        @max_flaky_examples = value.nil? ? nil : parse_non_negative_integer(value, source: 'max_flaky_examples')
+      end
+
+      def freeze_configuration_at_suite_start=(value)
+        @freeze_configuration_at_suite_start = normalize_boolean(value, field: 'freeze_configuration_at_suite_start')
+      end
+
+      def warn_on_delay_conflict=(value)
+        @warn_on_delay_conflict = normalize_boolean(value, field: 'warn_on_delay_conflict')
+      end
+
+      def detect_retry_gem_conflicts=(value)
+        @detect_retry_gem_conflicts = normalize_boolean(value, field: 'detect_retry_gem_conflicts')
+      end
+
       def clear_lets_on_failure=(value)
         @clear_lets_on_failure = normalize_boolean(value, field: 'clear_lets_on_failure')
       end
 
+      def reset_failure_policy=(mode)
+        @reset_failure_policy = normalize_symbol(mode, allowed: %i[raise continue], field: 'reset_failure_policy')
+      end
+
       def retry_budget=(limit_or_budget)
         @retry_budget =
-          if limit_or_budget.is_a?(RetryBudget)
+          if custom_budget?(limit_or_budget)
             limit_or_budget
           else
             RetryBudget.new(limit_or_budget)
@@ -73,52 +168,103 @@ module RSpec
       end
 
       def flaky_reporter=(reporter)
-        @flaky_reporter = reporter || FlakyReporter.null
+        normalized = reporter || FlakyReporter.null
+        raise ArgumentError, 'flaky_reporter must respond to #record' unless normalized.respond_to?(:record)
+
+        @flaky_reporter = normalized
+        @flaky_report_path = reporter_path(normalized)
       end
 
       def flaky_report_path=(path)
-        @flaky_reporter = path.nil? ? FlakyReporter.null : FlakyReporter.jsonl(path)
+        @flaky_report_path = path&.to_s
+        @flaky_reporter = @flaky_report_path.nil? ? FlakyReporter.null : FlakyReporter.jsonl(@flaky_report_path)
       end
 
-      private
+      def retry_if_mode=(mode)
+        @retry_if_mode = normalize_symbol(mode, allowed: %i[override and or], field: 'retry_if_mode')
+      end
 
-      def parse_non_negative_integer(value, source:)
-        parsed = begin
-          Integer(value)
-        rescue TypeError, ArgumentError
-          raise ArgumentError, "#{source} must be a non-negative integer"
-        end
+      def retry_on_default=(mode)
+        @retry_on_default = normalize_symbol(mode, allowed: %i[all standard_errors none], field: 'retry_on_default')
+      end
 
-        raise ArgumentError, "#{source} must be >= 0" if parsed.negative?
+      def report_retry_events=(value)
+        @report_retry_events = normalize_boolean(value, field: 'report_retry_events')
+      end
 
-        parsed
+      def strict_callbacks=(value)
+        @strict_callbacks = normalize_boolean(value, field: 'strict_callbacks')
       end
 
-      def normalize_backoff(value)
-        if value.is_a?(Numeric)
-          number = Float(value)
-          raise ArgumentError, 'backoff must be >= 0' if number.negative?
+      def strict_callable_arity=(value)
+        @strict_callable_arity = normalize_boolean(value, field: 'strict_callable_arity')
+      end
 
-          return number
-        end
+      def strict_matcher_validation=(value)
+        normalized = normalize_boolean(value, field: 'strict_matcher_validation')
+        validate_existing_matchers_for_strict_mode if normalized
+        @strict_matcher_validation = normalized
+      end
+
+      def metadata_report_keys=(values)
+        @metadata_report_keys = Array(values).flatten.compact.map(&:to_sym)
+      end
+
+      def max_retries=(value)
+        @max_retries = value.nil? ? nil : parse_non_negative_integer(value, source: 'max_retries')
+      end
 
-        return value if value.respond_to?(:call)
+      def max_elapsed_time=(value)
+        @max_elapsed_time = value.nil? ? nil : normalize_non_negative_float(value, field: 'max_elapsed_time')
+      end
+
+      def max_total_sleep=(value)
+        @max_total_sleep = value.nil? ? nil : normalize_non_negative_float(value, field: 'max_total_sleep')
+      end
 
-        raise ArgumentError, 'backoff must be a non-negative numeric value or callable'
+      def sleeper=(callable)
+        @sleeper = normalize_callable(callable, field: 'sleeper')
       end
 
-      def normalize_callable(callable, field:)
-        return nil if callable.nil?
+      def clock=(callable)
+        @clock = normalize_callable(callable, field: 'clock')
+      end
 
-        return callable if callable.respond_to?(:call)
+      def dry_run=(value)
+        @dry_run = normalize_boolean(value, field: 'dry_run')
+      end
+
+      def freeze
+        freeze_mutable_policy_state
+        super
+      end
 
-        raise ArgumentError, "#{field} must be nil or callable"
+      def snapshot
+        copy = dup
+        copy.instance_variable_set(:@retry_on, @retry_on.dup)
+        copy.instance_variable_set(:@skip_retry_on, @skip_retry_on.dup)
+        copy.instance_variable_set(:@metadata_report_keys, @metadata_report_keys.dup)
+        copy.freeze
       end
 
-      def normalize_boolean(value, field:)
-        return value if [true, false].include?(value)
+      private
+
+      def freeze_mutable_policy_state
+        @retry_on.freeze
+        @skip_retry_on.freeze
+        @metadata_report_keys.freeze
+      end
+
+      def reporter_path(reporter)
+        return nil unless reporter.respond_to?(:path)
+
+        path = reporter.path
+        path&.to_s
+      end
 
-        raise ArgumentError, "#{field} must be true or false"
+      def validate_existing_matchers_for_strict_mode
+        normalize_matchers(@retry_on || [], field: 'retry_on', strict_exception_matchers: true)
+        normalize_matchers(@skip_retry_on || [], field: 'skip_retry_on', strict_exception_matchers: true)
       end
     end
   end

data/lib/rspec/rewind/configuration_validation.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Rewind
+    module ConfigurationValidation
+      private
+
+      def parse_non_negative_integer(value, source:)
+        parsed = begin
+          Integer(value)
+        rescue TypeError, ArgumentError
+          raise ArgumentError, "#{source} must be a non-negative integer"
+        end
+
+        raise ArgumentError, "#{source} must be >= 0" if parsed.negative?
+
+        parsed
+      end
+
+      def normalize_backoff(value)
+        if value.is_a?(Numeric)
+          number = Float(value)
+          raise ArgumentError, 'backoff must be >= 0' if number.negative?
+
+          return number
+        end
+
+        return value if value.respond_to?(:call)
+
+        raise ArgumentError, 'backoff must be a non-negative numeric value or callable'
+      end
+
+      def normalize_non_negative_float(value, field:)
+        parsed = begin
+          Float(value)
+        rescue TypeError, ArgumentError
+          raise ArgumentError, "#{field} must be a numeric value"
+        end
+
+        raise ArgumentError, "#{field} must be >= 0" if parsed.negative?
+
+        parsed
+      end
+
+      def normalize_callable(callable, field:)
+        return nil if callable.nil?
+        return callable if callable.respond_to?(:call)
+
+        raise ArgumentError, "#{field} must be nil or callable"
+      end
+
+      def normalize_boolean(value, field:)
+        return value if [true, false].include?(value)
+
+        raise ArgumentError, "#{field} must be true or false"
+      end
+
+      def normalize_symbol(value, allowed:, field:)
+        symbol = value.to_sym
+        return symbol if allowed.include?(symbol)
+
+        raise ArgumentError, "#{field} must be one of #{allowed.join(', ')}"
+      rescue NoMethodError
+        raise ArgumentError, "#{field} must be one of #{allowed.join(', ')}"
+      end
+
+      def custom_budget?(value)
+        return true if value.is_a?(RetryBudget)
+        return false if value.nil? || value.is_a?(Numeric) || value.is_a?(String)
+
+        value.respond_to?(:consume!) && value.respond_to?(:remaining)
+      end
+    end
+  end
+end

data/lib/rspec/rewind/core.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+require 'rspec/core'
+require 'time'
+
+require_relative 'version'
+require_relative 'backoff'
+require_relative 'retry_budget'
+require_relative 'retry_summary'
+require_relative 'flaky_reporter'
+require_relative 'matcher_validation'
+require_relative 'configuration_validation'
+require_relative 'configuration'
+require_relative 'example_context'
+require_relative 'event'
+require_relative 'failure_fingerprint'
+require_relative 'retry_count_resolver'
+require_relative 'retry_delay_resolver'
+require_relative 'retry_event_builder'
+require_relative 'retry_notifier'
+require_relative 'flaky_transition'
+require_relative 'attempt_runner'
+require_relative 'retry_transition'
+require_relative 'retry_gate'
+require_relative 'retry_loop'
+require_relative 'runner_logger'
+require_relative 'runner_components'
+require_relative 'runner_component_factory'
+require_relative 'rspec_adapter'
+require_relative 'example_state_resetter'
+require_relative 'retry_policy'
+require_relative 'retry_decision'
+require_relative 'runner'
+require_relative 'example_methods'
+require_relative 'api'
+
+module RSpec
+  module Rewind
+    class << self
+      def configuration
+        @configuration ||= Configuration.new
+      end
+
+      def configure
+        yield(configuration)
+      end
+
+      def reset_configuration!
+        @configuration = Configuration.new
+      end
+
+      def install!
+        return false if installed?
+
+        warn_on_retry_gem_conflict
+
+        ::RSpec::Core::Example.include(ExampleMethods)
+        ::RSpec::Core::Example::Procsy.include(ExampleMethods) if defined?(::RSpec::Core::Example::Procsy)
+
+        ::RSpec.configure do |config|
+          config.before(:suite) do
+            RSpec::Rewind.prepare_suite!
+          end
+
+          config.around(:each) do |example|
+            if example.metadata[:rewind] == false
+              example.run
+            else
+              example.run_with_rewind
+            end
+          end
+
+          config.after(:suite) do
+            RSpec::Rewind.close_reporter
+          end
+        end
+
+        @installed = true
+      end
+
+      def installed?
+        !!@installed
+      end
+
+      def auto_install_disabled?
+        %w[0 false no off].include?(ENV.fetch('RSPEC_REWIND_AUTO_INSTALL', '').downcase)
+      end
+
+      def close_reporter
+        reporter = configuration.flaky_reporter
+        lifecycle_error = reporter_lifecycle_error(reporter)
+
+        publish_retry_summary
+        enforce_flaky_threshold!
+        raise lifecycle_error if lifecycle_error && configuration.strict_callbacks
+      end
+
+      def prepare_suite!
+        configuration.retry_summary.reset!
+        configuration.freeze if configuration.freeze_configuration_at_suite_start
+      end
+
+      def publish_retry_summary
+        return unless configuration.display_retry_summary
+
+        summary = configuration.retry_summary.to_message(budget: configuration.retry_budget)
+        ::RSpec.configuration.reporter.message(summary)
+      rescue StandardError
+        nil
+      end
+
+      def enforce_flaky_threshold!
+        count = configuration.retry_summary.flaky_examples
+        return unless (configuration.fail_on_flaky && count.positive?) || threshold_exceeded?(count)
+
+        raise FlakyThresholdExceeded, "rspec-rewind observed #{count} flaky example(s)"
+      end
+
+      def threshold_exceeded?(count)
+        max = configuration.max_flaky_examples
+        !max.nil? && count > max
+      end
+
+      def warn_on_retry_gem_conflict
+        return unless configuration.detect_retry_gem_conflicts
+        return unless Gem.loaded_specs.key?('rspec-retry') || defined?(::RSpec::Retry)
+
+        warn '[rspec-rewind] rspec-retry appears to be loaded; multiple retry hooks can interfere'
+      end
+
+      private
+
+      def reporter_lifecycle_error(reporter)
+        %i[flush close].filter_map do |method_name|
+          invoke_reporter_lifecycle(reporter, method_name)
+        end.first
+      end
+
+      def invoke_reporter_lifecycle(reporter, method_name)
+        return nil unless reporter.respond_to?(method_name)
+
+        reporter.public_send(method_name)
+        nil
+      rescue StandardError => e
+        warn "[rspec-rewind] flaky reporter #{method_name} failed: #{e.class}: #{e.message}"
+        e
+      end
+    end
+  end
+end

data/lib/rspec/rewind/event.rb

@@ -4,21 +4,72 @@ module RSpec
   module Rewind
     EVENT_SCHEMA_VERSION = 1
 
-    Event = Struct.new(
-      :schema_version,
-      :status,
-      :retry_reason,
-      :example_id,
-      :description,
-      :location,
-      :attempt,
-      :retries,
-      :exception_class,
-      :exception_message,
-      :duration,
-      :sleep_seconds,
-      :timestamp,
-      keyword_init: true
-    )
+    class Event
+      FIELDS = %i[
+        schema_version
+        status
+        retry_reason
+        decision_reason
+        example_id
+        description
+        location
+        attempt
+        retries
+        max_attempts
+        exception_class
+        exception_message
+        exception_backtrace_top
+        failure_fingerprint
+        duration
+        total_duration
+        attempt_durations
+        first_failure_duration
+        sleep_seconds
+        scheduled_sleep_seconds
+        actual_sleep_seconds
+        sleep_total
+        timestamp
+        budget_limit
+        budget_used
+        budget_remaining
+        matched_retry_on
+        matched_skip_retry_on
+        matcher_error
+        metadata
+      ].freeze
+
+      attr_reader(*FIELDS)
+
+      def initialize(**attributes)
+        FIELDS.each do |field|
+          instance_variable_set(:"@#{field}", immutable_value(attributes.fetch(field, nil)))
+        end
+
+        freeze
+      end
+
+      def to_h
+        FIELDS.to_h do |field|
+          [field, public_send(field)]
+        end
+      end
+
+      private
+
+      def immutable_value(value)
+        case value
+        when Array
+          value.map { |item| immutable_value(item) }.freeze
+        when Hash
+          value.each_with_object({}) do |(key, item), copy|
+            copy[immutable_value(key)] = immutable_value(item)
+          end.freeze
+        when String
+          value.dup.freeze
+        else
+          value
+        end
+      end
+    end
   end
 end

data/lib/rspec/rewind/example_state_resetter.rb

@@ -3,54 +3,26 @@
 module RSpec
   module Rewind
     class ExampleStateResetter
-      def initialize(configuration:)
+      attr_reader :last_exception
+
+      def initialize(configuration:, adapter: RSpecAdapter.new)
         @configuration = configuration
+        @adapter = adapter
+        @last_exception = nil
       end
 
       def reset(example_source)
-        clear_exception(example_source)
-        clear_execution_result(example_source)
-        clear_lets(example_source) if @configuration.clear_lets_on_failure
-      end
-
-      private
-
-      def clear_exception(example_source)
-        if example_source.respond_to?(:clear_exception)
-          example_source.clear_exception
-        elsif example_source.instance_variable_defined?(:@exception)
-          example_source.instance_variable_set(:@exception, nil)
-        end
-      end
-
-      def clear_execution_result(example_source)
-        return unless example_source.respond_to?(:execution_result)
+        @last_exception = nil
+        @adapter.clear_exception(example_source)
+        @adapter.clear_execution_result(example_source)
+        @adapter.clear_lets(example_source) if @configuration.clear_lets_on_failure
 
-        result = example_source.execution_result
-        return unless result
-
-        set_if_writer(result, :status, nil)
-        set_if_writer(result, :exception, nil)
-        set_if_writer(result, :pending_message, nil)
-        set_if_writer(result, :run_time, nil)
-      end
-
-      def clear_lets(example_source)
-        return unless example_source.respond_to?(:example_group_instance)
-
-        group_instance = example_source.example_group_instance
-        return unless group_instance
-
-        if group_instance.respond_to?(:clear_lets)
-          group_instance.clear_lets
-        elsif group_instance.instance_variable_defined?(:@__memoized)
-          group_instance.instance_variable_set(:@__memoized, nil)
-        end
-      end
+        true
+      rescue StandardError => e
+        @last_exception = e
+        raise if @configuration.reset_failure_policy == :raise
 
-      def set_if_writer(target, attribute, value)
-        writer = "#{attribute}="
-        target.public_send(writer, value) if target.respond_to?(writer)
+        false
       end
     end
   end

data/lib/rspec/rewind/failure_fingerprint.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Rewind
+    module FailureFingerprint
+      module_function
+
+      def build(exception)
+        return nil unless exception
+
+        [
+          exception.class.name,
+          exception.message,
+          exception.backtrace&.first
+        ].join(':')
+      end
+    end
+  end
+end