stoplight 5.5.0 → 5.7.0

data/lib/stoplight/domain/strategies/yellow_run_strategy.rb

@@ -15,9 +15,17 @@ module Stoplight
         #   @return [Stoplight::Domain::Config] The configuration for the light.
         protected attr_reader :config
 
-        # @!attribute [r] data_store
-        #   @return [Stoplight::DataStore::Base] The data store associated with the light.
-        protected attr_reader :data_store
+        # @!attribute [r] stare_store
+        #   @return [Stoplight::Domain::Storage::State]
+        protected attr_reader :state_store
+
+        # @!attribute [r] metrics_store
+        #   @return [Stoplight::Domain::Storage::Metrics]
+        protected attr_reader :metrics_store
+
+        # @!attribute [r] recovery_lock_store
+        #   @return [Stoplight::Domain::Storage::RecoveryLock]
+        protected attr_reader :recovery_lock_store
 
         # @!attribute [r] notifiers
         #   @return [Stoplight::Domain::StateTransitionNotifier]
@@ -27,40 +35,68 @@ module Stoplight
         #   @return [Stoplight::Domain::RecoveryProbeRequestRecorder]
         protected attr_reader :request_tracker
 
+        # @!attribute [r] red_run_strategy
+        #   @return [Stoplight::Domain::Strategies::RedRunStrategy]
+        protected attr_reader :red_run_strategy
+
         # @param config [Stoplight::Domain::Config]
-        # @param data_store [Stoplight::DataStore::Base]
         # @param notifiers [Array<Stoplight::Domain::StateTransitionNotifier>]
         # @param request_tracker [Stoplight::Domain::Tracker::RecoveryProbe]
-        def initialize(config:, data_store:, notifiers:, request_tracker:)
+        # @param red_run_strategy [Stoplight::Domain::Strategies::RedRunStrategy]
+        # @param recovery_lock_store [Stoplight::Domain::Storage::RecoveryLock]
+        def initialize(config:, notifiers:, request_tracker:, red_run_strategy:, state_store:, metrics_store:, recovery_lock_store:)
           @config = config
-          @data_store = data_store
           @notifiers = notifiers
           @request_tracker = request_tracker
+          @red_run_strategy = red_run_strategy
+          @state_store = state_store
+          @metrics_store = metrics_store
+          @recovery_lock_store = recovery_lock_store
         end
 
         # Executes the provided code block when the light is in the yellow state.
         #
         # @param fallback [Proc, nil] A fallback proc to execute in case of an error.
-        # @param metadata [Stoplight::Domain::Metadata] Metadata capturing the current state of the light.
+        # @param state_snapshot [Stoplight::Domain::StateSnapshot]
         # @yield The code block to execute.
         # @return [Object] The result of the code block if successful.
         # @raise [Exception] Re-raises the error if it is not tracked or no fallback is provided.
-        def execute(fallback, metadata:, &code)
-          enter_recovery(metadata)
-          # TODO: We need to employ a probabilistic approach here to avoid "thundering herd" problem
-          code.call.tap { record_recovery_probe_success }
-        rescue => error
-          if config.track_error?(error)
-            record_recovery_probe_failure(error)
-
-            if fallback
-              fallback.call(error)
+        def execute(fallback, state_snapshot:, &code)
+          # Everything withing this block executed exclusively:
+          #   - enter recovery
+          #   - execute user's code
+          #   - record outcome
+          #   - transition to green or red if needed
+          with_recovery_lock(fallback:, state_snapshot:) do
+            enter_recovery(state_snapshot)
+
+            code.call.tap { record_recovery_probe_success }
+          rescue => error
+            if config.track_error?(error)
+              record_recovery_probe_failure(error)
+
+              if fallback
+                fallback.call(error)
+              else
+                raise
+              end
             else
+              record_recovery_probe_success
               raise
             end
-          else
-            record_recovery_probe_success
-            raise
+          end
+        end
+
+        def with_recovery_lock(fallback:, state_snapshot:)
+          recovery_lock_token = recovery_lock_store.acquire_lock
+          if recovery_lock_token.nil?
+            return red_run_strategy.execute(fallback, state_snapshot:)
+          end
+
+          begin
+            yield
+          ensure
+            recovery_lock_store.release_lock(recovery_lock_token)
           end
         end
 
@@ -72,26 +108,17 @@ module Stoplight
           request_tracker.record_failure(error)
         end
 
-        # @param metadata [Stoplight::Domain::Metadata]
+        # @param state_snapshot [Stoplight::Domain::StateSnapshot]
         # @return [void]
-        private def enter_recovery(metadata)
-          return if metadata.recovery_started?
+        private def enter_recovery(state_snapshot)
+          return if state_snapshot.recovery_started?
 
-          if data_store.transition_to_color(config, Color::YELLOW)
-            notifiers.each do |notifier|
-              notifier.notify(config, Color::RED, Color::YELLOW, nil)
-            end
+          state_store.transition_to_color(Color::YELLOW)
+          metrics_store.clear
+          notifiers.each do |notifier|
+            notifier.notify(config, Color::RED, Color::YELLOW, nil)
           end
         end
-
-        # @return [Boolean]
-        def ==(other)
-          super &&
-            config == other.config &&
-            notifiers == other.notifiers &&
-            data_store == other.data_store &&
-            request_tracker == other.request_tracker
-        end
       end
     end
   end

data/lib/stoplight/domain/tracker/base.rb

@@ -6,35 +6,6 @@ module Stoplight
       # @api private
       # @abstract
       class Base
-        # @!attribute [r] data_store
-        #   @return [Stoplight::DataStore::Base] The data store associated with the light.
-        protected attr_reader :data_store
-
-        # @!attribute [r] traffic_control
-        #   @return [Stoplight::Domain::TrafficControl::Base]
-        protected attr_reader :notifiers
-
-        # @!attribute [r] config
-        #   @return [Stoplight::Domain::Config] The configuration for the light.
-        protected attr_reader :config
-
-        def ==(other)
-          other.is_a?(self.class) &&
-            config == other.config &&
-            data_store == other.data_store &&
-            notifiers == other.notifiers
-        end
-
-        def transition_and_notify(from_color, to_color, error = nil)
-          if data_store.transition_to_color(config, to_color)
-            notifiers.each do |notifier|
-              notifier.notify(config, from_color, to_color, error)
-            end
-            true
-          else
-            false
-          end
-        end
       end
     end
   end

data/lib/stoplight/domain/tracker/recovery_probe.rb

@@ -4,10 +4,6 @@ module Stoplight
   module Domain
     module Tracker
       class RecoveryProbe < Base
-        # @!attribute [r] data_store
-        #   @return [Stoplight::DataStore::Base] The data store associated with the light.
-        protected attr_reader :data_store
-
         # @!attribute [r] traffic_recovery
         #   @return [Stoplight::Domain::TrafficRecovery::Base]
         protected attr_reader :traffic_recovery
@@ -20,51 +16,59 @@ module Stoplight
         #   @return [Stoplight::Domain::Config] The configuration for the light.
         protected attr_reader :config
 
-        # @param data_store [Stoplight::Domain::DataStore]
+        # @!attribute [r] metrics_store
+        #   @return [Stoplight::Domain::Storage::Metrics]
+        protected attr_reader :metrics_store
+
+        # @!attribute [r] state_store
+        #   @return [Stoplight::Domain::Storage::State]
+        protected attr_reader :state_store
+
         # @param traffic_recovery [Stoplight::Domain::TrafficRecovery::Base]
         # @param notifiers [<Stoplight::Domain::StateTransitionNotifier>]
         # @param config [Stoplight::Domain::Config]
-        def initialize(data_store:, traffic_recovery:, notifiers:, config:)
-          @data_store = data_store
+        # @param metrics_store [Stoplight::Domain::Storage::Metrics]
+        # @param state_store [Stoplight::Domain::Storage::State]
+        def initialize(traffic_recovery:, notifiers:, config:, metrics_store:, state_store:)
           @traffic_recovery = traffic_recovery
           @notifiers = notifiers
           @config = config
+          @metrics_store = metrics_store
+          @state_store = state_store
         end
 
         # @param exception [Exception]
         def record_failure(exception)
-          metadata = data_store.record_recovery_probe_failure(config, exception)
+          metrics_store.record_failure(exception)
 
-          recover(metadata)
+          recover
         end
 
         def record_success
-          metadata = data_store.record_recovery_probe_success(config)
+          metrics_store.record_success
 
-          recover(metadata)
+          recover
         end
         RECOVERY_TRANSITIONS = {
           TrafficRecovery::GREEN => [Color::YELLOW, Color::GREEN],
-          TrafficRecovery::YELLOW => [Color::RED, Color::YELLOW],
           TrafficRecovery::RED => [Color::YELLOW, Color::RED]
         }.freeze
 
-        private def recover(metadata)
-          recovery_result = traffic_recovery.determine_color(config, metadata)
+        private def recover
+          recovery_metrics = metrics_store.metrics_snapshot
+          recovery_result = traffic_recovery.determine_color(config, recovery_metrics)
 
-          return if recovery_result == TrafficRecovery::PASS
+          return if recovery_result == TrafficRecovery::YELLOW
 
           from_color, to_color = RECOVERY_TRANSITIONS.fetch(recovery_result) do
             raise "recovery strategy returned unexpected color: #{recovery_result}"
           end
 
-          transition_and_notify(from_color, to_color, nil)
-        end
-
-        # @param other [any]
-        # @return [bool]
-        def ==(other)
-          super && traffic_recovery == other.traffic_recovery
+          state_store.transition_to_color(to_color)
+          metrics_store.clear
+          notifiers.each do |notifier|
+            notifier.notify(config, from_color, to_color, nil)
+          end
         end
       end
     end

data/lib/stoplight/domain/tracker/request.rb

@@ -10,10 +10,6 @@ module Stoplight
       #
       # @api private
       class Request < Base
-        # @!attribute [r] data_store
-        #   @return [Stoplight::DataStore::Base] The data store associated with the light.
-        protected attr_reader :data_store
-
         # @!attribute [r] traffic_control
         #   @return [Stoplight::Domain::TrafficControl::Base]
         protected attr_reader :traffic_control
@@ -26,41 +22,50 @@ module Stoplight
         #   @return [Stoplight::Domain::Config] The configuration for the light.
         protected attr_reader :config
 
-        # @param data_store [Stoplight::Domain::DataStore]
+        # @!attribute metrics_store
+        #   @return [Stoplight::Storage::Metrics]
+        protected attr_reader :metrics_store
+
+        # @!attribute [r] state_store
+        #   @return [Stoplight::Domain::Storage::State]
+        protected attr_reader :state_store
+
         # @param traffic_control [Stoplight::Domain::TrafficControl::Base]
         # @param notifiers [<Stoplight::Domain::StateTransitionNotifier>]
         # @param config [Stoplight::Domain::Config]
-        def initialize(data_store:, traffic_control:, notifiers:, config:)
-          @data_store = data_store
+        # @param metrics_store [Stoplight::Storage::Metrics]
+        # @param state_store [Stoplight::Domain::Storage::State]
+        def initialize(traffic_control:, notifiers:, config:, metrics_store:, state_store:)
           @traffic_control = traffic_control
           @notifiers = notifiers
           @config = config
+          @metrics_store = metrics_store
+          @state_store = state_store
         end
 
         # @param exception [Exception]
         # @return [void]
         def record_failure(exception)
-          metadata = data_store.record_failure(config, exception)
+          metrics_store.record_failure(exception)
+          metrics = metrics_store.metrics_snapshot
 
-          transition_to_red(exception, metadata:)
+          transition_to_red(exception, metrics:)
         end
 
         # @return [void]
-        def record_success
-          data_store.record_success(config)
-        end
+        def record_success = metrics_store.record_success
 
-        private def transition_to_red(exception, metadata:)
-          if traffic_control.stop_traffic?(config, metadata)
-            transition_and_notify(Color::GREEN, Color::RED, exception)
+        private def transition_to_red(exception, metrics:)
+          if traffic_control.stop_traffic?(config, metrics)
+            # Returns true only if not yet in red therefore preventing
+            # duplicate notifications
+            if state_store.transition_to_color(Color::RED)
+              notifiers.each do |notifier|
+                notifier.notify(config, Color::GREEN, Color::RED, exception)
+              end
+            end
           end
         end
-
-        # @param other [any]
-        # @return [bool]
-        def ==(other)
-          super && traffic_control == other.traffic_control
-        end
       end
     end
   end

data/lib/stoplight/domain/traffic_control/base.rb

@@ -18,11 +18,11 @@ module Stoplight
       #       end
       #     end
       #
-      #     def stop_traffic?(config, metadata)
-      #       total = metadata.successes + metadata.failures
+      #     def stop_traffic?(config, metrics)
+      #       total = metrics.successes + metrics.failures
       #       return false if total < 10 # Minimum sample size
       #
-      #       error_rate = metadata.failures.fdiv(total)
+      #       error_rate = metrics.failures.fdiv(total)
       #       error_rate >= 0.5 # Stop traffic when error rate reaches 50%
       #     end
       #   end
@@ -44,10 +44,10 @@ module Stoplight
         # current state and metrics.
         #
         # @param config [Stoplight::Domain::Config]
-        # @param metadata [Stoplight::Domain::Metadata]
+        # @param metrics [Stoplight::Domain::Metrics]
         # @return [Boolean] true if traffic should be stopped (rec), false otherwise (green)
         # :nocov:
-        def stop_traffic?(config, metadata)
+        def stop_traffic?(config, metrics)
           raise NotImplementedError
         end
         # :nocov:

data/lib/stoplight/domain/traffic_control/consecutive_errors.rb

@@ -42,14 +42,10 @@ module Stoplight
         # Determines if traffic should be stopped based on failure counts.
         #
         # @param config [Stoplight::Domain::Config]
-        # @param metadata [Stoplight::Domain::Metadata]
+        # @param metrics [Stoplight::Domain::Metrics]
         # @return [Boolean] true if failures have reached the threshold, false otherwise
-        def stop_traffic?(config, metadata)
-          if config.window_size
-            [metadata.consecutive_errors, metadata.errors].min >= config.threshold
-          else
-            metadata.consecutive_errors >= config.threshold
-          end
+        def stop_traffic?(config, metrics)
+          metrics.consecutive_errors >= config.threshold
         end
       end
     end

data/lib/stoplight/domain/traffic_control/error_rate.rb

@@ -40,10 +40,10 @@ module Stoplight
         end
 
         # @param config [Stoplight::Domain::Config]
-        # @param metadata [Stoplight::Domain::Metadata]
+        # @param metrics [Stoplight::Domain::Metrics]
         # @return [Boolean]
-        def stop_traffic?(config, metadata)
-          metadata.requests >= min_requests && metadata.error_rate >= config.threshold
+        def stop_traffic?(config, metrics)
+          metrics.requests >= min_requests && metrics.error_rate >= config.threshold
         end
       end
     end

data/lib/stoplight/domain/traffic_recovery/base.rb

@@ -14,14 +14,14 @@ module Stoplight
       #       @min_samples = min_samples
       #     end
       #
-      #     def determine_color(config, metadata)
-      #       total_probes = metadata.recovery_probe_successes + metadata.recovery_probe_errors
+      #     def determine_color(config, metrics)
+      #       total_probes = metrics.recovery_probe_successes + metrics.recovery_probe_errors
       #
       #       if total_probes < @min_samples
       #         return Color::YELLOW # Keep recovering, not enough samples
       #       end
       #
-      #       success_rate = metadata.recovery_probe_successes.fdiv(total_probes)
+      #       success_rate = metrics.recovery_probe_successes.fdiv(total_probes)
       #       if success_rate >= @min_success_rate
       #         Color::GREEN # Recovery successful
       #       elsif success_rate <= 0.2
@@ -49,10 +49,10 @@ module Stoplight
         # current metrics and recovery progress.
         #
         # @param config [Stoplight::Domain::Config]
-        # @param metadata [Stoplight::Domain::Metadata]
+        # @param metrics [Stoplight::Domain::Metrics]
         # @return [TrafficRecovery::Decision]
         # :nocov:
-        def determine_color(config, metadata)
+        def determine_color(config, metrics)
           raise NotImplementedError
         end
         # :nocov:

data/lib/stoplight/domain/traffic_recovery/consecutive_successes.rb

@@ -49,16 +49,12 @@ module Stoplight
         # Determines if traffic should be resumed based on successes counts.
         #
         # @param config [Stoplight::Domain::Config]
-        # @param metadata [Stoplight::Domain::Metadata]
+        # @param recovery_metrics [Stoplight::Domain::Metrics]
         # @return [TrafficRecovery::Decision]
-        def determine_color(config, metadata)
-          return TrafficRecovery::PASS if metadata.color != Color::YELLOW
-
-          recovery_started_at = metadata.recovery_started_at || metadata.recovery_scheduled_after
-
-          if metadata.last_error_at && metadata.last_error_at >= recovery_started_at
+        def determine_color(config, recovery_metrics)
+          if recovery_metrics.consecutive_errors > 0
             TrafficRecovery::RED
-          elsif [metadata.consecutive_successes, metadata.recovery_probe_successes].min >= config.recovery_threshold
+          elsif recovery_metrics.consecutive_successes >= config.recovery_threshold
             TrafficRecovery::GREEN
           else
             TrafficRecovery::YELLOW

data/lib/stoplight/domain/traffic_recovery.rb

@@ -7,7 +7,6 @@ module Stoplight
       GREEN = Decision.new("green")
       YELLOW = Decision.new("yellow")
       RED = Decision.new("red")
-      PASS = Decision.new("pass")
     end
   end
 end

data/lib/stoplight/infrastructure/data_store/fail_safe.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module Stoplight
+  module Infrastructure
+    module DataStore
+      # A wrapper around a data store that provides fail-safe mechanisms using a
+      # circuit breaker. It ensures that operations on the data store can gracefully
+      # handle failures by falling back to default values when necessary.
+      #
+      # @api private
+      class FailSafe < Domain::DataStore
+        # @!attribute data_store
+        #  @return [Stoplight::DataStore::Base] The underlying primary data store being used
+        attr_reader :data_store
+
+        # @!attribute error_notifier
+        #   @return [Proc]
+        attr_reader :error_notifier
+
+        # @!attribute failover_data_store
+        #   @return [Stoplight::DataStore::Base] The fallback data store used when the primary fails.
+        attr_reader :failover_data_store
+
+        # @!attribute circuit_breaker
+        #   @return [Stoplight::Light] The circuit breaker used to handle data store failures.
+        private attr_reader :circuit_breaker
+
+        # @param data_store [Stoplight::Domain::DataStore]
+        # @param error_notifier [Proc]
+        # @param failover_data_store [Stoplight::Domain::DataStore]
+        # @param circuit_breaker [Stoplight::Domain::Light]
+        def initialize(data_store:, error_notifier:, failover_data_store:, circuit_breaker:)
+          @data_store = data_store
+          @error_notifier = error_notifier
+          @failover_data_store = failover_data_store
+          @circuit_breaker = circuit_breaker
+        end
+
+        def names
+          with_fallback(:names) do
+            data_store.names
+          end
+        end
+
+        def get_metrics(config, *args, **kwargs)
+          with_fallback(:get_metrics, config, *args, **kwargs) do
+            data_store.get_metrics(config, *args, **kwargs)
+          end
+        end
+
+        def get_recovery_metrics(config, *args, **kwargs)
+          with_fallback(:get_recovery_metrics, config, *args, **kwargs) do
+            data_store.get_recovery_metrics(config, *args, **kwargs)
+          end
+        end
+
+        def get_state_snapshot(config)
+          with_fallback(:get_state_snapshot, config) do
+            data_store.get_state_snapshot(config)
+          end
+        end
+
+        def clear_metrics(config)
+          with_fallback(:clear_metrics, config) do
+            data_store.clear_metrics(config)
+          end
+        end
+
+        def clear_recovery_metrics(config)
+          with_fallback(:clear_recovery_metrics, config) do
+            data_store.clear_recovery_metrics(config)
+          end
+        end
+
+        def record_failure(config, *args, **kwargs)
+          with_fallback(:record_failure, config, *args, **kwargs) do
+            data_store.record_failure(config, *args, **kwargs)
+          end
+        end
+
+        def record_success(config, *args, **kwargs)
+          with_fallback(:record_success, config, *args, **kwargs) do
+            data_store.record_success(config, *args, **kwargs)
+          end
+        end
+
+        def record_recovery_probe_success(config, *args, **kwargs)
+          with_fallback(:record_recovery_probe_success, config, *args, **kwargs) do
+            data_store.record_recovery_probe_success(config, *args, **kwargs)
+          end
+        end
+
+        def record_recovery_probe_failure(config, *args, **kwargs)
+          with_fallback(:record_recovery_probe_failure, config, *args, **kwargs) do
+            data_store.record_recovery_probe_failure(config, *args, **kwargs)
+          end
+        end
+
+        def set_state(config, *args, **kwargs)
+          with_fallback(:set_state, config, *args, **kwargs) do
+            data_store.set_state(config, *args, **kwargs)
+          end
+        end
+
+        def transition_to_color(config, *args, **kwargs)
+          with_fallback(:transition_to_color, config, *args, **kwargs) do
+            data_store.transition_to_color(config, *args, **kwargs)
+          end
+        end
+
+        def delete_light(config, *args, **kwargs)
+          with_fallback(:delete_light, config, *args, **kwargs) do
+            data_store.delete_light(config, *args, **kwargs)
+          end
+        end
+
+        # @param config [Stoplight::Domain::Config]
+        def acquire_recovery_lock(config)
+          with_fallback(:acquire_recovery_lock, config) do
+            data_store.acquire_recovery_lock(config)
+          end
+        end
+
+        # Routes release to correct store based on token type.
+        # Redis tokens release via primary (with error notification on failure).
+        # Memory tokens release via failover directly.
+        #
+        # @param recovery_lock_token [Stoplight::Domain::RecoveryLockToken]
+        def release_recovery_lock(recovery_lock_token)
+          case recovery_lock_token
+          in Redis::RecoveryLockToken
+            fallback = proc do |error|
+              error_notifier.call(error) if error
+            end
+
+            circuit_breaker.run(fallback) do
+              data_store.release_recovery_lock(recovery_lock_token)
+            end
+          in Memory::RecoveryLockToken
+            failover_data_store.release_recovery_lock(recovery_lock_token)
+          end
+        end
+
+        def ==(other)
+          other.is_a?(self.class) && other.data_store == data_store && other.error_notifier == error_notifier &&
+            other.failover_data_store == failover_data_store
+        end
+
+        # @param method_name [Symbol] protected method name
+        private def with_fallback(method_name, *args, **kwargs, &code)
+          fallback = proc do |error|
+            config = args.first
+            error_notifier.call(error) if config && error
+            @failover_data_store.public_send(method_name, *args, **kwargs)
+          end
+
+          circuit_breaker.run(fallback, &code)
+        end
+      end
+    end
+  end
+end

data/lib/stoplight/infrastructure/data_store/memory/metrics.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Stoplight
+  module Infrastructure
+    module DataStore
+      class Memory
+        class Metrics
+          attr_accessor :consecutive_errors
+          attr_accessor :consecutive_successes
+          attr_accessor :last_error
+          attr_accessor :last_success_at
+
+          def initialize(consecutive_errors: 0, consecutive_successes: 0, last_error: nil, last_success_at: nil)
+            @consecutive_errors = consecutive_errors
+            @consecutive_successes = consecutive_successes
+            @last_error = last_error
+            @last_success_at = last_success_at
+          end
+
+          def last_error_at
+            @last_error&.time
+          end
+        end
+      end
+    end
+  end
+end