stoplight 5.6.0 → 5.7.0

data/lib/stoplight/domain/storage/recovery_lock.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Stoplight
+  module Domain
+    module Storage
+      # Encapsulates recovery lock management for coordinating recovery probes.
+      #
+      # When a circuit enters YELLOW state (half-open), it begins sending
+      # "recovery probes" - test requests to check if the protected service
+      # has recovered. In distributed deployments with multiple instances,
+      # recovery locks ensure only ONE instance sends probes at a time.
+      #
+      # Without coordination, all instances would simultaneously:
+      # 1. Detect the circuit is YELLOW
+      # 2. Send recovery probes to the struggling service
+      # 3. Potentially overwhelm it with "test" traffic
+      #
+      # Lock Lifecycle:
+      #
+      #   Instance A: acquire_lock -> probe -> release_lock
+      #   Instance B: acquire_lock -> nil (already held) -> skip probe
+      #   Instance C: acquire_lock -> nil (already held) -> skip probe
+      #
+      # Lock Semantics:
+      # - Returns +nil+ if lock is already held. Never blocks waiting for lock availability
+      # - Locks must automatically expire when persisted storage is used
+      # - Failed releases are acceptable (timeout provides safety)
+      #
+      # @abstract
+      # @see Stoplight::Domain::Strategies::YellowRunStrategy
+      class RecoveryLock
+        # Attempts to acquire recovery lock for exclusive probe execution.
+        #
+        # This method tries to acquire a lock that serializes recovery probe
+        # execution across multiple instances. If the lock is already held by
+        # another instance, returns +nil+ immediately without blocking.
+        #
+        # @return [Stoplight::Domain::RecoveryLockToken, nil]
+        #   - +RecoveryLockToken+: Lock acquired, caller should send probe
+        #   - +nil+: Lock unavailable, another instance is probing
+        #
+        def acquire_lock = raise NotImplementedError
+
+        # Releases a previously acquired lock.
+        #
+        # This method releases the lock token returned by +#acquire_lock+,
+        # allowing other instances to acquire it. Release should be called
+        # in an ensure block to guarantee cleanup even if probe fails.
+        #
+        # @param lock [Stoplight::Domain::RecoveryLockToken] The token returned by +#acquire_lock+
+        # @return [void]
+        def release_lock(lock) = raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/stoplight/domain/storage/state.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Stoplight
+  module Domain
+    module Storage
+      # Encapsulates circuit breaker state storage.
+      #
+      # State management handles the current operational mode of a circuit breaker:
+      # - Color (GREEN/YELLOW/RED) - whether the circuit is open or closed
+      # - Lock state (LOCKED_GREEN/LOCKED_RED/UNLOCKED) - manual overrides
+      # - State transitions - tracking color changes for notifications #
+      #
+      # State requires stronger consistency than metrics because:
+      # - Multiple instances must agree on circuit color
+      # - Race conditions during transitions must be handled
+      # - Lock states must be immediately visible across instances
+      #
+      # @abstract
+      # @see Stoplight::Domain::Storage::Metrics
+      class State
+        # Retrieves current state snapshot for decision-making.
+        #
+        # The snapshot is an immutable view of the circuit's current state,
+        # including its color and lock status. This method is called on every
+        # circuit breaker invocation to determine whether to allow traffic.
+        #
+        # This is called on every request, so implementations should be fast.
+        #
+        # @return [Stoplight::Domain::StateSnapshot]
+        def state_snapshot = raise NotImplementedError
+
+        # Sets the lock state of the circuit.
+        #
+        # Locks allow manual override of circuit behavior:
+        # - LOCKED_GREEN: Force circuit closed (allow all traffic)
+        # - LOCKED_RED: Force circuit open (block all traffic)
+        # - UNLOCKED: Follow normal circuit breaker rules
+        #
+        # Lock states take precedence over color states. A locked circuit
+        # ignores failure thresholds and stays in the locked state until
+        # explicitly unlocked.
+        #
+        # Use Cases:
+        # - Emergency traffic control during incidents
+        # - Maintenance windows (lock RED to prevent traffic)
+        # - Gradual rollout (lock GREEN during testing)
+        #
+        # @param state [String] The new state to set.
+        # @return [String] The state that was set.
+        def set_state(state) = raise NotImplementedError
+
+        # Transitions the Stoplight to the specified color.
+        #
+        # This method performs a color transition operation that works across distributed instances
+        # of the light. It ensures that in a multi-instance environment, only one instance
+        # is considered the "first" to perform the transition (and therefore responsible for
+        # triggering notifications).
+        #
+        # @param color [String] The target color/state to transition to.
+        #   Should be one of Stoplight::Color::GREEN, Stoplight::Color::YELLOW, or Stoplight::Color::RED.
+        #
+        # @return [Boolean] Returns +true+ if this instance was the first to perform this specific transition
+        #   (and should therefore trigger notifications). Returns +false+ if another instance already
+        #   initiated this transition.
+        #
+        # @note In distributed environments with multiple instances, race conditions can occur when instances
+        #   attempt conflicting transitions simultaneously (e.g., one instance tries to transition from
+        #   YELLOW to GREEN while another tries YELLOW to RED). The implementation handles this, but
+        #   be aware that the last operation may determine the final color of the light.
+        #
+        def transition_to_color(color) = raise NotImplementedError
+
+        # Clears all state data for this circuit.
+        #
+        # This removes the circuit from storage entirely, resetting it to
+        # default (unlocked, green) state. The next invocation will start
+        # with fresh state.
+        #
+        # @note This does NOT clear metrics. If you want to fully
+        # reset a circuit, clear both state and metrics stores.
+        #
+        # @return [void]
+        def clear = raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/stoplight/domain/strategies/run_strategy.rb

@@ -16,11 +16,6 @@ module Stoplight
           raise NotImplementedError, "Subclasses must implement the execute method"
         end
         # :nocov:
-
-        # @return [Boolean]
-        def ==(other)
-          other.is_a?(self.class)
-        end
       end
     end
   end

data/lib/stoplight/domain/strategies/yellow_run_strategy.rb

@@ -15,9 +15,17 @@ module Stoplight
         #   @return [Stoplight::Domain::Config] The configuration for the light.
         protected attr_reader :config
 
-        # @!attribute [r] data_store
-        #   @return [Stoplight::DataStore::Base] The data store associated with the light.
-        protected attr_reader :data_store
+        # @!attribute [r] stare_store
+        #   @return [Stoplight::Domain::Storage::State]
+        protected attr_reader :state_store
+
+        # @!attribute [r] metrics_store
+        #   @return [Stoplight::Domain::Storage::Metrics]
+        protected attr_reader :metrics_store
+
+        # @!attribute [r] recovery_lock_store
+        #   @return [Stoplight::Domain::Storage::RecoveryLock]
+        protected attr_reader :recovery_lock_store
 
         # @!attribute [r] notifiers
         #   @return [Stoplight::Domain::StateTransitionNotifier]
@@ -27,15 +35,23 @@ module Stoplight
         #   @return [Stoplight::Domain::RecoveryProbeRequestRecorder]
         protected attr_reader :request_tracker
 
+        # @!attribute [r] red_run_strategy
+        #   @return [Stoplight::Domain::Strategies::RedRunStrategy]
+        protected attr_reader :red_run_strategy
+
         # @param config [Stoplight::Domain::Config]
-        # @param data_store [Stoplight::DataStore::Base]
         # @param notifiers [Array<Stoplight::Domain::StateTransitionNotifier>]
         # @param request_tracker [Stoplight::Domain::Tracker::RecoveryProbe]
-        def initialize(config:, data_store:, notifiers:, request_tracker:)
+        # @param red_run_strategy [Stoplight::Domain::Strategies::RedRunStrategy]
+        # @param recovery_lock_store [Stoplight::Domain::Storage::RecoveryLock]
+        def initialize(config:, notifiers:, request_tracker:, red_run_strategy:, state_store:, metrics_store:, recovery_lock_store:)
           @config = config
-          @data_store = data_store
           @notifiers = notifiers
           @request_tracker = request_tracker
+          @red_run_strategy = red_run_strategy
+          @state_store = state_store
+          @metrics_store = metrics_store
+          @recovery_lock_store = recovery_lock_store
         end
 
         # Executes the provided code block when the light is in the yellow state.
@@ -46,21 +62,41 @@ module Stoplight
         # @return [Object] The result of the code block if successful.
         # @raise [Exception] Re-raises the error if it is not tracked or no fallback is provided.
         def execute(fallback, state_snapshot:, &code)
-          enter_recovery(state_snapshot)
-          # TODO: We need to employ a probabilistic approach here to avoid "thundering herd" problem
-          code.call.tap { record_recovery_probe_success }
-        rescue => error
-          if config.track_error?(error)
-            record_recovery_probe_failure(error)
-
-            if fallback
-              fallback.call(error)
+          # Everything withing this block executed exclusively:
+          #   - enter recovery
+          #   - execute user's code
+          #   - record outcome
+          #   - transition to green or red if needed
+          with_recovery_lock(fallback:, state_snapshot:) do
+            enter_recovery(state_snapshot)
+
+            code.call.tap { record_recovery_probe_success }
+          rescue => error
+            if config.track_error?(error)
+              record_recovery_probe_failure(error)
+
+              if fallback
+                fallback.call(error)
+              else
+                raise
+              end
             else
+              record_recovery_probe_success
               raise
             end
-          else
-            record_recovery_probe_success
-            raise
+          end
+        end
+
+        def with_recovery_lock(fallback:, state_snapshot:)
+          recovery_lock_token = recovery_lock_store.acquire_lock
+          if recovery_lock_token.nil?
+            return red_run_strategy.execute(fallback, state_snapshot:)
+          end
+
+          begin
+            yield
+          ensure
+            recovery_lock_store.release_lock(recovery_lock_token)
           end
         end
 
@@ -77,22 +113,12 @@ module Stoplight
         private def enter_recovery(state_snapshot)
           return if state_snapshot.recovery_started?
 
-          if data_store.transition_to_color(config, Color::YELLOW)
-            data_store.clear_windowed_metrics(config)
-            notifiers.each do |notifier|
-              notifier.notify(config, Color::RED, Color::YELLOW, nil)
-            end
+          state_store.transition_to_color(Color::YELLOW)
+          metrics_store.clear
+          notifiers.each do |notifier|
+            notifier.notify(config, Color::RED, Color::YELLOW, nil)
           end
         end
-
-        # @return [Boolean]
-        def ==(other)
-          super &&
-            config == other.config &&
-            notifiers == other.notifiers &&
-            data_store == other.data_store &&
-            request_tracker == other.request_tracker
-        end
       end
     end
   end

data/lib/stoplight/domain/tracker/base.rb

@@ -6,35 +6,6 @@ module Stoplight
       # @api private
       # @abstract
       class Base
-        # @!attribute [r] data_store
-        #   @return [Stoplight::DataStore::Base] The data store associated with the light.
-        protected attr_reader :data_store
-
-        # @!attribute [r] traffic_control
-        #   @return [Stoplight::Domain::TrafficControl::Base]
-        protected attr_reader :notifiers
-
-        # @!attribute [r] config
-        #   @return [Stoplight::Domain::Config] The configuration for the light.
-        protected attr_reader :config
-
-        def ==(other)
-          other.is_a?(self.class) &&
-            config == other.config &&
-            data_store == other.data_store &&
-            notifiers == other.notifiers
-        end
-
-        def transition_and_notify(from_color, to_color, error = nil)
-          if data_store.transition_to_color(config, to_color)
-            notifiers.each do |notifier|
-              notifier.notify(config, from_color, to_color, error)
-            end
-            true
-          else
-            false
-          end
-        end
       end
     end
   end

data/lib/stoplight/domain/tracker/recovery_probe.rb

@@ -4,10 +4,6 @@ module Stoplight
   module Domain
     module Tracker
       class RecoveryProbe < Base
-        # @!attribute [r] data_store
-        #   @return [Stoplight::DataStore::Base] The data store associated with the light.
-        protected attr_reader :data_store
-
         # @!attribute [r] traffic_recovery
         #   @return [Stoplight::Domain::TrafficRecovery::Base]
         protected attr_reader :traffic_recovery
@@ -20,54 +16,59 @@ module Stoplight
         #   @return [Stoplight::Domain::Config] The configuration for the light.
         protected attr_reader :config
 
-        # @param data_store [Stoplight::Domain::DataStore]
+        # @!attribute [r] metrics_store
+        #   @return [Stoplight::Domain::Storage::Metrics]
+        protected attr_reader :metrics_store
+
+        # @!attribute [r] state_store
+        #   @return [Stoplight::Domain::Storage::State]
+        protected attr_reader :state_store
+
         # @param traffic_recovery [Stoplight::Domain::TrafficRecovery::Base]
         # @param notifiers [<Stoplight::Domain::StateTransitionNotifier>]
         # @param config [Stoplight::Domain::Config]
-        def initialize(data_store:, traffic_recovery:, notifiers:, config:)
-          @data_store = data_store
+        # @param metrics_store [Stoplight::Domain::Storage::Metrics]
+        # @param state_store [Stoplight::Domain::Storage::State]
+        def initialize(traffic_recovery:, notifiers:, config:, metrics_store:, state_store:)
           @traffic_recovery = traffic_recovery
           @notifiers = notifiers
           @config = config
+          @metrics_store = metrics_store
+          @state_store = state_store
         end
 
         # @param exception [Exception]
         def record_failure(exception)
-          data_store.record_recovery_probe_failure(config, exception)
+          metrics_store.record_failure(exception)
 
           recover
         end
 
         def record_success
-          data_store.record_recovery_probe_success(config)
+          metrics_store.record_success
 
           recover
         end
         RECOVERY_TRANSITIONS = {
           TrafficRecovery::GREEN => [Color::YELLOW, Color::GREEN],
-          TrafficRecovery::YELLOW => [Color::RED, Color::YELLOW],
           TrafficRecovery::RED => [Color::YELLOW, Color::RED]
         }.freeze
 
         private def recover
-          recovery_metrics = data_store.get_recovery_metrics(config)
-          state_snapshot = data_store.get_state_snapshot(config) # TODO: is this really necessary?
-
-          recovery_result = traffic_recovery.determine_color(config, recovery_metrics, state_snapshot)
+          recovery_metrics = metrics_store.metrics_snapshot
+          recovery_result = traffic_recovery.determine_color(config, recovery_metrics)
 
-          return if recovery_result == TrafficRecovery::PASS
+          return if recovery_result == TrafficRecovery::YELLOW
 
           from_color, to_color = RECOVERY_TRANSITIONS.fetch(recovery_result) do
             raise "recovery strategy returned unexpected color: #{recovery_result}"
           end
 
-          transition_and_notify(from_color, to_color, nil)
-        end
-
-        # @param other [any]
-        # @return [bool]
-        def ==(other)
-          super && traffic_recovery == other.traffic_recovery
+          state_store.transition_to_color(to_color)
+          metrics_store.clear
+          notifiers.each do |notifier|
+            notifier.notify(config, from_color, to_color, nil)
+          end
         end
       end
     end

data/lib/stoplight/domain/tracker/request.rb

@@ -10,10 +10,6 @@ module Stoplight
       #
       # @api private
       class Request < Base
-        # @!attribute [r] data_store
-        #   @return [Stoplight::DataStore::Base] The data store associated with the light.
-        protected attr_reader :data_store
-
         # @!attribute [r] traffic_control
         #   @return [Stoplight::Domain::TrafficControl::Base]
         protected attr_reader :traffic_control
@@ -26,42 +22,50 @@ module Stoplight
         #   @return [Stoplight::Domain::Config] The configuration for the light.
         protected attr_reader :config
 
-        # @param data_store [Stoplight::Domain::DataStore]
+        # @!attribute metrics_store
+        #   @return [Stoplight::Storage::Metrics]
+        protected attr_reader :metrics_store
+
+        # @!attribute [r] state_store
+        #   @return [Stoplight::Domain::Storage::State]
+        protected attr_reader :state_store
+
         # @param traffic_control [Stoplight::Domain::TrafficControl::Base]
         # @param notifiers [<Stoplight::Domain::StateTransitionNotifier>]
         # @param config [Stoplight::Domain::Config]
-        def initialize(data_store:, traffic_control:, notifiers:, config:)
-          @data_store = data_store
+        # @param metrics_store [Stoplight::Storage::Metrics]
+        # @param state_store [Stoplight::Domain::Storage::State]
+        def initialize(traffic_control:, notifiers:, config:, metrics_store:, state_store:)
           @traffic_control = traffic_control
           @notifiers = notifiers
           @config = config
+          @metrics_store = metrics_store
+          @state_store = state_store
         end
 
         # @param exception [Exception]
         # @return [void]
         def record_failure(exception)
-          data_store.record_failure(config, exception)
-          metrics = data_store.get_metrics(config)
+          metrics_store.record_failure(exception)
+          metrics = metrics_store.metrics_snapshot
 
           transition_to_red(exception, metrics:)
         end
 
         # @return [void]
-        def record_success
-          data_store.record_success(config)
-        end
+        def record_success = metrics_store.record_success
 
         private def transition_to_red(exception, metrics:)
           if traffic_control.stop_traffic?(config, metrics)
-            transition_and_notify(Color::GREEN, Color::RED, exception)
+            # Returns true only if not yet in red therefore preventing
+            # duplicate notifications
+            if state_store.transition_to_color(Color::RED)
+              notifiers.each do |notifier|
+                notifier.notify(config, Color::GREEN, Color::RED, exception)
+              end
+            end
           end
         end
-
-        # @param other [any]
-        # @return [bool]
-        def ==(other)
-          super && traffic_control == other.traffic_control
-        end
       end
     end
   end

data/lib/stoplight/domain/traffic_recovery/base.rb

@@ -50,10 +50,9 @@ module Stoplight
         #
         # @param config [Stoplight::Domain::Config]
         # @param metrics [Stoplight::Domain::Metrics]
-        # @param state_snapshot [Stoplight::Domain::StateSnapshot]
         # @return [TrafficRecovery::Decision]
         # :nocov:
-        def determine_color(config, metrics, state_snapshot)
+        def determine_color(config, metrics)
           raise NotImplementedError
         end
         # :nocov:

data/lib/stoplight/domain/traffic_recovery/consecutive_successes.rb

@@ -50,15 +50,9 @@ module Stoplight
         #
         # @param config [Stoplight::Domain::Config]
         # @param recovery_metrics [Stoplight::Domain::Metrics]
-        # @param state_snapshot [Stoplight::Domain::StateSnapshot]
         # @return [TrafficRecovery::Decision]
-        def determine_color(config, recovery_metrics, state_snapshot)
-          return TrafficRecovery::PASS if state_snapshot.color != Color::YELLOW
-
-          recovery_started_at = state_snapshot.recovery_started_at || state_snapshot.recovery_scheduled_after
-
-          # TODO: Need to add metrics cleanup and we can just use recovery_metrics.errors > 0
-          if recovery_metrics.last_error_at && recovery_metrics.last_error_at >= recovery_started_at
+        def determine_color(config, recovery_metrics)
+          if recovery_metrics.consecutive_errors > 0
             TrafficRecovery::RED
           elsif recovery_metrics.consecutive_successes >= config.recovery_threshold
             TrafficRecovery::GREEN

data/lib/stoplight/domain/traffic_recovery.rb

@@ -7,7 +7,6 @@ module Stoplight
       GREEN = Decision.new("green")
       YELLOW = Decision.new("yellow")
       RED = Decision.new("red")
-      PASS = Decision.new("pass")
     end
   end
 end