stoplight 5.6.0 → 5.7.0

data/lib/stoplight/infrastructure/data_store/fail_safe.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module Stoplight
+  module Infrastructure
+    module DataStore
+      # A wrapper around a data store that provides fail-safe mechanisms using a
+      # circuit breaker. It ensures that operations on the data store can gracefully
+      # handle failures by falling back to default values when necessary.
+      #
+      # @api private
+      class FailSafe < Domain::DataStore
+        # @!attribute data_store
+        #  @return [Stoplight::DataStore::Base] The underlying primary data store being used
+        attr_reader :data_store
+
+        # @!attribute error_notifier
+        #   @return [Proc]
+        attr_reader :error_notifier
+
+        # @!attribute failover_data_store
+        #   @return [Stoplight::DataStore::Base] The fallback data store used when the primary fails.
+        attr_reader :failover_data_store
+
+        # @!attribute circuit_breaker
+        #   @return [Stoplight::Light] The circuit breaker used to handle data store failures.
+        private attr_reader :circuit_breaker
+
+        # @param data_store [Stoplight::Domain::DataStore]
+        # @param error_notifier [Proc]
+        # @param failover_data_store [Stoplight::Domain::DataStore]
+        # @param circuit_breaker [Stoplight::Domain::Light]
+        def initialize(data_store:, error_notifier:, failover_data_store:, circuit_breaker:)
+          @data_store = data_store
+          @error_notifier = error_notifier
+          @failover_data_store = failover_data_store
+          @circuit_breaker = circuit_breaker
+        end
+
+        def names
+          with_fallback(:names) do
+            data_store.names
+          end
+        end
+
+        def get_metrics(config, *args, **kwargs)
+          with_fallback(:get_metrics, config, *args, **kwargs) do
+            data_store.get_metrics(config, *args, **kwargs)
+          end
+        end
+
+        def get_recovery_metrics(config, *args, **kwargs)
+          with_fallback(:get_recovery_metrics, config, *args, **kwargs) do
+            data_store.get_recovery_metrics(config, *args, **kwargs)
+          end
+        end
+
+        def get_state_snapshot(config)
+          with_fallback(:get_state_snapshot, config) do
+            data_store.get_state_snapshot(config)
+          end
+        end
+
+        def clear_metrics(config)
+          with_fallback(:clear_metrics, config) do
+            data_store.clear_metrics(config)
+          end
+        end
+
+        def clear_recovery_metrics(config)
+          with_fallback(:clear_recovery_metrics, config) do
+            data_store.clear_recovery_metrics(config)
+          end
+        end
+
+        def record_failure(config, *args, **kwargs)
+          with_fallback(:record_failure, config, *args, **kwargs) do
+            data_store.record_failure(config, *args, **kwargs)
+          end
+        end
+
+        def record_success(config, *args, **kwargs)
+          with_fallback(:record_success, config, *args, **kwargs) do
+            data_store.record_success(config, *args, **kwargs)
+          end
+        end
+
+        def record_recovery_probe_success(config, *args, **kwargs)
+          with_fallback(:record_recovery_probe_success, config, *args, **kwargs) do
+            data_store.record_recovery_probe_success(config, *args, **kwargs)
+          end
+        end
+
+        def record_recovery_probe_failure(config, *args, **kwargs)
+          with_fallback(:record_recovery_probe_failure, config, *args, **kwargs) do
+            data_store.record_recovery_probe_failure(config, *args, **kwargs)
+          end
+        end
+
+        def set_state(config, *args, **kwargs)
+          with_fallback(:set_state, config, *args, **kwargs) do
+            data_store.set_state(config, *args, **kwargs)
+          end
+        end
+
+        def transition_to_color(config, *args, **kwargs)
+          with_fallback(:transition_to_color, config, *args, **kwargs) do
+            data_store.transition_to_color(config, *args, **kwargs)
+          end
+        end
+
+        def delete_light(config, *args, **kwargs)
+          with_fallback(:delete_light, config, *args, **kwargs) do
+            data_store.delete_light(config, *args, **kwargs)
+          end
+        end
+
+        # @param config [Stoplight::Domain::Config]
+        def acquire_recovery_lock(config)
+          with_fallback(:acquire_recovery_lock, config) do
+            data_store.acquire_recovery_lock(config)
+          end
+        end
+
+        # Routes release to correct store based on token type.
+        # Redis tokens release via primary (with error notification on failure).
+        # Memory tokens release via failover directly.
+        #
+        # @param recovery_lock_token [Stoplight::Domain::RecoveryLockToken]
+        def release_recovery_lock(recovery_lock_token)
+          case recovery_lock_token
+          in Redis::RecoveryLockToken
+            fallback = proc do |error|
+              error_notifier.call(error) if error
+            end
+
+            circuit_breaker.run(fallback) do
+              data_store.release_recovery_lock(recovery_lock_token)
+            end
+          in Memory::RecoveryLockToken
+            failover_data_store.release_recovery_lock(recovery_lock_token)
+          end
+        end
+
+        def ==(other)
+          other.is_a?(self.class) && other.data_store == data_store && other.error_notifier == error_notifier &&
+            other.failover_data_store == failover_data_store
+        end
+
+        # @param method_name [Symbol] protected method name
+        private def with_fallback(method_name, *args, **kwargs, &code)
+          fallback = proc do |error|
+            config = args.first
+            error_notifier.call(error) if config && error
+            @failover_data_store.public_send(method_name, *args, **kwargs)
+          end
+
+          circuit_breaker.run(fallback, &code)
+        end
+      end
+    end
+  end
+end

data/lib/stoplight/infrastructure/data_store/memory/recovery_lock_store.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "concurrent/map"
+
+module Stoplight
+  module Infrastructure
+    module DataStore
+      class Memory
+        # Process-local recovery lock using Ruby's Thread::Mutex.
+        #
+        # This only serializes recovery within a single Ruby process.
+        # Multiple processes/servers will NOT coordinate - each process
+        # can send probes independently.
+        #
+        # Mutex Lifecycle:
+        # - One mutex created per unique light_name (lazily)
+        # - Mutexes persist for process lifetime (never GC'd)
+        #
+        class RecoveryLockStore
+          # @!attribute locks
+          #   Stores one mutex per unique light_name for the lifetime of the process.
+          #   Mutexes are never garbage collected.
+          #   @return [Concurrent::Map<Thread::Mutex>]
+          private attr_reader :locks
+
+          def initialize
+            @locks = Concurrent::Map.new
+          end
+
+          # @param light_name [String]
+          # @return [Stoplight::Infrastructure::DataStore::Memory::RecoveryLockToken, nil]
+          def acquire_lock(light_name)
+            lock = lock_for(light_name)
+            RecoveryLockToken.new(light_name:) if lock.try_lock
+          end
+
+          # @param recovery_lock_token [Stoplight::Infrastructure::DataStore::Memory::RecoveryLockToken]
+          # @return [void]
+          def release_lock(recovery_lock_token)
+            lock_for(recovery_lock_token.light_name).unlock
+          end
+
+          # @param light_name [String]
+          # @return [Thread::Mutex]
+          private def lock_for(light_name)
+            locks.compute_if_absent(light_name) do
+              Thread::Mutex.new
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/stoplight/infrastructure/data_store/memory/recovery_lock_token.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Stoplight
+  module Infrastructure
+    module DataStore
+      class Memory
+        class RecoveryLockToken < Domain::RecoveryLockToken
+          # @!attribute light_name
+          #   @return [String]
+          attr_reader :light_name
+
+          # @param light_name [String]
+          def initialize(light_name:)
+            @light_name = light_name
+          end
+        end
+      end
+    end
+  end
+end

data/lib/stoplight/infrastructure/data_store/memory.rb

@@ -11,22 +11,28 @@ module Stoplight
 
         KEY_SEPARATOR = ":"
 
-        def initialize
+        # @!attribute recovery_lock_store
+        #   @return [Stoplight::Infrastructure::DataStore::Memory::RecoveryLockStore]
+        #   @api private
+        private attr_reader :recovery_lock_store
+
+        # @param recovery_lock_store [Stoplight::Infrastructure::DataStore::Memory::RecoveryLockStore]
+        def initialize(recovery_lock_store:)
+          @recovery_lock_store = recovery_lock_store
           @errors = Hash.new { |errors, light_name| errors[light_name] = SlidingWindow.new }
           @successes = Hash.new { |successes, light_name| successes[light_name] = SlidingWindow.new }
+          @metrics = Hash.new { |metrics, light_name| metrics[light_name] = Metrics.new }
 
-          @recovery_probe_errors = Hash.new { |recovery_probe_errors, light_name| recovery_probe_errors[light_name] = SlidingWindow.new }
-          @recovery_probe_successes = Hash.new { |recovery_probe_successes, light_name| recovery_probe_successes[light_name] = SlidingWindow.new }
+          @recovery_metrics = Hash.new { |metrics, light_name| metrics[light_name] = Metrics.new }
 
           @states = Hash.new { |states, light_name| states[light_name] = State.new }
-          @metrics = Hash.new { |metrics, light_name| metrics[light_name] = Metrics.new }
 
-          super # MonitorMixin
+          super() # MonitorMixin
         end
 
         # @return [Array<String>]
         def names
-          synchronize { @metrics.keys | @states.keys }
+          synchronize { @metrics.keys | @states.keys | @recovery_metrics.keys }
         end
 
         # @param config [Stoplight::Domain::Config]
@@ -46,12 +52,14 @@ module Stoplight
 
             errors = @errors[light_name].sum_in_window(window_start) if config.window_size
             successes = @successes[light_name].sum_in_window(window_start) if config.window_size
+            consecutive_errors = config.window_size ? [metrics.consecutive_errors, errors].min : metrics.consecutive_errors
+            consecutive_successes = config.window_size ? [metrics.consecutive_successes.to_i, successes].min : metrics.consecutive_successes.to_i
 
             Domain::Metrics.new(
               errors:,
               successes:,
-              total_consecutive_errors: metrics.consecutive_errors,
-              total_consecutive_successes: metrics.consecutive_successes,
+              consecutive_errors:,
+              consecutive_successes:,
               last_error: metrics.last_error,
               last_success_at: metrics.last_success_at
             )
@@ -63,21 +71,12 @@ module Stoplight
           light_name = config.name
 
           synchronize do
-            current_time = self.current_time
-            recovery_window_start = (current_time - config.cool_off_time)
-            if config.window_size
-              (current_time - config.window_size)
-            else
-              current_time
-            end
-
-            metrics = @metrics[light_name]
+            metrics = @recovery_metrics[light_name]
 
             Domain::Metrics.new(
-              errors: @recovery_probe_errors[light_name].sum_in_window(recovery_window_start),
-              successes: @recovery_probe_successes[light_name].sum_in_window(recovery_window_start),
-              total_consecutive_errors: metrics.consecutive_errors,
-              total_consecutive_successes: metrics.consecutive_successes,
+              errors: nil, successes: nil,
+              consecutive_errors: metrics.consecutive_errors,
+              consecutive_successes: metrics.consecutive_successes,
               last_error: metrics.last_error,
               last_success_at: metrics.last_success_at
             )
@@ -121,12 +120,20 @@ module Stoplight
           end
         end
 
-        def clear_windowed_metrics(config)
-          if config.window_size
-            synchronize do
-              @errors[config.name] = SlidingWindow.new
-              @successes[config.name] = SlidingWindow.new
+        def clear_metrics(config)
+          light_name = config.name
+          synchronize do
+            if config.window_size
+              @errors[light_name] = SlidingWindow.new
+              @successes[light_name] = SlidingWindow.new
             end
+            @metrics[light_name] = Metrics.new
+          end
+        end
+
+        def clear_recovery_metrics(config)
+          synchronize do
+            @recovery_metrics[config.name] = Metrics.new
           end
         end
 
@@ -159,9 +166,7 @@ module Stoplight
           failure = Domain::Failure.from_error(exception, time: current_time)
 
           synchronize do
-            @recovery_probe_errors[light_name].increment
-
-            metrics = @metrics[light_name]
+            metrics = @recovery_metrics[light_name]
 
             if metrics.last_error_at.nil? || failure.occurred_at > metrics.last_error_at
               metrics.last_error = failure
@@ -179,9 +184,7 @@ module Stoplight
           current_time = self.current_time
 
           synchronize do
-            @recovery_probe_successes[light_name].increment
-
-            metrics = @metrics[light_name]
+            metrics = @recovery_metrics[light_name]
             if metrics.last_success_at.nil? || current_time > metrics.last_success_at
               metrics.last_success_at = current_time
             end
@@ -208,6 +211,20 @@ module Stoplight
           "#<#{self.class.name}>"
         end
 
+        # @param config [Stoplight::Domain::Config]
+        # @return [void]
+        def delete_light(config)
+          light_name = config.name
+
+          synchronize do
+            @states.delete(light_name)
+            @recovery_metrics.delete(light_name)
+            @metrics.delete(light_name)
+            @errors.delete(light_name)
+            @successes.delete(light_name)
+          end
+        end
+
         # Combined method that performs the state transition based on color
         #
         # @param config [Stoplight::Domain::Config] The light configuration
@@ -226,6 +243,18 @@ module Stoplight
           end
         end
 
+        # @param config [Stoplight::Domain::Config]
+        # @return [Stoplight::Infrastructure::DataStore::Memory::RecoveryLockToken, nil]
+        def acquire_recovery_lock(config)
+          recovery_lock_store.acquire_lock(config.name)
+        end
+
+        # @param lock [Stoplight::Infrastructure::DataStore::Memory::RecoveryLockToken]
+        # @return [void]
+        def release_recovery_lock(lock)
+          recovery_lock_store.release_lock(lock)
+        end
+
         # Transitions to GREEN state and ensures only one notification
         #
         # @param config [Stoplight::Domain::Config] The light configuration

data/lib/stoplight/infrastructure/data_store/redis/lua_scripts/record_recovery_probe_failure.lua

@@ -0,0 +1,27 @@
+local failure_ts = tonumber(ARGV[1])
+local failure_json = ARGV[2]
+
+local metadata_key = KEYS[1]
+
+
+-- Update metadata
+local meta = redis.call('HMGET', metadata_key, 'last_error_at', 'consecutive_errors')
+local prev_failure_ts = tonumber(meta[1])
+local prev_consecutive_errors = tonumber(meta[2])
+
+if not prev_failure_ts or failure_ts > prev_failure_ts then
+  redis.call(
+    'HSET', metadata_key,
+    'last_error_at', failure_ts,
+    'last_error_json', failure_json,
+    'consecutive_errors', (prev_consecutive_errors or 0) + 1,
+    'consecutive_successes', 0
+  )
+else
+  redis.call(
+    'HSET', metadata_key,
+    'consecutive_errors', (prev_consecutive_errors or 0) + 1,
+    'consecutive_successes', 0
+  )
+end
+

data/lib/stoplight/infrastructure/data_store/redis/lua_scripts/record_recovery_probe_success.lua

@@ -0,0 +1,23 @@
+local request_ts = tonumber(ARGV[1])
+
+local metadata_key = KEYS[1]
+
+-- Update metadata
+local meta = redis.call('HMGET', metadata_key, 'last_success_at', 'consecutive_successes')
+local prev_success_ts = tonumber(meta[1])
+local prev_consecutive_successes = tonumber(meta[2])
+
+if not prev_success_ts or request_ts > prev_success_ts then
+  redis.call(
+    'HSET', metadata_key,
+    'last_success_at', request_ts,
+    'consecutive_errors', 0,
+    'consecutive_successes', (prev_consecutive_successes or 0) + 1
+  )
+else
+  redis.call(
+    'HSET', metadata_key,
+    'consecutive_errors', 0,
+    'consecutive_successes', (prev_consecutive_successes or 0) + 1
+  )
+end

data/lib/stoplight/infrastructure/data_store/redis/lua_scripts/release_lock.lua

@@ -0,0 +1,6 @@
+local token = ARGV[1]
+local lock_key = KEYS[1]
+
+if redis.call("get", lock_key) == token then
+  return redis.call("del", lock_key)
+end

data/lib/stoplight/infrastructure/data_store/redis/recovery_lock_store.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "forwardable"
+
+module Stoplight
+  module Infrastructure
+    module DataStore
+      class Redis
+        # Distributed recovery recovery_lock using Redis SET NX (set-if-not-exists).
+        #
+        # Lock Acquisition:
+        # - Uses unique UUID token to prevent accidental release of others' locks
+        # - Atomic SET with NX flag ensures only one process acquires recovery_lock
+        # - TTL (px: lock_timeout) auto-releases recovery_lock if process crashes
+        #
+        # Lock Release:
+        # - Lua script ensures only token holder can release (token comparison)
+        # - Best-effort release; TTL cleanup handles failures
+        #
+        # Failure Modes:
+        # - Lock contention: Returns false, caller should skip probe
+        # - Redis unavailable: raises an error and let caller decide
+        # - Crashed holder: raises an error and let caller decide. Lock auto-expires after lock_timeout
+        # - Release failure: Lock auto-expires after lock_timeout
+        #
+        class RecoveryLockStore
+          # @!attribute redis
+          #   @return [RedisClient]
+          protected attr_reader :redis
+
+          # @!attribute lock_timeout
+          #   @return [Integer]
+          protected attr_reader :lock_timeout
+
+          # @!attribute scripting
+          #   @return [Stoplight::Infrastructure::DataStore::Redis::Scripting]
+          protected attr_reader :scripting
+
+          # @param redis [RedisClient | ConnectionPool]
+          # @param lock_timeout [Integer] recovery_lock timeout in milliseconds
+          # @param scripting [Stoplight::Infrastructure::DataStore::Redis::Scripting]
+          def initialize(redis:, lock_timeout:, scripting:)
+            @redis = redis
+            @lock_timeout = lock_timeout
+            @scripting = scripting
+          end
+
+          # @param light_name [String]
+          # @return [Stoplight::Infrastructure::DataStore::Redis::RecoveryLockToken, nil]
+          def acquire_lock(light_name)
+            recovery_lock = RecoveryLockToken.new(light_name:)
+
+            acquired = !!redis.then do |client|
+              client.set(recovery_lock.lock_key, recovery_lock.token, nx: true, px: lock_timeout)
+            end
+
+            recovery_lock if acquired
+          end
+
+          # @param recovery_lock [Stoplight::Infrastructure::DataStore::Redis::RecoveryLockToken]
+          # @return [void]
+          def release_lock(recovery_lock)
+            scripting.call(
+              :release_lock,
+              keys: [recovery_lock.lock_key], args: [recovery_lock.token]
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/stoplight/infrastructure/data_store/redis/recovery_lock_token.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "forwardable"
+
+module Stoplight
+  module Infrastructure
+    module DataStore
+      class Redis
+        class RecoveryLockToken < Domain::RecoveryLockToken
+          extend Forwardable
+
+          def_delegator "Stoplight::Infrastructure::DataStore::Redis", :key
+          private :key
+
+          # @!attribute light_name
+          #   @return [String]
+          attr_reader :light_name
+
+          # @!attribute token
+          #   @return [String]
+          attr_reader :token
+
+          # @param light_name [String]
+          def initialize(light_name:)
+            @light_name = light_name
+            @token = SecureRandom.uuid
+          end
+
+          def lock_key = key(:locks, :recovery, light_name)
+        end
+      end
+    end
+  end
+end

data/lib/stoplight/infrastructure/data_store/redis/scripting.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Stoplight
+  module Infrastructure
+    module DataStore
+      class Redis
+        # Manages Lua scripts for Redis operations.
+        #
+        # This class provides execution of Lua scripts by caching their SHA digests
+        # and automatically reloading scripts if they're evicted from Redis memory.
+        #
+        # @example
+        #   script_manager = ScriptManager.new(redis: redis_client)
+        #   script_manager.call(:increment_counter, keys: ["counter:1"], args: [5])
+        #
+        # @note Scripts are loaded lazily on first use and cached in memory
+        # @note Script files must be named `<script_name>.lua` and located in scripts_root
+        class Scripting
+          SCRIPTS_ROOT = File.join(__dir__, "lua_scripts")
+          # @!attribute scripts_root
+          #   @return [String]
+          protected attr_reader :scripts_root
+
+          # @!attribute shas
+          #   @return [Hash{Symbol, String}]
+          private attr_reader :shas
+
+          # @!attribute redis
+          #   @return [RedisClient | ConnectionPool]
+          protected attr_reader :redis
+
+          # @param redis [RedisClient | ConnectionPool]
+          # @param scripts_root [String]
+          def initialize(redis:, scripts_root: SCRIPTS_ROOT)
+            @scripts_root = scripts_root
+            @redis = redis
+            @shas = {}
+          end
+
+          def call(script_name, keys: [], args: [])
+            redis.then do |client|
+              client.evalsha(script_sha(script_name), keys: keys, argv: args)
+            end
+          rescue ::Redis::CommandError => error
+            if error.message.include?("NOSCRIPT")
+              reload_script(script_name)
+              retry
+            else
+              raise
+            end
+          end
+
+          private def reload_script(script_name)
+            shas.delete(script_name)
+            script_sha(script_name)
+          end
+
+          private def script_sha(script_name)
+            if shas.key?(script_name)
+              shas[script_name]
+            else
+              script = File.read(File.join(scripts_root, "#{script_name}.lua"))
+
+              shas[script_name] = redis.then { |client| client.script("load", script) }
+            end
+          end
+        end
+      end
+    end
+  end
+end