stoplight 5.5.0 → 5.7.0

data/lib/stoplight/infrastructure/data_store/memory/recovery_lock_store.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "concurrent/map"
+
+module Stoplight
+  module Infrastructure
+    module DataStore
+      class Memory
+        # Process-local recovery lock using Ruby's Thread::Mutex.
+        #
+        # This only serializes recovery within a single Ruby process.
+        # Multiple processes/servers will NOT coordinate - each process
+        # can send probes independently.
+        #
+        # Mutex Lifecycle:
+        # - One mutex created per unique light_name (lazily)
+        # - Mutexes persist for process lifetime (never GC'd)
+        #
+        class RecoveryLockStore
+          # @!attribute locks
+          #   Stores one mutex per unique light_name for the lifetime of the process.
+          #   Mutexes are never garbage collected.
+          #   @return [Concurrent::Map<Thread::Mutex>]
+          private attr_reader :locks
+
+          def initialize
+            @locks = Concurrent::Map.new
+          end
+
+          # @param light_name [String]
+          # @return [Stoplight::Infrastructure::DataStore::Memory::RecoveryLockToken, nil]
+          def acquire_lock(light_name)
+            lock = lock_for(light_name)
+            RecoveryLockToken.new(light_name:) if lock.try_lock
+          end
+
+          # @param recovery_lock_token [Stoplight::Infrastructure::DataStore::Memory::RecoveryLockToken]
+          # @return [void]
+          def release_lock(recovery_lock_token)
+            lock_for(recovery_lock_token.light_name).unlock
+          end
+
+          # @param light_name [String]
+          # @return [Thread::Mutex]
+          private def lock_for(light_name)
+            locks.compute_if_absent(light_name) do
+              Thread::Mutex.new
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/stoplight/infrastructure/data_store/memory/recovery_lock_token.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Stoplight
+  module Infrastructure
+    module DataStore
+      class Memory
+        class RecoveryLockToken < Domain::RecoveryLockToken
+          # @!attribute light_name
+          #   @return [String]
+          attr_reader :light_name
+
+          # @param light_name [String]
+          def initialize(light_name:)
+            @light_name = light_name
+          end
+        end
+      end
+    end
+  end
+end

data/lib/stoplight/infrastructure/data_store/memory/state.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Stoplight
+  module Infrastructure
+    module DataStore
+      class Memory
+        class State
+          attr_accessor :recovered_at
+          attr_accessor :locked_state
+          attr_accessor :recovery_scheduled_after
+          attr_accessor :recovery_started_at
+          attr_accessor :breached_at
+
+          def initialize
+            @locked_state = Domain::State::UNLOCKED
+          end
+        end
+      end
+    end
+  end
+end

data/lib/stoplight/infrastructure/data_store/memory.rb

@@ -11,68 +11,96 @@ module Stoplight
 
         KEY_SEPARATOR = ":"
 
-        def initialize
+        # @!attribute recovery_lock_store
+        #   @return [Stoplight::Infrastructure::DataStore::Memory::RecoveryLockStore]
+        #   @api private
+        private attr_reader :recovery_lock_store
+
+        # @param recovery_lock_store [Stoplight::Infrastructure::DataStore::Memory::RecoveryLockStore]
+        def initialize(recovery_lock_store:)
+          @recovery_lock_store = recovery_lock_store
           @errors = Hash.new { |errors, light_name| errors[light_name] = SlidingWindow.new }
           @successes = Hash.new { |successes, light_name| successes[light_name] = SlidingWindow.new }
+          @metrics = Hash.new { |metrics, light_name| metrics[light_name] = Metrics.new }
 
-          @recovery_probe_errors = Hash.new { |recovery_probe_errors, light_name| recovery_probe_errors[light_name] = SlidingWindow.new }
-          @recovery_probe_successes = Hash.new { |recovery_probe_successes, light_name| recovery_probe_successes[light_name] = SlidingWindow.new }
-
-          @metadata = Hash.new do |metadata, light_name|
-            metadata[light_name] = Domain::Metadata.new(
-              current_time: Time.now,
-              successes: 0,
-              errors: 0,
-              recovery_probe_successes: 0,
-              recovery_probe_errors: 0,
-              last_error: nil,
-              last_error_at: nil,
-              last_success_at: nil,
-              consecutive_errors: 0,
-              consecutive_successes: 0,
-              breached_at: nil,
-              locked_state: Domain::State::UNLOCKED,
-              recovery_scheduled_after: nil,
-              recovery_started_at: nil,
-              recovered_at: nil
-            )
-          end
-          super # MonitorMixin
+          @recovery_metrics = Hash.new { |metrics, light_name| metrics[light_name] = Metrics.new }
+
+          @states = Hash.new { |states, light_name| states[light_name] = State.new }
+
+          super() # MonitorMixin
         end
 
         # @return [Array<String>]
         def names
-          synchronize { @metadata.keys }
+          synchronize { @metrics.keys | @states.keys | @recovery_metrics.keys }
         end
 
         # @param config [Stoplight::Domain::Config]
-        # @return [Stoplight::Domain::Metadata]
-        def get_metadata(config)
+        # @return [Stoplight::Domain::Metrics]
+        def get_metrics(config)
           light_name = config.name
 
           synchronize do
             current_time = self.current_time
-            recovery_window_start = (current_time - config.cool_off_time)
-            recovered_at = @metadata[light_name].recovered_at
             window_start = if config.window_size
-              [recovered_at, (current_time - config.window_size)].compact.max
+              (current_time - config.window_size)
             else
               current_time
             end
 
-            @metadata[light_name].with(
-              current_time:,
-              errors: @errors[config.name].sum_in_window(window_start),
-              successes: @successes[config.name].sum_in_window(window_start),
-              recovery_probe_errors: @recovery_probe_errors[config.name].sum_in_window(recovery_window_start),
-              recovery_probe_successes: @recovery_probe_successes[config.name].sum_in_window(recovery_window_start)
+            metrics = @metrics[light_name]
+
+            errors = @errors[light_name].sum_in_window(window_start) if config.window_size
+            successes = @successes[light_name].sum_in_window(window_start) if config.window_size
+            consecutive_errors = config.window_size ? [metrics.consecutive_errors, errors].min : metrics.consecutive_errors
+            consecutive_successes = config.window_size ? [metrics.consecutive_successes.to_i, successes].min : metrics.consecutive_successes.to_i
+
+            Domain::Metrics.new(
+              errors:,
+              successes:,
+              consecutive_errors:,
+              consecutive_successes:,
+              last_error: metrics.last_error,
+              last_success_at: metrics.last_success_at
+            )
+          end
+        end
+
+        # @return [Stoplight::Domain::Metrics]
+        def get_recovery_metrics(config)
+          light_name = config.name
+
+          synchronize do
+            metrics = @recovery_metrics[light_name]
+
+            Domain::Metrics.new(
+              errors: nil, successes: nil,
+              consecutive_errors: metrics.consecutive_errors,
+              consecutive_successes: metrics.consecutive_successes,
+              last_error: metrics.last_error,
+              last_success_at: metrics.last_success_at
             )
           end
         end
 
+        # @return [Stoplight::Domain::StateSnapshot]
+        def get_state_snapshot(config)
+          time, state = synchronize do
+            [current_time, @states[config.name]]
+          end
+
+          Domain::StateSnapshot.new(
+            time:,
+            locked_state: state.locked_state,
+            recovery_scheduled_after: state.recovery_scheduled_after,
+            recovery_started_at: state.recovery_started_at,
+            breached_at: state.breached_at
+          )
+        end
+
         # @param config [Stoplight::Domain::Config]
         # @param exception [Exception]
-        # @return [Stoplight::Domain::Metadata]
+        # @return [void]
         def record_failure(config, exception)
           current_time = self.current_time
           light_name = config.name
@@ -81,21 +109,31 @@ module Stoplight
           synchronize do
             @errors[light_name].increment if config.window_size
 
-            metadata = @metadata[light_name]
-            @metadata[light_name] = if metadata.last_error_at.nil? || current_time > metadata.last_error_at
-              metadata.with(
-                last_error_at: current_time,
-                last_error: failure,
-                consecutive_errors: metadata.consecutive_errors.succ,
-                consecutive_successes: 0
-              )
-            else
-              metadata.with(
-                consecutive_errors: metadata.consecutive_errors.succ,
-                consecutive_successes: 0
-              )
+            metrics = @metrics[light_name]
+
+            if metrics.last_error_at.nil? || failure.occurred_at > metrics.last_error_at
+              metrics.last_error = failure
+            end
+
+            metrics.consecutive_errors += 1
+            metrics.consecutive_successes = 0
+          end
+        end
+
+        def clear_metrics(config)
+          light_name = config.name
+          synchronize do
+            if config.window_size
+              @errors[light_name] = SlidingWindow.new
+              @successes[light_name] = SlidingWindow.new
             end
-            get_metadata(config)
+            @metrics[light_name] = Metrics.new
+          end
+        end
+
+        def clear_recovery_metrics(config)
+          synchronize do
+            @recovery_metrics[config.name] = Metrics.new
           end
         end
 
@@ -108,74 +146,51 @@ module Stoplight
           synchronize do
             @successes[light_name].increment if config.window_size
 
-            metadata = @metadata[light_name]
-            @metadata[light_name] = if metadata.last_success_at.nil? || current_time > metadata.last_success_at
-              metadata.with(
-                last_success_at: current_time,
-                consecutive_errors: 0,
-                consecutive_successes: metadata.consecutive_successes.succ
-              )
-            else
-              metadata.with(
-                consecutive_errors: 0,
-                consecutive_successes: metadata.consecutive_successes.succ
-              )
+            metrics = @metrics[light_name]
+
+            if metrics.last_success_at.nil? || current_time > metrics.last_success_at
+              metrics.last_success_at = current_time
             end
+
+            metrics.consecutive_errors = 0
+            metrics.consecutive_successes += 1
           end
         end
 
         # @param config [Stoplight::Domain::Config]
         # @param exception [Exception]
-        # @return [Stoplight::Domain::Metadata]
+        # @return [void]
         def record_recovery_probe_failure(config, exception)
           light_name = config.name
           current_time = self.current_time
           failure = Domain::Failure.from_error(exception, time: current_time)
 
           synchronize do
-            @recovery_probe_errors[light_name].increment
-
-            metadata = @metadata[light_name]
-            @metadata[light_name] = if metadata.last_error_at.nil? || current_time > metadata.last_error_at
-              metadata.with(
-                last_error_at: current_time,
-                last_error: failure,
-                consecutive_errors: metadata.consecutive_errors.succ,
-                consecutive_successes: 0
-              )
-            else
-              metadata.with(
-                consecutive_errors: metadata.consecutive_errors.succ,
-                consecutive_successes: 0
-              )
+            metrics = @recovery_metrics[light_name]
+
+            if metrics.last_error_at.nil? || failure.occurred_at > metrics.last_error_at
+              metrics.last_error = failure
             end
-            get_metadata(config)
+
+            metrics.consecutive_errors += 1
+            metrics.consecutive_successes = 0
           end
         end
 
         # @param config [Stoplight::Domain::Config]
-        # @return [Stoplight::Domain::Metadata]
+        # @return [void]
         def record_recovery_probe_success(config)
           light_name = config.name
           current_time = self.current_time
 
           synchronize do
-            @recovery_probe_successes[light_name].increment
-
-            metadata = @metadata[light_name]
-            @metadata[light_name] = if metadata.last_success_at.nil? || current_time > metadata.last_success_at
-              metadata.with(
-                last_success_at: current_time,
-                consecutive_errors: 0,
-                consecutive_successes: metadata.consecutive_successes.succ
-              )
-            else
-              metadata.with(
-                consecutive_errors: 0,
-                consecutive_successes: metadata.consecutive_successes.succ
-              )
+            metrics = @recovery_metrics[light_name]
+            if metrics.last_success_at.nil? || current_time > metrics.last_success_at
+              metrics.last_success_at = current_time
             end
-            get_metadata(config)
+
+            metrics.consecutive_errors = 0
+            metrics.consecutive_successes += 1
           end
         end
 
@@ -186,8 +201,7 @@ module Stoplight
           light_name = config.name
 
           synchronize do
-            metadata = @metadata[light_name]
-            @metadata[light_name] = metadata.with(locked_state: state)
+            @states[light_name].locked_state = state
           end
           state
         end
@@ -197,6 +211,20 @@ module Stoplight
           "#<#{self.class.name}>"
         end
 
+        # @param config [Stoplight::Domain::Config]
+        # @return [void]
+        def delete_light(config)
+          light_name = config.name
+
+          synchronize do
+            @states.delete(light_name)
+            @recovery_metrics.delete(light_name)
+            @metrics.delete(light_name)
+            @errors.delete(light_name)
+            @successes.delete(light_name)
+          end
+        end
+
         # Combined method that performs the state transition based on color
         #
         # @param config [Stoplight::Domain::Config] The light configuration
@@ -215,6 +243,18 @@ module Stoplight
           end
         end
 
+        # @param config [Stoplight::Domain::Config]
+        # @return [Stoplight::Infrastructure::DataStore::Memory::RecoveryLockToken, nil]
+        def acquire_recovery_lock(config)
+          recovery_lock_store.acquire_lock(config.name)
+        end
+
+        # @param lock [Stoplight::Infrastructure::DataStore::Memory::RecoveryLockToken]
+        # @return [void]
+        def release_recovery_lock(lock)
+          recovery_lock_store.release_lock(lock)
+        end
+
         # Transitions to GREEN state and ensures only one notification
         #
         # @param config [Stoplight::Domain::Config] The light configuration
@@ -224,16 +264,15 @@ module Stoplight
           current_time = self.current_time
 
           synchronize do
-            metadata = @metadata[light_name]
-            if metadata.recovered_at
+            state = @states[light_name]
+
+            if state.recovered_at
               false
             else
-              @metadata[light_name] = metadata.with(
-                recovered_at: current_time,
-                recovery_started_at: nil,
-                breached_at: nil,
-                recovery_scheduled_after: nil
-              )
+              state.recovered_at = current_time
+              state.recovery_started_at = nil
+              state.breached_at = nil
+              state.recovery_scheduled_after = nil
               true
             end
           end
@@ -248,21 +287,17 @@ module Stoplight
           current_time = self.current_time
 
           synchronize do
-            metadata = @metadata[light_name]
-            if metadata.recovery_started_at.nil?
-              @metadata[light_name] = metadata.with(
-                recovery_started_at: current_time,
-                recovery_scheduled_after: nil,
-                recovered_at: nil,
-                breached_at: nil
-              )
+            state = @states[light_name]
+            if state.recovery_started_at.nil?
+              state.recovery_started_at = current_time
+              state.recovery_scheduled_after = nil
+              state.recovered_at = nil
+              state.breached_at = nil
               true
             else
-              @metadata[light_name] = metadata.with(
-                recovery_scheduled_after: nil,
-                recovered_at: nil,
-                breached_at: nil
-              )
+              state.recovery_scheduled_after = nil
+              state.recovered_at = nil
+              state.breached_at = nil
               false
             end
           end
@@ -278,21 +313,17 @@ module Stoplight
           recovery_scheduled_after = current_time + config.cool_off_time
 
           synchronize do
-            metadata = @metadata[light_name]
-            if metadata.breached_at
-              @metadata[light_name] = metadata.with(
-                recovery_scheduled_after: recovery_scheduled_after,
-                recovery_started_at: nil,
-                recovered_at: nil
-              )
+            state = @states[light_name]
+            if state.breached_at
+              state.recovery_scheduled_after = recovery_scheduled_after
+              state.recovery_started_at = nil
+              state.recovered_at = nil
               false
             else
-              @metadata[light_name] = metadata.with(
-                breached_at: current_time,
-                recovery_scheduled_after: recovery_scheduled_after,
-                recovery_started_at: nil,
-                recovered_at: nil
-              )
+              state.breached_at = current_time
+              state.recovery_scheduled_after = recovery_scheduled_after
+              state.recovery_started_at = nil
+              state.recovered_at = nil
               true
             end
           end

data/lib/stoplight/infrastructure/data_store/redis/lua_scripts/get_metrics.lua

@@ -0,0 +1,26 @@
+local number_of_metric_buckets = tonumber(ARGV[1])
+local window_start_ts = tonumber(ARGV[2])
+local window_end_ts = tonumber(ARGV[3])
+local metrics_keys = {}
+for idx = 4, #ARGV do
+  table.insert(metrics_keys, ARGV[idx])
+end
+
+local metadata_key = KEYS[1]
+
+local function count_events(start_idx, bucket_count, start_ts)
+  local total = 0
+  for idx = start_idx, start_idx + bucket_count - 1 do
+    total = total + tonumber(redis.call('ZCOUNT', KEYS[idx], start_ts, window_end_ts))
+  end
+  return total
+end
+
+local offset = 2
+local successes = count_events(2, number_of_metric_buckets, window_start_ts)
+
+offset = offset + number_of_metric_buckets
+local errors = count_events(offset, number_of_metric_buckets, window_start_ts)
+
+local metrics = redis.call('HMGET',  metadata_key, unpack(metrics_keys))
+return {successes, errors, unpack(metrics)}

data/lib/stoplight/infrastructure/data_store/redis/lua_scripts/record_recovery_probe_failure.lua

@@ -0,0 +1,27 @@
+local failure_ts = tonumber(ARGV[1])
+local failure_json = ARGV[2]
+
+local metadata_key = KEYS[1]
+
+
+-- Update metadata
+local meta = redis.call('HMGET', metadata_key, 'last_error_at', 'consecutive_errors')
+local prev_failure_ts = tonumber(meta[1])
+local prev_consecutive_errors = tonumber(meta[2])
+
+if not prev_failure_ts or failure_ts > prev_failure_ts then
+  redis.call(
+    'HSET', metadata_key,
+    'last_error_at', failure_ts,
+    'last_error_json', failure_json,
+    'consecutive_errors', (prev_consecutive_errors or 0) + 1,
+    'consecutive_successes', 0
+  )
+else
+  redis.call(
+    'HSET', metadata_key,
+    'consecutive_errors', (prev_consecutive_errors or 0) + 1,
+    'consecutive_successes', 0
+  )
+end
+

data/lib/stoplight/infrastructure/data_store/redis/lua_scripts/record_recovery_probe_success.lua

@@ -0,0 +1,23 @@
+local request_ts = tonumber(ARGV[1])
+
+local metadata_key = KEYS[1]
+
+-- Update metadata
+local meta = redis.call('HMGET', metadata_key, 'last_success_at', 'consecutive_successes')
+local prev_success_ts = tonumber(meta[1])
+local prev_consecutive_successes = tonumber(meta[2])
+
+if not prev_success_ts or request_ts > prev_success_ts then
+  redis.call(
+    'HSET', metadata_key,
+    'last_success_at', request_ts,
+    'consecutive_errors', 0,
+    'consecutive_successes', (prev_consecutive_successes or 0) + 1
+  )
+else
+  redis.call(
+    'HSET', metadata_key,
+    'consecutive_errors', 0,
+    'consecutive_successes', (prev_consecutive_successes or 0) + 1
+  )
+end

data/lib/stoplight/infrastructure/data_store/redis/lua_scripts/release_lock.lua

@@ -0,0 +1,6 @@
+local token = ARGV[1]
+local lock_key = KEYS[1]
+
+if redis.call("get", lock_key) == token then
+  return redis.call("del", lock_key)
+end

data/lib/stoplight/infrastructure/data_store/redis/recovery_lock_store.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "forwardable"
+
+module Stoplight
+  module Infrastructure
+    module DataStore
+      class Redis
+        # Distributed recovery recovery_lock using Redis SET NX (set-if-not-exists).
+        #
+        # Lock Acquisition:
+        # - Uses unique UUID token to prevent accidental release of others' locks
+        # - Atomic SET with NX flag ensures only one process acquires recovery_lock
+        # - TTL (px: lock_timeout) auto-releases recovery_lock if process crashes
+        #
+        # Lock Release:
+        # - Lua script ensures only token holder can release (token comparison)
+        # - Best-effort release; TTL cleanup handles failures
+        #
+        # Failure Modes:
+        # - Lock contention: Returns false, caller should skip probe
+        # - Redis unavailable: raises an error and let caller decide
+        # - Crashed holder: raises an error and let caller decide. Lock auto-expires after lock_timeout
+        # - Release failure: Lock auto-expires after lock_timeout
+        #
+        class RecoveryLockStore
+          # @!attribute redis
+          #   @return [RedisClient]
+          protected attr_reader :redis
+
+          # @!attribute lock_timeout
+          #   @return [Integer]
+          protected attr_reader :lock_timeout
+
+          # @!attribute scripting
+          #   @return [Stoplight::Infrastructure::DataStore::Redis::Scripting]
+          protected attr_reader :scripting
+
+          # @param redis [RedisClient | ConnectionPool]
+          # @param lock_timeout [Integer] recovery_lock timeout in milliseconds
+          # @param scripting [Stoplight::Infrastructure::DataStore::Redis::Scripting]
+          def initialize(redis:, lock_timeout:, scripting:)
+            @redis = redis
+            @lock_timeout = lock_timeout
+            @scripting = scripting
+          end
+
+          # @param light_name [String]
+          # @return [Stoplight::Infrastructure::DataStore::Redis::RecoveryLockToken, nil]
+          def acquire_lock(light_name)
+            recovery_lock = RecoveryLockToken.new(light_name:)
+
+            acquired = !!redis.then do |client|
+              client.set(recovery_lock.lock_key, recovery_lock.token, nx: true, px: lock_timeout)
+            end
+
+            recovery_lock if acquired
+          end
+
+          # @param recovery_lock [Stoplight::Infrastructure::DataStore::Redis::RecoveryLockToken]
+          # @return [void]
+          def release_lock(recovery_lock)
+            scripting.call(
+              :release_lock,
+              keys: [recovery_lock.lock_key], args: [recovery_lock.token]
+            )
+          end
+        end
+      end
+    end
+  end
+end