stoplight 5.5.0 → 5.7.0

data/lib/stoplight/infrastructure/data_store/redis/recovery_lock_token.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "forwardable"
+
+module Stoplight
+  module Infrastructure
+    module DataStore
+      class Redis
+        class RecoveryLockToken < Domain::RecoveryLockToken
+          extend Forwardable
+
+          def_delegator "Stoplight::Infrastructure::DataStore::Redis", :key
+          private :key
+
+          # @!attribute light_name
+          #   @return [String]
+          attr_reader :light_name
+
+          # @!attribute token
+          #   @return [String]
+          attr_reader :token
+
+          # @param light_name [String]
+          def initialize(light_name:)
+            @light_name = light_name
+            @token = SecureRandom.uuid
+          end
+
+          def lock_key = key(:locks, :recovery, light_name)
+        end
+      end
+    end
+  end
+end

data/lib/stoplight/infrastructure/data_store/redis/scripting.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Stoplight
+  module Infrastructure
+    module DataStore
+      class Redis
+        # Manages Lua scripts for Redis operations.
+        #
+        # This class provides execution of Lua scripts by caching their SHA digests
+        # and automatically reloading scripts if they're evicted from Redis memory.
+        #
+        # @example
+        #   script_manager = ScriptManager.new(redis: redis_client)
+        #   script_manager.call(:increment_counter, keys: ["counter:1"], args: [5])
+        #
+        # @note Scripts are loaded lazily on first use and cached in memory
+        # @note Script files must be named `<script_name>.lua` and located in scripts_root
+        class Scripting
+          SCRIPTS_ROOT = File.join(__dir__, "lua_scripts")
+          # @!attribute scripts_root
+          #   @return [String]
+          protected attr_reader :scripts_root
+
+          # @!attribute shas
+          #   @return [Hash{Symbol, String}]
+          private attr_reader :shas
+
+          # @!attribute redis
+          #   @return [RedisClient | ConnectionPool]
+          protected attr_reader :redis
+
+          # @param redis [RedisClient | ConnectionPool]
+          # @param scripts_root [String]
+          def initialize(redis:, scripts_root: SCRIPTS_ROOT)
+            @scripts_root = scripts_root
+            @redis = redis
+            @shas = {}
+          end
+
+          def call(script_name, keys: [], args: [])
+            redis.then do |client|
+              client.evalsha(script_sha(script_name), keys: keys, argv: args)
+            end
+          rescue ::Redis::CommandError => error
+            if error.message.include?("NOSCRIPT")
+              reload_script(script_name)
+              retry
+            else
+              raise
+            end
+          end
+
+          private def reload_script(script_name)
+            shas.delete(script_name)
+            script_sha(script_name)
+          end
+
+          private def script_sha(script_name)
+            if shas.key?(script_name)
+              shas[script_name]
+            else
+              script = File.read(File.join(scripts_root, "#{script_name}.lua"))
+
+              shas[script_name] = redis.then { |client| client.script("load", script) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/stoplight/infrastructure/data_store/redis.rb

@@ -74,30 +74,55 @@ module Stoplight
         KEY_SEPARATOR = ":"
         KEY_PREFIX = %w[stoplight v5].join(KEY_SEPARATOR)
 
+        # @!attribute recovery_lock_store
+        #   @return [Stoplight::Infrastructure::DataStore::Redis::RecoveryLockStore]
+        protected attr_reader :recovery_lock_store
+
+        # @!attribute scripting
+        #   @return [Stoplight::Infrastructure::DataStore::Redis::Scripting]
+        protected attr_reader :scripting
+
+        # @!attribute redis
+        #   @return [::Redis | ConnectionPool<::Redis>]
+        protected attr_reader :redis
+
+        # @!attribute warn_on_clock_skew
+        #   @return [Boolean]
+        protected attr_reader :warn_on_clock_skew
+
         # @param redis [::Redis, ConnectionPool<::Redis>]
+        # @param recovery_lock_store [Stoplight::Infrastructure::DataStore::Redis::RecoveryLockStore]
         # @param warn_on_clock_skew [Boolean] (true) Whether to warn about clock skew between Redis and
+        # @param scripting [Stoplight::Infrastructure::DataStore::Redis::Scripting]
         #   the application server
-        def initialize(redis, warn_on_clock_skew: true)
+        def initialize(redis:, recovery_lock_store:, scripting:, warn_on_clock_skew: true)
           @warn_on_clock_skew = warn_on_clock_skew
           @redis = redis
+          @recovery_lock_store = recovery_lock_store
+          @scripting = scripting
         end
 
         def names
-          pattern = key("metadata", "*")
-          prefix_regex = /^#{key("metadata", "")}/
-          @redis.then do |client|
-            client.scan_each(match: pattern).to_a.map do |key|
-              key.sub(prefix_regex, "")
+          metadata, recovery_metrics = @redis.then do |client|
+            [
+              [key("metadata", "*"), /^#{key("metadata", "")}/],
+              [key("recovery_metrics", "*"), /^#{key("recovery_metrics", "")}/]
+            ].map do |(pattern, prefix_regex)|
+              client.scan_each(match: pattern).to_a.map do |key|
+                key.sub(prefix_regex, "")
+              end
             end
           end
+          metadata + recovery_metrics
         end
 
-        def get_metadata(config)
-          detect_clock_skew
+        # @param config [Stoplight::Domain::Config]
+        # @return [Stoplight::Domain::Metrics]
+        def get_metrics(config)
+          config.name
 
           window_end_ts = current_time.to_f
-          window_start_ts = window_end_ts - [config.window_size, METRICS_RETENTION_TIME].compact.min.to_f
-          recovery_window_start_ts = window_end_ts - config.cool_off_time.to_i
+          window_start_ts = window_end_ts - config.window_size.to_i
 
           if config.window_size
             failure_keys = failure_bucket_keys(config, window_end: window_end_ts)
@@ -106,129 +131,172 @@ module Stoplight
             failure_keys = []
             success_keys = []
           end
-          recovery_probe_failure_keys = recovery_probe_failure_bucket_keys(config, window_end: window_end_ts)
-          recovery_probe_success_keys = recovery_probe_success_bucket_keys(config, window_end: window_end_ts)
-
-          successes, errors, recovery_probe_successes, recovery_probe_errors, meta = @redis.with do |client|
-            client.evalsha(
-              get_metadata_sha,
-              argv: [
-                failure_keys.count,
-                recovery_probe_failure_keys.count,
-                window_start_ts,
-                window_end_ts,
-                recovery_window_start_ts
-              ],
-              keys: [
-                metadata_key(config),
-                *success_keys,
-                *failure_keys,
-                *recovery_probe_success_keys,
-                *recovery_probe_failure_keys
-              ]
+
+          successes, errors, last_success_at, last_error_json, consecutive_errors, consecutive_successes = scripting.call(
+            :get_metrics,
+            args: [
+              failure_keys.count,
+              window_start_ts,
+              window_end_ts,
+              "last_success_at", "last_error_json", "consecutive_errors", "consecutive_successes"
+            ],
+            keys: [
+              metadata_key(config),
+              *success_keys,
+              *failure_keys
+            ]
+          )
+          consecutive_errors = config.window_size ? [consecutive_errors.to_i, errors].min : consecutive_errors.to_i
+          consecutive_successes = config.window_size ? [consecutive_successes.to_i, successes].min : consecutive_successes.to_i
+
+          Domain::Metrics.new(
+            successes: (successes if config.window_size),
+            errors: (errors if config.window_size),
+            consecutive_errors:,
+            consecutive_successes:,
+            last_error: deserialize_failure(last_error_json),
+            last_success_at: (Time.at(last_success_at.to_f) if last_success_at)
+          )
+        end
+
+        # @param config [Stoplight::Domain::Config]
+        # @return [Stoplight::Domain::Metrics]
+        def get_recovery_metrics(config)
+          last_success_at, last_error_json, consecutive_errors, consecutive_successes = @redis.with do |client|
+            client.hmget(
+              recovery_metrics_key(config),
+              "last_success_at", "last_error_json", "consecutive_errors", "consecutive_successes"
             )
           end
-          meta_hash = meta.each_slice(2).to_h.transform_keys(&:to_sym)
-          last_error_json = meta_hash.delete(:last_error_json)
-          last_error = deserialize_failure(last_error_json) if last_error_json
-
-          Domain::Metadata.new(
-            current_time:,
-            successes:,
-            errors:,
-            recovery_probe_successes:,
-            recovery_probe_errors:,
-            last_error:,
-            last_error_at: (Time.at(meta_hash[:last_error_at].to_f) if meta_hash[:last_error_at]),
-            last_success_at: (Time.at(meta_hash[:last_success_at].to_f) if meta_hash[:last_success_at]),
-            consecutive_errors: meta_hash[:consecutive_errors].to_i,
-            consecutive_successes: meta_hash[:consecutive_successes].to_i,
-            breached_at: (Time.at(meta_hash[:breached_at].to_f) if meta_hash[:breached_at]),
-            locked_state: meta_hash[:locked_state] || Domain::State::UNLOCKED,
-            recovery_scheduled_after: (Time.at(meta_hash[:recovery_scheduled_after].to_f) if meta_hash[:recovery_scheduled_after]),
-            recovery_started_at: (Time.at(meta_hash[:recovery_started_at].to_f) if meta_hash[:recovery_started_at]),
-            recovered_at: (Time.at(meta_hash[:recovered_at].to_f) if meta_hash[:recovered_at])
+
+          Domain::Metrics.new(
+            successes: nil, errors: nil,
+            consecutive_errors: consecutive_errors.to_i,
+            consecutive_successes: consecutive_successes.to_i,
+            last_error: deserialize_failure(last_error_json),
+            last_success_at: (Time.at(last_success_at.to_f) if last_success_at)
+          )
+        end
+
+        # @return [Stoplight::Domain::StateSnapshot]
+        def get_state_snapshot(config)
+          detect_clock_skew
+
+          breached_at_raw, locked_state, recovery_scheduled_after_raw, recovery_started_at_raw = @redis.with do |client|
+            client.hmget(metadata_key(config), :breached_at, :locked_state, :recovery_scheduled_after, :recovery_started_at)
+          end
+          breached_at = breached_at_raw&.to_f
+          recovery_scheduled_after = recovery_scheduled_after_raw&.to_f
+          recovery_started_at = recovery_started_at_raw&.to_f
+
+          Domain::StateSnapshot.new(
+            breached_at: (Time.at(breached_at) if breached_at),
+            locked_state: locked_state || Domain::State::UNLOCKED,
+            recovery_scheduled_after: (Time.at(recovery_scheduled_after) if recovery_scheduled_after),
+            recovery_started_at: (Time.at(recovery_started_at) if recovery_started_at),
+            time: current_time
+          )
+        end
+
+        def clear_metrics(config)
+          if config.window_size
+            window_end_ts = current_time.to_i
+            @redis.with do |client|
+              client.multi do |tx|
+                tx.unlink(
+                  *failure_bucket_keys(config, window_end: window_end_ts),
+                  *success_bucket_keys(config, window_end: window_end_ts)
+                )
+                tx.hdel(metadata_key(config), "last_success_at", "last_error_json", "consecutive_errors", "consecutive_successes")
+              end
+            end
+          else
+            @redis.with do |client|
+              client.hdel(metadata_key(config), "last_success_at", "last_error_json", "consecutive_errors", "consecutive_successes")
+            end
+          end
+        end
+
+        def clear_recovery_metrics(config)
+          @redis.with do |client|
+            client.del(recovery_metrics_key(config))
+          end
+        end
+
+        private def state_snapshot_from_hash(data, time: current_time)
+          breached_at = data[:breached_at]&.to_f
+          recovery_scheduled_after = data[:recovery_scheduled_after]&.to_f
+          recovery_started_at = data[:recovery_started_at]&.to_f
+
+          Domain::StateSnapshot.new(
+            breached_at: (Time.at(breached_at) if breached_at),
+            locked_state: data[:locked_state] || Domain::State::UNLOCKED,
+            recovery_scheduled_after: (Time.at(recovery_scheduled_after) if recovery_scheduled_after),
+            recovery_started_at: (Time.at(recovery_started_at) if recovery_started_at),
+            time:
           )
         end
 
         # @param config [Stoplight::Domain::Config] The light configuration.
         # @param exception [Exception]
-        # @return [Stoplight::Domain::Metadata] The updated metadata after recording the failure.
+        # @return [void]
         def record_failure(config, exception)
           current_time = self.current_time
           current_ts = current_time.to_f
           failure = Domain::Failure.from_error(exception, time: current_time)
 
-          @redis.then do |client|
-            client.evalsha(
-              record_failure_sha,
-              argv: [current_ts, SecureRandom.hex(12), serialize_failure(failure), metrics_ttl, metadata_ttl],
-              keys: [
-                metadata_key(config),
-                config.window_size && errors_key(config, time: current_ts)
-              ].compact
-            )
-          end
-          get_metadata(config)
+          scripting.call(
+            :record_failure,
+            args: [current_ts, SecureRandom.hex(12), serialize_failure(failure), metrics_ttl, metadata_ttl],
+            keys: [
+              metadata_key(config),
+              config.window_size && errors_key(config, time: current_ts)
+            ].compact
+          )
         end
 
         def record_success(config, request_id: SecureRandom.hex(12))
           current_ts = current_time.to_f
 
-          @redis.then do |client|
-            client.evalsha(
-              record_success_sha,
-              argv: [current_ts, request_id, metrics_ttl, metadata_ttl],
-              keys: [
-                metadata_key(config),
-                config.window_size && successes_key(config, time: current_ts)
-              ].compact
-            )
-          end
+          scripting.call(
+            :record_success,
+            args: [current_ts, request_id, metrics_ttl, metadata_ttl],
+            keys: [
+              metadata_key(config),
+              config.window_size && successes_key(config, time: current_ts)
+            ].compact
+          )
         end
 
         # Records a failed recovery probe for a specific light configuration.
         #
         # @param config [Stoplight::Domain::Config] The light configuration.
         # @param exception [Exception]
-        # @return [Stoplight::Domain::Metadata] The updated metadata after recording the failure.
+        # @return [void]
         def record_recovery_probe_failure(config, exception)
           current_time = self.current_time
           current_ts = current_time.to_f
           failure = Domain::Failure.from_error(exception, time: current_time)
 
-          @redis.then do |client|
-            client.evalsha(
-              record_failure_sha,
-              argv: [current_ts, SecureRandom.uuid, serialize_failure(failure), metrics_ttl, metrics_ttl],
-              keys: [
-                metadata_key(config),
-                recovery_probe_errors_key(config, time: current_ts)
-              ].compact
-            )
-          end
-          get_metadata(config)
+          scripting.call(
+            :record_recovery_probe_failure,
+            args: [current_ts, serialize_failure(failure)],
+            keys: [recovery_metrics_key(config)]
+          )
         end
 
         # Records a successful recovery probe for a specific light configuration.
         #
         # @param config [Stoplight::Domain::Config] The light configuration.
-        # @param request_id [String] The unique identifier for the request
-        # @return [Stoplight::Domain::Metadata] The updated metadata after recording the success.
-        def record_recovery_probe_success(config, request_id: SecureRandom.hex(12))
+        # @return [void]
+        def record_recovery_probe_success(config)
           current_ts = current_time.to_f
 
-          @redis.then do |client|
-            client.evalsha(
-              record_success_sha,
-              argv: [current_ts, request_id, metrics_ttl, metadata_ttl],
-              keys: [
-                metadata_key(config),
-                recovery_probe_successes_key(config, time: current_ts)
-              ].compact
-            )
-          end
-          get_metadata(config)
+          scripting.call(
+            :record_recovery_probe_success,
+            args: [current_ts],
+            keys: [recovery_metrics_key(config)]
+          )
         end
 
         def set_state(config, state)
@@ -260,6 +328,18 @@ module Stoplight
           end
         end
 
+        # @param config [Stoplight::Domain::Config]
+        # @return [Stoplight::Infrastructure::DataStore::Redis::RecoveryLockToken, nil]
+        def acquire_recovery_lock(config)
+          recovery_lock_store.acquire_lock(config.name)
+        end
+
+        # @param lock [Stoplight::Infrastructure::DataStore::Redis::RecoveryLockToken]
+        # @return [void]
+        def release_recovery_lock(lock)
+          recovery_lock_store.release_lock(lock)
+        end
+
         # Transitions to GREEN state and ensures only one notification
         #
         # @param config [Stoplight::Domain::Config] The light configuration
@@ -268,13 +348,11 @@ module Stoplight
           current_ts = current_time.to_f
           meta_key = metadata_key(config)
 
-          became_green = @redis.then do |client|
-            client.evalsha(
-              transition_to_green_sha,
-              argv: [current_ts],
-              keys: [meta_key]
-            )
-          end
+          became_green = scripting.call(
+            :transition_to_green,
+            args: [current_ts],
+            keys: [meta_key]
+          )
           became_green == 1
         end
 
@@ -283,16 +361,14 @@ module Stoplight
         # @param config [Stoplight::Domain::Config] The light configuration
         # @return [Boolean] true if this is the first instance to detect this transition
         private def transition_to_yellow(config)
-          current_ts = current_time.to_i
+          current_ts = current_time.to_f
           meta_key = metadata_key(config)
 
-          became_yellow = @redis.then do |client|
-            client.evalsha(
-              transition_to_yellow_sha,
-              argv: [current_ts],
-              keys: [meta_key]
-            )
-          end
+          became_yellow = scripting.call(
+            :transition_to_yellow,
+            args: [current_ts],
+            keys: [meta_key]
+          )
           became_yellow == 1
         end
 
@@ -301,24 +377,34 @@ module Stoplight
         # @param config [Stoplight::Domain::Config] The light configuration
         # @return [Boolean] true if this is the first instance to detect this transition
         private def transition_to_red(config)
-          current_ts = current_time.to_i
+          current_ts = current_time.to_f
           meta_key = metadata_key(config)
           recovery_scheduled_after_ts = current_ts + config.cool_off_time
 
-          became_red = @redis.then do |client|
-            client.evalsha(
-              transition_to_red_sha,
-              argv: [current_ts, recovery_scheduled_after_ts],
-              keys: [meta_key]
-            )
-          end
+          became_red = scripting.call(
+            :transition_to_red,
+            args: [current_ts, recovery_scheduled_after_ts],
+            keys: [meta_key]
+          )
 
           became_red == 1
         end
 
-        # @param failure_json [String]
-        # @return [Domain::Failure]
+        # Removes all traces of a light from Redis metadata (metrics will expire by TTL).
+        #
+        # @param config [Stoplight::Domain::Config] The light configuration.
+        # @return [void]
+        def delete_light(config)
+          @redis.then do |client|
+            client.del(metadata_key(config), recovery_metrics_key(config))
+          end
+        end
+
+        # @param failure_json [String, nil]
+        # @return [Domain::Failure, nil]
         private def deserialize_failure(failure_json)
+          return if failure_json.nil?
+
           object = JSON.parse(failure_json)
           error_object = object["error"]
 
@@ -389,12 +475,8 @@ module Stoplight
           self.class.bucket_key(config.name, metric: "failure", time:)
         end
 
-        private def recovery_probe_successes_key(config, time:)
-          self.class.bucket_key(config.name, metric: "recovery_probe_success", time:)
-        end
-
-        private def recovery_probe_errors_key(config, time:)
-          self.class.bucket_key(config.name, metric: "recovery_probe_failure", time:)
+        private def recovery_metrics_key(config)
+          key("recovery_metrics", config.name)
         end
 
         private def metadata_key(config)
@@ -433,42 +515,6 @@ module Stoplight
           rand <= probability
         end
 
-        private def record_success_sha
-          @record_success_sha ||= @redis.then do |client|
-            client.script("load", Lua::RECORD_SUCCESS)
-          end
-        end
-
-        private def get_metadata_sha
-          @get_metadata_sha ||= @redis.then do |client|
-            client.script("load", Lua::GET_METADATA)
-          end
-        end
-
-        private def transition_to_yellow_sha
-          @transition_to_yellow_sha ||= @redis.then do |client|
-            client.script("load", Lua::TRANSITION_TO_YELLOW)
-          end
-        end
-
-        private def transition_to_red_sha
-          @transition_to_red_sha ||= @redis.then do |client|
-            client.script("load", Lua::TRANSITION_TO_RED)
-          end
-        end
-
-        private def transition_to_green_sha
-          @transition_to_green_sha ||= @redis.then do |client|
-            client.script("load", Lua::TRANSITION_TO_GREEN)
-          end
-        end
-
-        private def record_failure_sha
-          @record_failure_sha ||= @redis.then do |client|
-            client.script("load", Lua::RECORD_FAILURE)
-          end
-        end
-
         private def current_time
           Time.now
         end

data/lib/stoplight/infrastructure/notifier/fail_safe.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Stoplight
+  module Infrastructure
+    module Notifier
+      # A wrapper around a notifier that provides fail-safe mechanisms using a
+      # circuit breaker. It ensures that a notification can gracefully
+      # handle failures.
+      #
+      # @api private
+      class FailSafe < Domain::StateTransitionNotifier
+        # @!attribute [r] notifier
+        #   @return [Stoplight::Domain::StateTransitionNotifier] The underlying notifier being wrapped.
+        attr_reader :notifier
+
+        # @!attribute [r] error_notifier
+        #   @return [Stoplight::Domain::StateTransitionNotifier] The underlying notifier being wrapped.
+        attr_reader :error_notifier
+
+        # Initializes a new instance of the +FailSafe+ class.
+        #
+        # @param notifier [Stoplight::Domain::StateTransitionNotifier] The notifier to wrap.
+        # @param error_notifier [Proc] called when wrapped data store fails
+        def initialize(notifier:, error_notifier:)
+          @notifier = notifier
+          @error_notifier = error_notifier
+        end
+
+        # Sends a notification using the wrapped notifier with fail-safe mechanisms.
+        #
+        # @param config [Stoplight::Domain::Config] The light configuration.
+        # @param from_color [String] The initial color of the light.
+        # @param to_color [String] The target color of the light.
+        # @param error [Exception, nil] An optional error to include in the notification.
+        # @return [void]
+        def notify(config, from_color, to_color, error = nil)
+          fallback = proc do |exception|
+            error_notifier.call(exception) if exception
+            nil
+          end
+
+          circuit_breaker.run(fallback) do
+            notifier.notify(config, from_color, to_color, error)
+          end
+        end
+
+        # @return [Boolean]
+        def ==(other)
+          other.is_a?(self.class) && notifier == other.notifier
+        end
+
+        # @return [Stoplight::Light] The circuit breaker used to handle failures.
+        private def circuit_breaker
+          @circuit_breaker ||= Stoplight.system_light(
+            "stoplight:notifier:fail_safe:#{notifier.class.name}",
+            notifiers: []
+          )
+        end
+      end
+    end
+  end
+end