stoplight 4.1.1 → 5.0.2

data/lib/stoplight/data_store/redis.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'redlock'
+require "forwardable"
 
 module Stoplight
   module DataStore
@@ -12,172 +12,432 @@ module Stoplight
     # This data structure enables us to query errors that happened within a specific
     # period. We use this feature to support +window_size+ option.
     #
-    # To avoid uncontrolled memory consumption, we keep at most +light.threshold+ number
-    # of errors happened within last +light.window_size+ seconds (by default infinity).
+    # To avoid uncontrolled memory consumption, we keep at most +config.threshold+ number
+    # of errors happened within last +config.window_size+ seconds (by default infinity).
     #
     # @see Base
     class Redis < Base
-      KEY_SEPARATOR = ':'
-      KEY_PREFIX = %w[stoplight v4].join(KEY_SEPARATOR)
+      extend Forwardable
+
+      class << self
+        # Generates a Redis key by joining the prefix with the provided pieces.
+        #
+        # @param pieces [Array<String, Integer>] Parts of the key to be joined.
+        # @return [String] The generated Redis key.
+        # @api private
+        def key(*pieces)
+          [KEY_PREFIX, *pieces].join(KEY_SEPARATOR)
+        end
+
+        # Retrieves the list of Redis bucket keys required to cover a specific time window.
+        #
+        # @param light_name [String] The name of the light (used as part of the Redis key).
+        # @param metric [String] The metric type (e.g., "errors").
+        # @param window_end [Time, Numeric] The end time of the window (can be a Time object or a numeric timestamp).
+        # @param window_size [Numeric] The size of the time window in seconds.
+        # @return [Array<String>] A list of Redis keys for the buckets that cover the time window.
+        # @api private
+        def buckets_for_window(light_name, metric:, window_end:, window_size:)
+          window_end_ts = window_end.to_i
+          window_start_ts = window_end_ts - [window_size, Base::METRICS_RETENTION_TIME].compact.min.to_i
+
+          # Find bucket timestamps that contain any part of the window
+          start_bucket = (window_start_ts / bucket_size) * bucket_size
+
+          # End bucket is the last bucket that contains data within our window
+          end_bucket = ((window_end_ts - 1) / bucket_size) * bucket_size
+
+          (start_bucket..end_bucket).step(bucket_size).map do |bucket_start|
+            bucket_key(light_name, metric: metric, time: bucket_start)
+          end
+        end
+
+        # Generates a Redis key for a specific metric and time.
+        #
+        # @param light_name [String] The name of the light.
+        # @param metric [String] The metric type (e.g., "errors").
+        # @param time [Time, Numeric] The time for which to generate the key.
+        # @return [String] The generated Redis key.
+        def bucket_key(light_name, metric:, time:)
+          key("metrics", light_name, metric, (time.to_i / bucket_size) * bucket_size)
+        end
+
+        BUCKET_SIZE = 3600 # 1h
+        private_constant :BUCKET_SIZE
 
-      # @param redis [::Redis]
-      def initialize(redis, redlock: Redlock::Client.new([redis]))
+        private def bucket_size
+          BUCKET_SIZE
+        end
+      end
+
+      KEY_SEPARATOR = ":"
+      KEY_PREFIX = %w[stoplight v5].join(KEY_SEPARATOR)
+
+      # @param redis [::Redis, ConnectionPool<::Redis>]
+      # @param warn_on_clock_skew [Boolean] (true) Whether to warn about clock skew between Redis and
+      #   the application server
+      def initialize(redis, warn_on_clock_skew: true)
+        @warn_on_clock_skew = warn_on_clock_skew
         @redis = redis
-        @redlock = redlock
       end
 
       def names
-        state_names = @redis.hkeys(states_key)
-
-        pattern = key('failures', '*')
-        prefix_regex = /^#{key('failures', '')}/
-        failure_names = @redis.scan_each(match: pattern).to_a.map do |key|
-          key.sub(prefix_regex, '')
+        pattern = key("metadata", "*")
+        prefix_regex = /^#{key("metadata", "")}/
+        @redis.then do |client|
+          client.scan_each(match: pattern).to_a.map do |key|
+            key.sub(prefix_regex, "")
+          end
         end
+      end
 
-        (state_names + failure_names).uniq
+      def get_metadata(config)
+        detect_clock_skew
+
+        window_end = Time.now
+        window_end_ts = window_end.to_i
+        window_start_ts = window_end_ts - [config.window_size, Base::METRICS_RETENTION_TIME].compact.min.to_i
+        recovery_window_start_ts = window_end_ts - config.cool_off_time.to_i
+
+        if config.window_size
+          failure_keys = failure_bucket_keys(config, window_end: window_end_ts)
+          success_keys = success_bucket_keys(config, window_end: window_end_ts)
+        else
+          failure_keys = []
+          success_keys = []
+        end
+        recovery_probe_failure_keys = recovery_probe_failure_bucket_keys(config, window_end: window_end_ts)
+        recovery_probe_success_keys = recovery_probe_success_bucket_keys(config, window_end: window_end_ts)
+
+        successes, errors, recovery_probe_successes, recovery_probe_errors, meta = @redis.with do |client|
+          client.evalsha(
+            get_metadata_sha,
+            argv: [
+              failure_keys.count,
+              recovery_probe_failure_keys.count,
+              window_start_ts,
+              window_end_ts,
+              recovery_window_start_ts
+            ],
+            keys: [
+              metadata_key(config),
+              *success_keys,
+              *failure_keys,
+              *recovery_probe_success_keys,
+              *recovery_probe_failure_keys
+            ]
+          )
+        end
+        meta_hash = meta.each_slice(2).to_h.transform_keys(&:to_sym)
+        last_error_json = meta_hash.delete(:last_error_json)
+        last_error = normalize_failure(last_error_json, config.error_notifier) if last_error_json
+
+        Metadata.new(
+          successes: successes,
+          errors: errors,
+          recovery_probe_successes: recovery_probe_successes,
+          recovery_probe_errors: recovery_probe_errors,
+          last_error:,
+          **meta_hash
+        )
       end
 
-      def get_all(light)
-        failures, state = @redis.multi do |transaction|
-          query_failures(light, transaction: transaction)
-          transaction.hget(states_key, light.name)
+      # @param config [Stoplight::Light::Config] The light configuration.
+      # @param failure [Stoplight::Failure] The failure to record.
+      # @return [Stoplight::Metadata] The updated metadata after recording the failure.
+      def record_failure(config, failure)
+        current_ts = failure.time.to_i
+        failure_json = failure.to_json
+
+        @redis.then do |client|
+          client.evalsha(
+            record_failure_sha,
+            argv: [current_ts, SecureRandom.hex(12), failure_json, metrics_ttl, metadata_ttl],
+            keys: [
+              metadata_key(config),
+              config.window_size && errors_key(config, time: current_ts)
+            ].compact
+          )
         end
+        get_metadata(config)
+      end
 
-        [
-          normalize_failures(failures, light.error_notifier),
-          normalize_state(state)
-        ]
+      def record_success(config, request_id: SecureRandom.hex(12), request_time: Time.now)
+        request_ts = request_time.to_i
+
+        @redis.then do |client|
+          client.evalsha(
+            record_success_sha,
+            argv: [request_ts, request_id, metrics_ttl, metadata_ttl],
+            keys: [
+              metadata_key(config),
+              config.window_size && successes_key(config, time: request_ts)
+            ].compact
+          )
+        end
       end
 
-      def get_failures(light)
-        normalize_failures(query_failures(light), light.error_notifier)
+      # Records a failed recovery probe for a specific light configuration.
+      #
+      # @param config [Stoplight::Light::Config] The light configuration.
+      # @param failure [Failure] The failure to record.
+      # @return [Stoplight::Metadata] The updated metadata after recording the failure.
+      def record_recovery_probe_failure(config, failure)
+        current_ts = failure.time.to_i
+        failure_json = failure.to_json
+
+        @redis.then do |client|
+          client.evalsha(
+            record_failure_sha,
+            argv: [current_ts, SecureRandom.uuid, failure_json, metrics_ttl, metrics_ttl],
+            keys: [
+              metadata_key(config),
+              recovery_probe_errors_key(config, time: current_ts)
+            ].compact
+          )
+        end
+        get_metadata(config)
       end
 
-      # Saves a new failure to the errors HSet and cleans up outdated errors.
-      def record_failure(light, failure)
-        *, size = @redis.multi do |transaction|
-          failures_key = failures_key(light)
+      # Records a successful recovery probe for a specific light configuration.
+      #
+      # @param config [Stoplight::Light::Config] The light configuration.
+      # @param request_id [String] The unique identifier for the request
+      # @param request_time [Time] The time of the request
+      # @return [Stoplight::Metadata] The updated metadata after recording the success.
+      def record_recovery_probe_success(config, request_id: SecureRandom.hex(12), request_time: Time.now)
+        request_ts = request_time.to_i
+
+        @redis.then do |client|
+          client.evalsha(
+            record_success_sha,
+            argv: [request_ts, request_id, metrics_ttl, metadata_ttl],
+            keys: [
+              metadata_key(config),
+              recovery_probe_successes_key(config, time: request_ts)
+            ].compact
+          )
+        end
+        get_metadata(config)
+      end
 
-          transaction.zadd(failures_key, failure.time.to_i, failure.to_json)
-          remove_outdated_failures(light, failure.time, transaction: transaction)
-          transaction.zcard(failures_key)
+      def set_state(config, state)
+        @redis.then do |client|
+          client.hset(metadata_key(config), "locked_state", state)
         end
+        state
+      end
 
-        size
+      # Combined method that performs the state transition based on color
+      #
+      # @param config [Stoplight::Light::Config] The light configuration
+      # @param color [String] The color to transition to ("green", "yellow", or "red")
+      # @param current_time [Time] Current timestamp
+      # @return [Boolean] true if this is the first instance to detect this transition
+      def transition_to_color(config, color, current_time: Time.now)
+        current_time.to_i
+
+        case color
+        when Color::GREEN
+          transition_to_green(config)
+        when Color::YELLOW
+          transition_to_yellow(config, current_time:)
+        when Color::RED
+          transition_to_red(config, current_time:)
+        else
+          raise ArgumentError, "Invalid color: #{color}"
+        end
       end
 
-      def clear_failures(light)
-        failures, = @redis.multi do |transaction|
-          query_failures(light, transaction: transaction)
-          transaction.del(failures_key(light))
+      # Transitions to GREEN state and ensures only one notification
+      #
+      # @param config [Stoplight::Light::Config] The light configuration
+      # @return [Boolean] true if this is the first instance to detect this transition
+      private def transition_to_green(config, current_time: Time.now)
+        current_ts = current_time.to_i
+        meta_key = metadata_key(config)
+
+        became_green = @redis.then do |client|
+          client.evalsha(
+            transition_to_green_sha,
+            argv: [current_ts],
+            keys: [meta_key]
+          )
         end
+        became_green == 1
+      end
 
-        normalize_failures(failures, light.error_notifier)
+      # Transitions to YELLOW (recovery) state and ensures only one notification
+      #
+      # @param config [Stoplight::Light::Config] The light configuration
+      # @param current_time [Time] Current timestamp
+      # @return [Boolean] true if this is the first instance to detect this transition
+      private def transition_to_yellow(config, current_time: Time.now)
+        current_ts = current_time.to_i
+        meta_key = metadata_key(config)
+
+        became_yellow = @redis.then do |client|
+          client.evalsha(
+            transition_to_yellow_sha,
+            argv: [current_ts],
+            keys: [meta_key]
+          )
+        end
+        became_yellow == 1
       end
 
-      def get_state(light)
-        query_state(light) || State::UNLOCKED
+      # Transitions to RED state and ensures only one notification
+      #
+      # @param config [Stoplight::Light::Config] The light configuration
+      # @param current_time [Time] Current timestamp
+      # @return [Boolean] true if this is the first instance to detect this transition
+      private def transition_to_red(config, current_time: Time.now)
+        current_ts = current_time.to_i
+        meta_key = metadata_key(config)
+        recovery_scheduled_after_ts = current_ts + config.cool_off_time
+
+        became_red = @redis.then do |client|
+          client.evalsha(
+            transition_to_red_sha,
+            argv: [current_ts, recovery_scheduled_after_ts],
+            keys: [meta_key]
+          )
+        end
+
+        became_red == 1
       end
 
-      def set_state(light, state)
-        @redis.hset(states_key, light.name, state)
-        state
+      private def normalize_failure(failure, error_notifier)
+        Failure.from_json(failure)
+      rescue => e
+        error_notifier.call(e)
+        Failure.from_error(e)
       end
 
-      def clear_state(light)
-        state, = @redis.multi do |transaction|
-          query_state(light, transaction: transaction)
-          transaction.hdel(states_key, light.name)
-        end
+      def_delegator "self.class", :key
 
-        normalize_state(state)
+      private def failure_bucket_keys(config, window_end:)
+        self.class.buckets_for_window(
+          config.name,
+          metric: "failure",
+          window_end: window_end,
+          window_size: config.window_size
+        )
       end
 
-      LOCK_TTL = 2_000 # milliseconds
+      private def success_bucket_keys(config, window_end:)
+        self.class.buckets_for_window(
+          config.name,
+          metric: "success",
+          window_end: window_end,
+          window_size: config.window_size
+        )
+      end
 
-      def with_notification_lock(light, from_color, to_color)
-        @redlock.lock(notification_lock_key(light), LOCK_TTL) do
-          if last_notification(light) != [from_color, to_color]
-            set_last_notification(light, from_color, to_color)
+      private def recovery_probe_failure_bucket_keys(config, window_end:)
+        self.class.buckets_for_window(
+          config.name,
+          metric: "recovery_probe_failure",
+          window_end: window_end,
+          window_size: config.cool_off_time
+        )
+      end
 
-            yield
-          end
-        end
+      private def recovery_probe_success_bucket_keys(config, window_end:)
+        self.class.buckets_for_window(
+          config.name,
+          metric: "recovery_probe_success",
+          window_end: window_end,
+          window_size: config.cool_off_time
+        )
       end
 
-      private
+      private def successes_key(config, time:)
+        self.class.bucket_key(config.name, metric: "success", time:)
+      end
 
-      # @param light [Stoplight::Light]
-      # @param time [Time]
-      def remove_outdated_failures(light, time, transaction: @redis)
-        failures_key = failures_key(light)
+      private def errors_key(config, time:)
+        self.class.bucket_key(config.name, metric: "failure", time:)
+      end
 
-        # Remove all errors happened before the window start
-        transaction.zremrangebyscore(failures_key, 0, time.to_i - light.window_size)
-        # Keep at most +light.threshold+ number of errors
-        transaction.zremrangebyrank(failures_key, 0, -light.threshold - 1)
+      private def recovery_probe_successes_key(config, time:)
+        self.class.bucket_key(config.name, metric: "recovery_probe_success", time:)
       end
 
-      # @param light [Stoplight::Light]
-      # @return [Array, nil]
-      def last_notification(light)
-        @redis.get(last_notification_key(light))&.split('->')
+      private def recovery_probe_errors_key(config, time:)
+        self.class.bucket_key(config.name, metric: "recovery_probe_failure", time:)
       end
 
-      # @param light [Stoplight::Light]
-      # @param from_color [String]
-      # @param to_color [String]
-      # @return [void]
-      def set_last_notification(light, from_color, to_color)
-        @redis.set(last_notification_key(light), [from_color, to_color].join('->'))
+      private def metadata_key(config)
+        key("metadata", config.name)
       end
 
-      def query_failures(light, transaction: @redis)
-        window_start = Time.now.to_i - light.window_size
+      METRICS_TTL = 86400 # 1 day
+      private_constant :METRICS_TTL
 
-        transaction.zrange(failures_key(light), Float::INFINITY, window_start, rev: true, by_score: true)
+      private def metrics_ttl
+        METRICS_TTL
       end
 
-      def normalize_failures(failures, error_notifier)
-        failures.map do |json|
-          Failure.from_json(json)
-        rescue StandardError => e
-          error_notifier.call(e)
-          Failure.from_error(e)
+      METADATA_TTL = 86400 * 7 # 7 days
+      private_constant :METADATA_TTL
+
+      private def metadata_ttl
+        METADATA_TTL
+      end
+
+      SKEW_TOLERANCE = 5 # seconds
+      private_constant :SKEW_TOLERANCE
+
+      private def detect_clock_skew
+        return unless @warn_on_clock_skew
+        return unless should_sample?(0.01) # 1% chance
+
+        redis_seconds, _redis_millis = @redis.then(&:time)
+        app_seconds = Time.now.to_i
+        if (redis_seconds - app_seconds).abs > SKEW_TOLERANCE
+          warn("Detected clock skew between Redis and the application server. Redis time: #{redis_seconds}, Application time: #{app_seconds}. See https://github.com/bolshakov/stoplight/wiki/Clock-Skew-and-Stoplight-Reliability")
         end
       end
 
-      def query_state(light, transaction: @redis)
-        transaction.hget(states_key, light.name)
+      private def should_sample?(probability)
+        rand <= probability
       end
 
-      def normalize_state(state)
-        state || State::UNLOCKED
+      private def record_success_sha
+        @record_success_sha ||= @redis.then do |client|
+          client.script("load", Lua::RECORD_SUCCESS)
+        end
       end
 
-      # We store a list of failures happened in the  +light+ in this key
-      #
-      # @param light [Stoplight::Light]
-      # @return [String]
-      def failures_key(light)
-        key('failures', light.name)
+      private def get_metadata_sha
+        @get_metadata_sha ||= @redis.then do |client|
+          client.script("load", Lua::GET_METADATA)
+        end
       end
 
-      def notification_lock_key(light)
-        key('notification_lock', light.name)
+      private def transition_to_yellow_sha
+        @transition_to_yellow_sha ||= @redis.then do |client|
+          client.script("load", Lua::TRANSITION_TO_YELLOW)
+        end
       end
 
-      def last_notification_key(light)
-        key('last_notification', light.name)
+      private def transition_to_red_sha
+        @transition_to_red_sha ||= @redis.then do |client|
+          client.script("load", Lua::TRANSITION_TO_RED)
+        end
       end
 
-      def states_key
-        key('states')
+      private def transition_to_green_sha
+        @transition_to_green_sha ||= @redis.then do |client|
+          client.script("load", Lua::TRANSITION_TO_GREEN)
+        end
       end
 
-      def key(*pieces)
-        ([KEY_PREFIX] + pieces).join(KEY_SEPARATOR)
+      private def record_failure_sha
+        @record_failure_sha ||= @redis.then do |client|
+          client.script("load", Lua::RECORD_FAILURE)
+        end
       end
     end
   end

data/lib/stoplight/default.rb

@@ -6,24 +6,26 @@ module Stoplight
 
     DATA_STORE = DataStore::Memory.new
 
-    ERROR_HANDLER = ->(error, handler) { handler.call(error) }
-
     ERROR_NOTIFIER = ->(error) { warn error }
 
-    FALLBACK = nil
-
     FORMATTER = lambda do |light, from_color, to_color, error|
-      words = ['Switching', light.name, 'from', from_color, 'to', to_color]
-      words += ['because', error.class, error.message] if error
-      words.join(' ')
+      words = ["Switching", light.name, "from", from_color, "to", to_color]
+      words += ["because", error.class, error.message] if error
+      words.join(" ")
     end
 
     NOTIFIERS = [
-      Notifier::IO.new($stderr)
+      Notifier::FailSafe.wrap(Notifier::IO.new($stderr))
     ].freeze
 
     THRESHOLD = 3
 
-    WINDOW_SIZE = Float::INFINITY
+    WINDOW_SIZE = nil
+
+    TRACKED_ERRORS = [StandardError].freeze
+    SKIPPED_ERRORS = [].freeze
+
+    TRAFFIC_CONTROL = TrafficControl::ConsecutiveFailures.new
+    TRAFFIC_RECOVERY = TrafficRecovery::SingleSuccess.new
   end
 end

data/lib/stoplight/error.rb

@@ -2,20 +2,8 @@
 
 module Stoplight
   module Error
-    HANDLER = lambda do |error|
-      raise error if AVOID_RESCUING.any? { |klass| error.is_a?(klass) }
-    end
-
-    AVOID_RESCUING = [
-      NoMemoryError,
-      ScriptError,
-      SecurityError,
-      SignalException,
-      SystemExit,
-      SystemStackError
-    ].freeze
-
     Base = Class.new(StandardError)
+    ConfigurationError = Class.new(Base)
     IncorrectColor = Class.new(Base)
     RedLight = Class.new(Base)
   end

data/lib/stoplight/failure.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
-require 'json'
-require 'time'
+require "json"
+require "time"
 
 module Stoplight
   class Failure # rubocop:disable Style/Documentation
-    TIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%N%:z'
+    TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%N%:z"
 
     # @return [String]
     attr_reader :error_class
@@ -16,8 +16,8 @@ module Stoplight
 
     # @param error [Exception]
     # @return (see #initialize)
-    def self.from_error(error)
-      new(error.class.name, error.message, Time.now)
+    def self.from_error(error, time: Time.now)
+      new(error.class.name, error.message, time)
     end
 
     # @param json [String]
@@ -26,11 +26,11 @@ module Stoplight
     # @raise [ArgumentError]
     def self.from_json(json)
       object = JSON.parse(json)
-      error_object = object['error']
+      error_object = object["error"]
 
-      error_class = error_object['class']
-      error_message = error_object['message']
-      time = Time.parse(object['time'])
+      error_class = error_object["class"]
+      error_message = error_object["message"]
+      time = Time.at(object["time"])
 
       new(error_class, error_message, time)
     end
@@ -41,15 +41,16 @@ module Stoplight
     def initialize(error_class, error_message, time)
       @error_class = error_class
       @error_message = error_message
-      @time = time
+      @time = Time.at(time.to_i) # truncate to seconds
     end
 
     # @param other [Failure]
     # @return [Boolean]
     def ==(other)
-      error_class == other.error_class &&
+      other.is_a?(self.class) &&
+        error_class == other.error_class &&
         error_message == other.error_message &&
-        time == other.time
+        time.to_i == other.time.to_i
     end
 
     # @param options [Object, nil]
@@ -61,7 +62,7 @@ module Stoplight
             class: error_class,
             message: error_message
           },
-          time: time.strftime(TIME_FORMAT)
+          time: time.to_i
         },
         options
       )