circuitbox 1.1.0 → 2.0.0.pre4

data/lib/circuitbox/circuit_breaker.rb

@@ -1,15 +1,20 @@
+# frozen_string_literal: true
+
+require_relative 'circuit_breaker/logger_messages'
+
 class Circuitbox
   class CircuitBreaker
-    attr_accessor :service, :circuit_options, :exceptions, :partition,
-                  :logger, :circuit_store, :notifier
+    include LoggerMessages
+
+    attr_reader :service, :circuit_options, :exceptions,
+                :logger, :circuit_store, :notifier, :time_class
 
     DEFAULTS = {
-      sleep_window:     300,
+      sleep_window: 90,
       volume_threshold: 5,
-      error_threshold:  50,
-      timeout_seconds:  1,
-      time_window:      60,
-    }
+      error_threshold: 50,
+      time_window: 60
+    }.freeze
 
     #
     # Configuration options
@@ -17,242 +22,213 @@ class Circuitbox
     # `sleep_window`      - seconds to sleep the circuit
     # `volume_threshold`  - number of requests before error rate calculation occurs
     # `error_threshold`   - percentage of failed requests needed to trip circuit
-    # `timeout_seconds`   - seconds until it will timeout the request
-    # `exceptions`        - exceptions other than Timeout::Error that count as failures
+    # `exceptions`        - exceptions that count as failures
     # `time_window`       - interval of time used to calculate error_rate (in seconds) - default is 60s
     # `logger`            - Logger to use - defaults to Rails.logger if defined, otherwise STDOUT
     #
     def initialize(service, options = {})
-      @service = service
-      @circuit_options = options
-      @circuit_store   = options.fetch(:cache) { Circuitbox.circuit_store }
-      @notifier        = options.fetch(:notifier_class) { Notifier }
+      @service = service.to_s
+      @circuit_options = DEFAULTS.merge(options)
+      @circuit_store   = options.fetch(:cache) { Circuitbox.default_circuit_store }
+      @notifier = options.fetch(:notifier) { Circuitbox.default_notifier }
+
+      if @circuit_options[:timeout_seconds]
+        warn('timeout_seconds was removed in circuitbox 2.0. '\
+             'Check the upgrade guide at https://github.com/yammer/circuitbox')
+      end
 
-      @exceptions = options.fetch(:exceptions) { [] }
-      @exceptions = [Timeout::Error] if @exceptions.blank?
+      @exceptions = options.fetch(:exceptions)
+      raise ArgumentError.new('exceptions need to be an array') unless @exceptions.is_a?(Array)
 
-      @logger     = options.fetch(:logger) { defined?(Rails) ? Rails.logger : Logger.new(STDOUT) }
-      @time_class   = options.fetch(:time_class) { Time }
-      sanitize_options
+      @logger     = options.fetch(:logger) { Circuitbox.default_logger }
+      @time_class = options.fetch(:time_class) { Time }
+      @state_change_mutex = Mutex.new
+      check_sleep_window
     end
 
     def option_value(name)
-      value = circuit_options.fetch(name) { DEFAULTS.fetch(name) }
+      value = circuit_options[name]
       value.is_a?(Proc) ? value.call : value
     end
 
-    def run!(run_options = {})
-      @partition = run_options.delete(:partition) # sorry for this hack.
-
+    def run(circuitbox_exceptions: true, &block)
       if open?
-        logger.debug "[CIRCUIT] open: skipping #{service}"
-        open! unless open_flag?
-        raise Circuitbox::OpenCircuitError.new(service)
+        skipped!
+        raise Circuitbox::OpenCircuitError.new(service) if circuitbox_exceptions
       else
-        close! if was_open?
-        logger.debug "[CIRCUIT] closed: querying #{service}"
+        logger.debug(circuit_running_message)
 
         begin
-          response = if exceptions.include? Timeout::Error
-            timeout_seconds = run_options.fetch(:timeout_seconds) { option_value(:timeout_seconds) }
-            timeout (timeout_seconds) { yield }
-          else
-            yield
-          end
-
-          logger.debug "[CIRCUIT] closed: #{service} querie success"
+          response = Timer.measure(service, notifier, 'runtime', &block)
+
           success!
-        rescue *exceptions => exception
-          logger.debug "[CIRCUIT] closed: detected #{service} failure"
+        rescue *exceptions => e
+          # Other stores could raise an exception that circuitbox is asked to watch.
+          # setting to nil keeps the same behavior as the previous defination of run.
+          response = nil
           failure!
-          open! if half_open?
-          raise Circuitbox::ServiceFailureError.new(service, exception)
+          raise Circuitbox::ServiceFailureError.new(service, e) if circuitbox_exceptions
         end
       end
 
-      return response
-    end
-
-    def run(run_options = {})
-      begin
-        run!(run_options, &Proc.new)
-      rescue Circuitbox::Error
-        nil
-      end
+      response
     end
 
     def open?
-      if open_flag?
-        true
-      elsif passed_volume_threshold? && passed_rate_threshold?
-        true
-      else
-        false
-      end
+      circuit_store.key?(open_storage_key)
     end
 
     def error_rate(failures = failure_count, success = success_count)
       all_count = failures + success
-      return 0.0 unless all_count > 0
-      failure_count.to_f / all_count.to_f * 100
+      return 0.0 unless all_count.positive?
+
+      (failures / all_count.to_f) * 100
     end
 
     def failure_count
-      circuit_store.load(stat_storage_key(:failure), raw: true).to_i
+      circuit_store.load(stat_storage_key('failure'), raw: true).to_i
     end
 
     def success_count
-      circuit_store.load(stat_storage_key(:success), raw: true).to_i
+      circuit_store.load(stat_storage_key('success'), raw: true).to_i
     end
 
     def try_close_next_time
-      circuit_store.delete(storage_key(:asleep))
+      circuit_store.delete(open_storage_key)
     end
 
-  private
-    def open!
-      log_event :open
-      logger.debug "[CIRCUIT] opening #{service} circuit"
-      circuit_store.store(storage_key(:asleep), true, expires: option_value(:sleep_window))
-      half_open!
-      was_open!
-    end
+    private
 
-    ### BEGIN - all this is just here to produce a close notification
-    def close!
-      log_event :close
-      circuit_store.delete(storage_key(:was_open))
-    end
-
-    def was_open!
-      circuit_store.store(storage_key(:was_open), true)
-    end
+    def should_open?
+      failures = failure_count
+      successes = success_count
+      rate = error_rate(failures, successes)
 
-    def was_open?
-      circuit_store[storage_key(:was_open)].present?
+      passed_volume_threshold?(failures, successes) && passed_rate_threshold?(rate)
     end
-    ### END
 
-    def half_open!
-      circuit_store.store(storage_key(:half_open), true)
+    def passed_volume_threshold?(failures, successes)
+      failures + successes >= option_value(:volume_threshold)
     end
 
-    def open_flag?
-      circuit_store[storage_key(:asleep)].present?
+    def passed_rate_threshold?(rate)
+      rate >= option_value(:error_threshold)
     end
 
-    def half_open?
-      circuit_store[storage_key(:half_open)].present?
-    end
+    def half_open_failure
+      @state_change_mutex.synchronize do
+        return if open? || !half_open?
 
-    def passed_volume_threshold?
-      success_count + failure_count > option_value(:volume_threshold)
-    end
+        trip
+      end
 
-    def passed_rate_threshold?
-      read_and_log_error_rate >= option_value(:error_threshold)
+      # Running event and logger outside of the synchronize block to allow other threads
+      # that may be waiting to become unblocked
+      notify_opened
     end
 
-    def read_and_log_error_rate
-      failures = failure_count
-      success  = success_count
-      rate = error_rate(failures, success)
-      log_metrics(rate, failures, success)
-      rate
-    end
+    def open!
+      @state_change_mutex.synchronize do
+        return if open?
 
-    def success!
-      log_event :success
-      circuit_store.delete(storage_key(:half_open))
-    end
+        trip
+      end
 
-    def failure!
-      log_event :failure
+      # Running event and logger outside of the synchronize block to allow other threads
+      # that may be waiting to become unblocked
+      notify_opened
     end
 
-    # Store success/failure/open/close data in memcache
-    def log_event(event)
-      notifier.new(service,partition).notify(event)
-      log_event_to_process(event)
+    def notify_opened
+      notify_event('open')
+      logger.debug(circuit_opened_message)
     end
 
-    def log_metrics(error_rate, failures, successes)
-      n = notifier.new(service,partition)
-      n.metric_gauge(:error_rate, error_rate)
-      n.metric_gauge(:failure_count, failures)
-      n.metric_gauge(:success_count, successes)
+    def trip
+      circuit_store.store(open_storage_key, true, expires: option_value(:sleep_window))
+      circuit_store.store(half_open_storage_key, true)
     end
 
-    def sanitize_options
-      sleep_window = option_value(:sleep_window)
-      time_window  = option_value(:time_window)
-      if sleep_window < time_window
-        notifier.new(service,partition).notify_warning("sleep_window:#{sleep_window} is shorter than time_window:#{time_window}, the error_rate could not be reset properly after a sleep. sleep_window as been set to equal time_window.")
-        @circuit_options[:sleep_window] = option_value(:time_window)
+    def close!
+      @state_change_mutex.synchronize do
+        # If the circuit is not open, the half_open key will be deleted from the store
+        # if half_open exists the deleted value is returned and allows us to continue
+        # if half_open doesn't exist nil is returned, causing us to return early
+        return unless !open? && circuit_store.delete(half_open_storage_key)
       end
+
+      # Running event outside of the synchronize block to allow other threads
+      # that may be waiting to become unblocked
+      notify_event('close')
+      logger.debug(circuit_closed_message)
     end
 
-    # When there is a successful response within a count interval, clear the failures.
-    def clear_failures!
-      circuit_store.store(stat_storage_key(:failure), 0, raw: true)
+    def half_open?
+      circuit_store.key?(half_open_storage_key)
     end
 
-    # Logs to process memory.
-    def log_event_to_process(event)
-      key = stat_storage_key(event)
-      if circuit_store.load(key, raw: true)
-        circuit_store.increment(key)
-      else
-        # yes we want a string here, as the underlying stores impement this as a native type.
-        circuit_store.store(key, "1", raw: true)
-      end
+    def success!
+      increment_and_notify_event('success')
+      logger.debug(circuit_success_message)
+
+      close! if half_open?
     end
 
-    # Logs to Memcache.
-    def log_event_to_stat_store(key)
-      if stat_store.read(key, raw: true)
-        stat_store.increment(key)
-      else
-        stat_store.store(key, 1)
+    def failure!
+      increment_and_notify_event('failure')
+      logger.debug(circuit_failure_message)
+
+      if half_open?
+        half_open_failure
+      elsif should_open?
+        open!
       end
     end
 
-    # For returning stale responses when the circuit is open
-    def response_key(args)
-      Digest::SHA1.hexdigest(storage_key(:cache, args.inspect.to_s))
+    def skipped!
+      notify_event('skipped')
+      logger.debug(circuit_skipped_message)
     end
 
-    def stat_storage_key(event, options = {})
-      storage_key(:stats, align_time_on_minute, event, options)
+    # Send event notification to notifier
+    def notify_event(event)
+      notifier.notify(service, event)
     end
 
-
-    # return time representation in seconds
-    def align_time_on_minute(time=nil)
-      time      ||= @time_class.now.to_i
-      time_window = option_value(:time_window)
-      time - ( time % time_window ) # remove rest of integer division
+    # Increment stat store and send notification
+    def increment_and_notify_event(event)
+      circuit_store.increment(stat_storage_key(event), 1, expires: (option_value(:time_window) * 2))
+      notify_event(event)
     end
 
-    def storage_key(*args)
-      options = args.extract_options!
+    def check_sleep_window
+      sleep_window = option_value(:sleep_window)
+      time_window  = option_value(:time_window)
+      return unless sleep_window < time_window
 
-      key = if options[:without_partition]
-        "circuits:#{service}:#{args.join(":")}"
-      else
-        "circuits:#{service}:#{partition}:#{args.join(":")}"
-      end
+      warning_message = "sleep_window: #{sleep_window} is shorter than time_window: #{time_window}, "\
+                        "the error_rate would not be reset after a sleep."
+      notifier.notify_warning(service, warning_message)
+      warn("Circuit: #{service}, Warning: #{warning_message}")
+    end
 
-      return key
+    def stat_storage_key(event)
+      "circuits:#{service}:stats:#{align_time_to_window}:#{event}"
     end
 
-    def timeout(timeout_seconds, &block)
-      Timeout::timeout(timeout_seconds) { block.call }
+    # return time representation in seconds
+    def align_time_to_window
+      time = time_class.now.to_i
+      time_window = option_value(:time_window)
+      time - (time % time_window) # remove rest of integer division
     end
 
-    def self.reset
-      Circuitbox.reset
+    def open_storage_key
+      @open_storage_key ||= "circuits:#{service}:open"
     end
 
+    def half_open_storage_key
+      @half_open_storage_key ||= "circuits:#{service}:half_open"
+    end
   end
 end

data/lib/circuitbox/circuit_breaker/logger_messages.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class Circuitbox
+  class CircuitBreaker
+    module LoggerMessages
+      def circuit_skipped_message
+        @circuit_skipped_message ||= "[CIRCUIT] #{service}: skipped"
+      end
+
+      def circuit_running_message
+        @circuit_running_message ||= "[CIRCUIT] #{service}: running"
+      end
+
+      def circuit_success_message
+        @circuit_success_message ||= "[CIRCUIT] #{service}: success"
+      end
+
+      def circuit_failure_message
+        @circuit_failure_message ||= "[CIRCUIT] #{service}: failure"
+      end
+
+      def circuit_opened_message
+        @circuit_opened_message ||= "[CIRCUIT] #{service}: opened"
+      end
+
+      def circuit_closed_message
+        @circuit_closed_message ||= "[CIRCUIT] #{service}: closed"
+      end
+    end
+  end
+end

data/lib/circuitbox/configuration.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require_relative 'memory_store'
+require_relative 'timer'
+require_relative 'notifier/active_support'
+require_relative 'notifier/null'
+
+class Circuitbox
+  module Configuration
+    attr_writer :default_circuit_store,
+                :default_notifier,
+                :default_timer,
+                :default_logger
+
+    def configure
+      yield self
+      clear_cached_circuits!
+      nil
+    end
+
+    def default_circuit_store
+      @default_circuit_store ||= MemoryStore.new
+    end
+
+    def default_notifier
+      @default_notifier ||= if defined?(ActiveSupport::Notifications)
+                              Notifier::ActiveSupport.new
+                            else
+                              Notifier::Null.new
+                            end
+    end
+
+    def default_logger
+      @default_logger ||= if defined?(Rails)
+                            Rails.logger
+                          else
+                            Logger.new($stdout)
+                          end
+    end
+
+    private
+
+    def cached_circuits
+      @cached_circuits ||= {}
+    end
+
+    def clear_cached_circuits!
+      @cached_circuits = {}
+    end
+  end
+end