all_my_circuits 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 37ff5fd01761190c1ac3b39e9716cac0b242e244
+  data.tar.gz: 3978e9a09c6bc75e86227359036f783eb33616b6
+SHA512:
+  metadata.gz: ea62b8b682a65ecd8b1e03f02820040db8023b0de594ae0a1153988674514a665c504284d568fce3950fab060917892e55edc1ec217fe42178b549995f63eb38
+  data.tar.gz: a79c6a02bcad7e76310b75899b22b0b926f1d9a6e8b750d506d29eed1eb19354de9dbccbfbb0bacf2353d7f71053b12453ac7c77daadb1b725b930942e463a66

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in all_my_circuits.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,23 @@
+Copyright (c) 2015, Remind101, Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,77 @@
+# AllMyCircuits
+
+[![Circle CI](https://circleci.com/gh/remind101/all_my_circuits.svg?style=svg&circle-token=3c0f7294a61e095f789f369f96bdd266ddd80258)](https://circleci.com/gh/remind101/all_my_circuits)
+
+![funny image goes here](https://raw.githubusercontent.com/remind101/all_my_circuits/master/all_my_circuits.jpg?token=AAc0YcX8xOhT0o4_Ko-IxKEEQk2PTUJYks5VaR0ywA%3D%3D)
+
+AllMyCircuits is intended to be a threadsafe-ish circuit breaker implementation for Ruby.
+See [Martin Fowler's article on the Circuit Breaker pattern for more info](http://martinfowler.com/bliki/CircuitBreaker.html).
+
+# Usage
+
+    class MyService
+      include Singleton
+
+      def initialize
+        @circuit_breaker = AllMyCircuits::Breaker.new(
+          name: "my_service",
+          watch_errors: [Timeout::Error],               # defaults to AllMyCircuits::Breaker.net_errors,
+                                                        #   which includes typical net/http and socket errors
+          sleep_seconds: 10,                            # leave circuit open for 10 seconds, than try to call the service again
+          strategy: AllMyCircuits::Strategies::PercentageOverWindowStrategy.new(
+            requests_window: 100,                       # number of requests to calculate the average failure rate for
+            failure_rate_percent_threshold: 25          # open circuit if 25% or more requests within 100-request window fail
+                                                        #   must trip open again if the first request fails
+          }
+        )
+      end
+
+      def run
+        begin
+          @breaker.run do
+            Timeout.timeout(1.0) { my_risky_call }
+          end
+        rescue AllMyCircuits::BreakerOpen => e
+          # log me somewhere
+        rescue
+          # uh-oh, risky call failed once
+        end
+      end
+    end
+
+# Testing
+
+So, what have we got:
+
+  * Time-sensitive code: ...check
+  * Concurrent code:     ...check
+
+Dude, that's a real headache for someone who's not confortable enough with concurrent code.
+I haven't figured any awesome way of automated testing in this case.
+So, in the [script](https://github.com/remind101/all_my_circuits/tree/master/script) folder, there are:
+
+  * [fake_service.rb](https://github.com/remind101/all_my_circuits/blob/master/script/fake_service.rb)
+  * [graphing_stress_test.rb](https://github.com/remind101/all_my_circuits/blob/master/script/graphing_stress_test.rb)
+
+## fake_service.rb
+
+the Fake Service has 3 modes of operation: `up` (default), `die` and `slowdown`.
+
+  * `up` (default) - http://localhost:8081/up - normal mode of operation, latency up to 50ms
+  * `die` - http://localhost:8081/die - exceptions are raised left and right, slight delay in response
+  * `slowdown` - http://localhost:8081/slowdown - successful responses with a significant delay.
+
+## Graphing Stress Test
+
+runs `WORKERS` number of workers which continuously hit http://localhost:8081. Graphs are served at http://localhost:8080.
+This app allows to catch incorrect circuit breaker behavior visually.
+
+# Logging
+
+Logger can be configured by setting `AllMyCircuits.logger`.
+Default log device is STDERR, and default level is ERROR (can be overridden with `ALL_MY_CIRCUITS_LOG_LEVEL` environment variable).
+
+# TODO
+
+  * global controls through redis?
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.test_files = FileList["test/**/test_*.rb"]
+  t.verbose = true
+end
+
+task default: [:test]

data/all_my_circuits.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'all_my_circuits/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "all_my_circuits"
+  spec.version       = AllMyCircuits::VERSION
+  spec.authors       = ["Vlad Yarotsky"]
+  spec.email         = ["vlad@remind101.com"]
+
+  spec.summary       = %q{Circuit Breaker library with support for rolling-window absolute/percentage thresholds}
+  spec.description   = %q{}
+  spec.homepage      = "https://github.com/remind101/all_my_circuits"
+  spec.license       = "BSD"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "concurrent-ruby", "~> 0.8"
+
+  spec.add_development_dependency "bundler", "~> 1.9"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.6"
+  spec.add_development_dependency "puma", "~> 2.11"
+end

data/circle.yml

@@ -0,0 +1,12 @@
+machine:
+  ruby:
+    version:
+      2.2.2
+
+dependencies:
+  pre:
+    - gem install bundler -v "~> 1.9"
+
+test:
+  override:
+    - rake test

data/lib/all_my_circuits/breaker.rb

@@ -0,0 +1,264 @@
+require "concurrent/atomic"
+require "thread"
+
+module AllMyCircuits
+
+  class Breaker
+    include Logging
+
+    attr_reader :name
+
+    # Public: exceptions typically thrown when using Net::HTTP
+    #
+    def self.net_errors
+      require "timeout"
+      require "net/http"
+
+      [Timeout::Error, Errno::ECONNRESET, Errno::ECONNREFUSED, EOFError,
+       Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::HTTPFatalError, Net::HTTPServerError]
+    end
+
+    # Public: Initializes circuit breaker instance.
+    #
+    # Options
+    #
+    #   name          - name of the call wrapped into circuit breaker (e.g. "That Unstable Service").
+    #   watch_errors  - exceptions to count as failures. Other exceptions will simply get re-raised
+    #                   (default: AllMyCircuits::Breaker.net_errors).
+    #   sleep_seconds - number of seconds the circuit stays open before attempting to close.
+    #   strategy      - an AllMyCircuits::Strategies::AbstractStrategy-compliant object that controls
+    #                   when the circuit should be tripped open.
+    #                   Built-in strategies:
+    #                     AllMyCircuits::Strategies::PercentageOverWindowStrategy,
+    #                     AllMyCircuits::Strategies::NumberOverWindowStrategy.
+    #   notifier      - (optional) AllMyCircuits::Notifiers::AbstractNotifier-compliant object that
+    #                   is called whenever circuit breaker state (open, closed) changes.
+    #                   Built-in notifiers:
+    #                     AllMyCircuits::Notifiers::NullNotifier.
+    #
+    # Examples
+    #
+    #   AllMyCircuits::Breaker.new(
+    #     name: "My Unstable Service",
+    #     sleep_seconds: 5,
+    #     strategy: AllMyCircuits::Strategies::PercentageOverWindowStrategy.new(
+    #       requests_window: 20,                 # number of requests in the window to calculate failure rate for
+    #       failure_rate_percent_threshold: 25   # how many failures can occur within the window, in percent,
+    #     )                                      #   before the circuit opens
+    #   )
+    #
+    #   AllMyCircuits::Breaker.new(
+    #     name: "Another Unstable Service",
+    #     sleep_seconds: 5,
+    #     strategy: AllMyCircuits::Strategies::NumberOverWindowStrategy.new(
+    #       requests_window: 20,
+    #       failures_threshold: 25         # how many failures can occur within the window before the circuit opens
+    #     )
+    #   )
+    #
+    def initialize(name:,
+                   watch_errors: Breaker.net_errors,
+                   sleep_seconds:,
+                   strategy:,
+                   notifier: Notifiers::NullNotifier.new,
+                   clock: Clock)
+
+      @name = String(name).dup.freeze
+      @watch_errors = Array(watch_errors).dup
+      @sleep_seconds = Integer(sleep_seconds)
+
+      @strategy = strategy
+      @notifier = notifier
+
+      @state_lock = Mutex.new
+      @request_number = Concurrent::Atomic.new(0)
+      @last_open_or_probed = nil
+      @opened_at_request_number = 0
+      @clock = clock
+    end
+
+    # Public: executes supplied block of code and monitors failures.
+    # Once the number of failures reaches a certain threshold, the block is bypassed
+    # for a certain period of time.
+    #
+    # Consider the following examples of calls through circuit breaker (let it be 1 call per second,
+    # and let the circuit breaker be configured as in the example below):
+    #
+    # Legend
+    #
+    #   S - successful request
+    #   F - failed request
+    #   O - skipped request (circuit open)
+    #   | - open circuit interval end
+    #
+    #
+    #   1) S S F F S S F F S F O O O O O|S S S S S
+    #
+    #   Here among the first 10 requests (window), 5 failures occur (50%), the circuit is tripped open
+    #   for 5 seconds, and a few requests are skipped. Then, after 5 seconds, a request is issued to
+    #   see whether everything is back to normal, and the circuit is then closed again.
+    #
+    #   2) S S F F S S F F S F O O O O O|F O O O O O|S S S S S
+    #
+    #   Same situation, 10 requests, 5 failed, circuit is tripped open for 5 seconds. Then we
+    #   check that service is back to normal, and it is not. The circuit is open again.
+    #   After another 5 seconds we check again and close the circuit.
+    #
+    # Returns nothing.
+    # Raises AllMyCircuit::BreakerOpen with the name of the service when the circuit is open.
+    # Raises whatever has been raised by the supplied block.
+    #
+    # This call is thread-safe sans the supplied block.
+    #
+    # Examples
+    #
+    #   @cb = AllMyCircuits::Breaker.new(
+    #     name: "that bad service",
+    #     sleep_seconds: 5,
+    #     strategy: AllMyCircuits::Strategies::PercentageOverWindowStrategy.new(
+    #       requests_window: 10,
+    #       failure_rate_percent_threshold: 50
+    #     )
+    #   )
+    #
+    #   @client = MyBadServiceClient.new(timeout: 2)
+    #
+    #   begin
+    #     @cb.run do
+    #       @client.make_expensive_unreliable_http_call
+    #     end
+    #   rescue AllMyCircuits::BreakerOpen => e
+    #     []
+    #   rescue MyBadServiceClient::Error => e
+    #     MyLog << "an error has occured in call to my bad service"
+    #     []
+    #   end
+    #
+    def run
+      unless allow_request?
+        debug "declining request, circuit is open", name
+        raise BreakerOpen, @name
+      end
+
+      current_request_number = generate_request_number
+      begin
+        result = yield
+        success(current_request_number)
+        result
+      rescue *@watch_errors
+        error(current_request_number)
+        raise
+      end
+    end
+
+    private
+
+    # Internal: checks whether the circuit is closed, or if it is time
+    # to try one request to see if things are back to normal.
+    #
+    def allow_request?
+      @state_lock.synchronize do
+        !open? || allow_probe_request?
+      end
+    end
+
+    def generate_request_number
+      current_request_number = @request_number.update { |v| v + 1 }
+
+      debug "new request", name, current_request_number
+      current_request_number
+    end
+
+    # Internal: markes request as successful. Closes the circuit if necessary.
+    #
+    # Arguments
+    #   current_request_number - the number assigned to the request by the circuit breaker
+    #                            before it was sent.
+    #
+    def success(current_request_number)
+      will_notify_notifier_closed = false
+
+      @state_lock.synchronize do
+        if open?
+          # This ensures that we are not closing the circuit prematurely
+          # due to a response for an old request coming in.
+          if current_request_number > @opened_at_request_number
+            info "closing circuit", name, current_request_number
+            close!
+            will_notify_notifier_closed = true
+          else
+            debug "ignoring late success response", name, current_request_number
+          end
+        end
+        debug "request succeeded", name, current_request_number
+        @strategy.success
+      end
+
+      # We don't want to be doing this while holding the lock
+      if will_notify_notifier_closed
+        @notifier.closed
+      end
+    end
+
+    # Internal: marks request as failed. Opens the circuit if necessary.
+    #
+    def error(current_request_number)
+      will_notify_notifier_opened = false
+
+      @state_lock.synchronize do
+        if open?
+          debug "ignoring late error response (circuit is open)", name, current_request_number
+          return
+        end
+
+        debug "request failed. #{@strategy.inspect}", name, current_request_number
+        @strategy.error
+
+        if @strategy.should_open?
+          info "opening circuit", name, current_request_number
+          open!
+          will_notify_notifier_opened = true
+        end
+      end
+
+      # We don't want to be doing this while holding the lock
+      if will_notify_notifier_opened
+        @notifier.opened
+      end
+    end
+
+    def open?
+      @opened_at_request_number > 0
+    end
+
+    def allow_probe_request?
+      if open? && @clock.timestamp >= (@last_open_or_probed + @sleep_seconds)
+        debug "allowing probe request", name
+        # makes sure that we allow only one probe request by extending sleep interval
+        # and leaving the circuit open until closed by the success callback.
+        @last_open_or_probed = @clock.timestamp
+        return true
+      end
+      false
+    end
+
+    def open!
+      @last_open_or_probed = @clock.timestamp
+      # The most recent request encountered so far (may not be the current request in concurrent situation).
+      # This is necessary to prevent successful response to old request from opening the circuit prematurely.
+      # Imagine concurrent situation ("|" for request start, ">" for request end):
+      #   1|-----------> success
+      #     2|----> error, open circuit breaker
+      # In this case request 1) should not close the circuit.
+      @opened_at_request_number = @request_number.value
+      @strategy.opened
+    end
+
+    def close!
+      @last_open_or_probed = 0
+      @opened_at_request_number = 0
+      @strategy.closed
+    end
+  end
+
+end

data/lib/all_my_circuits/clock.rb

@@ -0,0 +1,9 @@
+module AllMyCircuits
+
+  class Clock
+    def self.timestamp
+      Time.now
+    end
+  end
+
+end

data/lib/all_my_circuits/exceptions.rb

@@ -0,0 +1,6 @@
+module AllMyCircuits
+
+  class Error < StandardError; end
+  class BreakerOpen < Error; end
+
+end

data/lib/all_my_circuits/logging.rb

@@ -0,0 +1,23 @@
+module AllMyCircuits
+
+  module Logging
+    def debug(message, breaker_name, request_number = nil)
+      AllMyCircuits.logger.debug(format_line(message, breaker_name, request_number))
+    end
+
+    def info(message, breaker_name, request_number = nil)
+      AllMyCircuits.logger.info(format_line(message, breaker_name, request_number))
+    end
+
+    private
+
+    def format_line(message, breaker_name, request_number)
+      if request_number.nil?
+        "[%s] %s" % [breaker_name, message]
+      else
+        "[%s] req. #%s: %s" % [breaker_name, request_number, message]
+      end
+    end
+  end
+
+end

data/lib/all_my_circuits/notifiers/abstract_notifier.rb

@@ -0,0 +1,27 @@
+module AllMyCircuits
+  module Notifiers
+
+    # Public: notifies some service about change of breaker's state.
+    # For example, one could send a Librato metric whenever a circuit
+    # breaker is opened or closed.
+    #
+    class AbstractNotifier
+      def initialize(breaker_name, **kwargs)
+        @breaker_name = breaker_name
+      end
+
+      # Public: called once the circuit is tripped open.
+      #
+      def opened
+        raise NotImplementedError
+      end
+
+      # Public: called once the circuit is closed.
+      #
+      def closed
+        raise NotImplementedError
+      end
+    end
+
+  end
+end

data/lib/all_my_circuits/notifiers/null_notifier.rb

@@ -0,0 +1,18 @@
+module AllMyCircuits
+  module Notifiers
+
+    # Public: no-op implementation of AbstractNotifier.
+    #
+    class NullNotifier < AbstractNotifier
+      def initialize(*args, **kwargs)
+      end
+
+      def opened
+      end
+
+      def closed
+      end
+    end
+
+  end
+end

data/lib/all_my_circuits/notifiers.rb

@@ -0,0 +1,8 @@
+module AllMyCircuits
+
+  module Notifiers
+    autoload :AbstractNotifier, "all_my_circuits/notifiers/abstract_notifier"
+    autoload :NullNotifier,     "all_my_circuits/notifiers/null_notifier"
+  end
+
+end

data/lib/all_my_circuits/null_breaker.rb

@@ -0,0 +1,23 @@
+module AllMyCircuits
+
+  # Public: no-op circuit breaker implementation, useful for testing
+  #
+  class NullBreaker
+    attr_reader :name
+    attr_accessor :closed
+
+    def initialize(name:, closed:)
+      @name = name
+      @closed = closed
+    end
+
+    def run
+      if @closed
+        yield
+      else
+        raise AllMyCircuits::BreakerOpen, @name
+      end
+    end
+  end
+
+end

data/lib/all_my_circuits/strategies/abstract_strategy.rb

@@ -0,0 +1,40 @@
+module AllMyCircuits
+  module Strategies
+
+    # Public: determines whether the circuit breaker should be tripped open
+    # upon another error within the supplied block.
+    #
+    # See AllMyCircuits::Strategies::NumberOverWindowStrategy,
+    # AllMyCircuits::Strategies::PercentageOverWindowStrategy for examples.
+    #
+    class AbstractStrategy
+      # Public: called whenever a request has ran successfully through circuit breaker.
+      #
+      def success
+      end
+
+      # Public: called whenever a request has failed within circuit breaker.
+      #
+      def error
+      end
+
+      # Public: called whenever circuit is tripped open.
+      #
+      def opened
+      end
+
+      # Public: called whenever circuit is closed.
+      #
+      def closed
+      end
+
+      # Public: called after each error within circuit breaker to determine
+      # whether it should be tripped open.
+      #
+      def should_open?
+        raise NotImplementedError
+      end
+    end
+
+  end
+end

data/lib/all_my_circuits/strategies/abstract_window_strategy/window.rb

@@ -0,0 +1,45 @@
+module AllMyCircuits
+  module Strategies
+    class AbstractWindowStrategy
+
+      class Window
+        def initialize(number_of_events)
+          number_of_events = Integer(number_of_events)
+          unless number_of_events > 0
+            raise ArgumentError, "window size must be a natural number"
+          end
+          @number_of_events = number_of_events
+          reset!
+        end
+
+        def reset!
+          @window = []
+          @counters = Hash.new { |h, k| h[k] = 0 }
+        end
+
+        def <<(event)
+          if full?
+            event_to_decrement = @window.shift
+            @counters[event_to_decrement] -= 1
+          end
+          @window.push(event)
+          @counters[event] += 1
+          self
+        end
+
+        def count(event = nil)
+          event.nil? ? @window.length : @counters[event]
+        end
+
+        def full?
+          @window.length == @number_of_events
+        end
+
+        def inspect
+          "#<%s:0x%x size: %d, counts: %s, full: %s" % [self.class.name, object_id, @number_of_events, @counters, full?]
+        end
+      end
+
+    end
+  end
+end

data/lib/all_my_circuits/strategies/abstract_window_strategy.rb

@@ -0,0 +1,33 @@
+module AllMyCircuits
+  module Strategies
+
+    class AbstractWindowStrategy < AbstractStrategy
+      autoload :Window, "all_my_circuits/strategies/abstract_window_strategy/window"
+
+      def initialize(requests_window:)
+        @requests_window = requests_window
+        @window = Window.new(@requests_window)
+      end
+
+      def success
+        @window << :succeeded
+      end
+
+      def error
+        @window << :failed
+      end
+
+      def opened
+      end
+
+      def closed
+        @window.reset!
+      end
+
+      def should_open?
+        raise NotImplementedError
+      end
+    end
+
+  end
+end

data/lib/all_my_circuits/strategies/number_over_window_strategy.rb

@@ -0,0 +1,32 @@
+module AllMyCircuits
+  module Strategies
+
+    # Public: opens the circuit whenever failures threshold is reached
+    # within the window. Threshold is represented by absolute number of
+    # failures within the window.
+    #
+    class NumberOverWindowStrategy < AbstractWindowStrategy
+
+      # Public: initializes a new instance.
+      #
+      # Options
+      #
+      #   requests_window    - number of consecutive requests tracked by the window.
+      #   failures_threshold - number of failures within the window after which
+      #                        the circuit is tripped open.
+      #
+      def initialize(failures_threshold:, **kwargs)
+        @failures_threshold = failures_threshold
+        super(**kwargs)
+      end
+
+      def should_open?
+        return unless @window.full?
+
+        failures = @window.count(:failed)
+        failures >= @failures_threshold
+      end
+    end
+
+  end
+end

data/lib/all_my_circuits/strategies/percentage_over_window_strategy.rb

@@ -0,0 +1,32 @@
+module AllMyCircuits
+  module Strategies
+
+    # Public: opens the circuit whenever failures threshold is reached
+    # within the window. Threshold is represented by a percentage of
+    # failures within the window.
+    #
+    class PercentageOverWindowStrategy < AbstractWindowStrategy
+
+      # Public: initializes a new instance.
+      #
+      # Options
+      #
+      #   requests_window                - number of consecutive requests tracked by the window.
+      #   failure_rate_percent_threshold - percent rate of failures within the window after which
+      #                                    the circuit is tripped open.
+      #
+      def initialize(failure_rate_percent_threshold:, **kwargs)
+        @failure_rate_percent_threshold = failure_rate_percent_threshold
+        super(**kwargs)
+      end
+
+      def should_open?
+        return false unless @window.full?
+
+        failure_rate_percent = ((@window.count(:failed).to_f / @window.count) * 100).ceil
+        failure_rate_percent >= @failure_rate_percent_threshold
+      end
+    end
+
+  end
+end

data/lib/all_my_circuits/strategies.rb

@@ -0,0 +1,10 @@
+module AllMyCircuits
+
+  module Strategies
+    autoload :AbstractStrategy,             "all_my_circuits/strategies/abstract_strategy"
+    autoload :AbstractWindowStrategy,       "all_my_circuits/strategies/abstract_window_strategy"
+    autoload :PercentageOverWindowStrategy, "all_my_circuits/strategies/percentage_over_window_strategy"
+    autoload :NumberOverWindowStrategy,     "all_my_circuits/strategies/number_over_window_strategy"
+  end
+
+end

data/lib/all_my_circuits/version.rb

@@ -0,0 +1,3 @@
+module AllMyCircuits
+  VERSION = "0.1.0"
+end

data/lib/all_my_circuits.rb

@@ -0,0 +1,20 @@
+require "all_my_circuits/version"
+require "logger"
+
+module AllMyCircuits
+  require "all_my_circuits/exceptions"
+
+  autoload :Breaker,     "all_my_circuits/breaker"
+  autoload :Clock,       "all_my_circuits/clock"
+  autoload :Logging,     "all_my_circuits/logging"
+  autoload :Notifiers,   "all_my_circuits/notifiers"
+  autoload :NullBreaker, "all_my_circuits/null_breaker"
+  autoload :Strategies,  "all_my_circuits/strategies"
+  autoload :VERSION,     "all_my_circuits/version"
+
+  class << self
+    attr_accessor :logger
+  end
+  @logger = Logger.new(STDERR)
+  @logger.level = Integer(ENV["ALL_MY_CIRCUITS_LOG_LEVEL"]) rescue Logger::ERROR
+end

data/script/fake_service.rb

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+
+require "rack"
+require "rack/handler/puma"
+require "rack/builder"
+
+class FakeService
+  attr_accessor :state
+
+  def initialize
+    @random = Random.new
+    @state = :up
+  end
+
+  def call(env)
+    case @state
+    when :up
+      sleep @random.rand(0.100)
+      [200, { "Content-Type" => "text/html" }, ["up"]]
+    when :die
+      sleep @random.rand(1.000)
+      [500, { "Content-Type" => "text/html" }, ["down"]]
+    when :slowdown
+      sleep @random.rand(5.000)
+      [200, { "Content-Type" => "text/html" }, ["slooow"]]
+    else
+      fail "what is that again?"
+    end
+  end
+end
+
+$service = FakeService.new
+
+def set_state_action(state)
+  lambda { |_| $service.state = state; [200, { "Content-Type" => "text/html" }, ["OK"]] }
+end
+
+app = Rack::Builder.new do
+  map("/" ) { run $service }
+  map("/die") { run set_state_action(:die) }
+  map("/slowdown") { run set_state_action(:slowdown) }
+  map("/up") { run set_state_action(:up) }
+end
+
+server = ::Rack::Handler::Puma
+server.run app, Port: 8081

data/script/graphing_server.rb

@@ -0,0 +1,126 @@
+require "rack"
+require "rack/handler/puma"
+require "json"
+
+class Stream
+  def initialize(data_queue)
+    @data_queue = data_queue
+  end
+
+  def each
+    batch_proto = { succeeded: 0, failed: 0, skipped: 0 }.freeze
+    next_batch = batch_proto.dup
+    loop do
+      data_batch = next_batch
+      next_batch = batch_proto.dup
+      frame_end = Time.now + 1
+      begin
+        while r = @data_queue.pop(true)
+          if r[:finished] <= frame_end
+            data_batch[r[:status]] += 1
+          else
+            next_batch[r[:status]] += 1
+            break
+          end
+        end
+      rescue ThreadError # queue exhausted
+        sleep 0.0001
+        retry
+      end
+      data_batch[:currentTime] = frame_end.to_f * 1000
+      serialized = "data: %s\n\n" % data_batch.to_json
+      yield serialized
+    end
+  end
+end
+
+class GraphingServer
+  def self.start(data_queue)
+    server = ::Rack::Handler::Puma
+    server.run Rack::Chunked.new(new(Stream.new(data_queue)))
+  rescue => e
+    warn "graphing server wtf #{e.inspect} #{e.backtrace}"
+    raise
+  end
+
+  PAGE = <<-PAGE
+    <!DOCTYPE html>
+    <html>
+      <head>
+        <title>AllMyCircuits Stress Test Graph</title>
+        <script src="//cdnjs.cloudflare.com/ajax/libs/jquery/2.1.4/jquery.min.js" type="text/javascript"></script>
+        <script src="https://rawgit.com/joewalnes/smoothie/master/smoothie.js" type="text/javascript"></script>
+        <style type="text/css">
+          html{
+            height: 100%;
+          }
+          body {
+            min-height: 100%;
+          }
+        </style>
+      </head>
+      <body>
+        <canvas id="workersSuccess" width="1000" height="300"></canvas>
+        <canvas id="workersFailure" width="1000" height="300"></canvas>
+        <canvas id="workersSkip" width="1000" height="300"></canvas>
+        <script type="text/javascript">
+          var chartHeight = window.innerHeight / 3 - 20;
+          var chartWidth = window.innerWidth - 20;
+
+          var charts = {
+            succeeded: {
+              elementId: "workersSuccess",
+              color: "rgb(0, 255, 0)"
+            },
+            failed: {
+              elementId: "workersFailure",
+              color: "rgb(255, 0, 0)"
+            },
+            skipped: {
+              elementId: "workersSkip",
+              color: "rgb(255, 255, 255)"
+            }
+          };
+
+          for (var chart in charts) {
+            var chartConfig = charts[chart];
+            var el = document.getElementById(chartConfig.elementId);
+            el.height = chartHeight;
+            el.width = chartWidth;
+
+            var chart = new SmoothieChart({ interpolation: 'step', minValue: 0, maxValueScale: 1.2 });
+            chart.streamTo(el, 1000);
+
+            chartConfig.series = new TimeSeries();
+            chart.addTimeSeries(chartConfig.series, { strokeStyle: chartConfig.color });
+          }
+
+          var source = new EventSource("/data");
+          source.onmessage = function(event) {
+            var points = JSON.parse(event.data);
+            var currentTime = points.currentTime;
+            charts.succeeded.series.append(currentTime, points.succeeded);
+            charts.failed.series.append(currentTime, points.failed);
+            charts.skipped.series.append(currentTime, points.skipped);
+          };
+        </script>
+      </body>
+    </html>
+  PAGE
+
+  def initialize(stream)
+    @stream = stream
+  end
+
+  def call(env)
+    req = Rack::Request.new(env)
+    case req.path
+    when "/"
+      [200, { "Content-Type" => "text/html" }, [PAGE]]
+    when "/data"
+      [200, { "Content-Type" => "text/event-stream", "Connection" => "keepalive", "Cache-Control" => "no-cache, no-store" }, @stream]
+    else
+      [404, { "Content-Type" => "text/html" }, []]
+    end
+  end
+end

data/script/graphing_stress_test.rb

@@ -0,0 +1,80 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.expand_path("../../lib", __FILE__)
+require "all_my_circuits"
+require_relative "./graphing_server"
+require "thread"
+require "net/http"
+require "uri"
+require "timeout"
+require "securerandom"
+
+WORKERS = Integer(ENV["WORKERS"]) rescue 25
+
+def setup
+  @responses_queue = Queue.new
+  @breaker = AllMyCircuits::Breaker.new(
+    name: "test service circuit breaker",
+    sleep_seconds: 5,
+    strategy: AllMyCircuits::Strategies::PercentageOverWindowStrategy.new(
+      requests_window: 40,
+      failure_rate_percent_threshold: 25
+    )
+  )
+  @datapoints_queue = Queue.new
+  @commands_queue = Queue.new
+end
+
+def run
+  setup
+  workers = run_workers
+  graphing_server = run_graphing_server
+
+  workers.each(&:join)
+  graphing_server.join
+end
+
+def run_workers
+  WORKERS.times.map do
+    Thread.new(@responses_queue, @breaker) do |responses, breaker|
+      loop do
+        begin
+          t1 = Time.now
+          @breaker.run do
+            uri = URI("http://localhost:8081")
+            http = Net::HTTP.new(uri.host, uri.port)
+            http.open_timeout = 2
+            http.read_timeout = 2
+            http.start do |h|
+              request = Net::HTTP::Get.new(uri)
+              response = h.request(request)
+              response.value
+              log "success"
+              responses.push(status: :succeeded, started: t1, finished: Time.now)
+            end
+          end
+        rescue AllMyCircuits::BreakerOpen
+          log "breaker open"
+          responses.push(status: :skipped, started: t1, finished: Time.now)
+        rescue
+          log "failure #{$!.inspect}: #{$!.backtrace.first(2).join(", ")}"
+          responses.push(status: :failed, started: t1, finished: Time.now)
+        end
+      end
+    end
+  end
+end
+
+def run_graphing_server
+  Thread.new(@responses_queue) do |responses_queue|
+    GraphingServer.start(responses_queue)
+  end
+end
+
+def log(msg)
+  @mtx ||= Mutex.new
+  timestamp = "%10.6f" % Time.now.to_f
+  @mtx.synchronize { puts "[#{Thread.current.object_id}] #{timestamp}: #{msg}" if ENV["DEBUG"] }
+end
+
+run

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification
+name: all_my_circuits
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Vlad Yarotsky
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-05-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.6'
+- !ruby/object:Gem::Dependency
+  name: puma
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.11'
+description: ''
+email:
+- vlad@remind101.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- all_my_circuits.gemspec
+- all_my_circuits.jpg
+- circle.yml
+- lib/all_my_circuits.rb
+- lib/all_my_circuits/breaker.rb
+- lib/all_my_circuits/clock.rb
+- lib/all_my_circuits/exceptions.rb
+- lib/all_my_circuits/logging.rb
+- lib/all_my_circuits/notifiers.rb
+- lib/all_my_circuits/notifiers/abstract_notifier.rb
+- lib/all_my_circuits/notifiers/null_notifier.rb
+- lib/all_my_circuits/null_breaker.rb
+- lib/all_my_circuits/strategies.rb
+- lib/all_my_circuits/strategies/abstract_strategy.rb
+- lib/all_my_circuits/strategies/abstract_window_strategy.rb
+- lib/all_my_circuits/strategies/abstract_window_strategy/window.rb
+- lib/all_my_circuits/strategies/number_over_window_strategy.rb
+- lib/all_my_circuits/strategies/percentage_over_window_strategy.rb
+- lib/all_my_circuits/version.rb
+- script/fake_service.rb
+- script/graphing_server.rb
+- script/graphing_stress_test.rb
+homepage: https://github.com/remind101/all_my_circuits
+licenses:
+- BSD
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Circuit Breaker library with support for rolling-window absolute/percentage
+  thresholds
+test_files: []