circuit_breakage 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dad99a3f4f0f6b939a2092e865ac7dca10b5fe4b
-  data.tar.gz: 6e7684487d5a883b407013767e8f6a454933c9fb
+  metadata.gz: 1b30cd904f63ca99c1f7b95a89bfef2a5c29237d
+  data.tar.gz: c55ce5e3e2bf0463175dcd2c25ace63543ef2e1e
 SHA512:
-  metadata.gz: 4895d56acb08b9c942e1f571162017b46b40da00b5872b5a5891d411dd5db3461d48d0f21c3ef8227efbf3aab48e3c7df10a1f79ea32e50b45cefc43a2b6d6cf
-  data.tar.gz: 3f9da79d01746dcf5e3d8e14ae4adc7eb3f831370ed1cbf55842d97eca744fd15f85662c572905038c7e4f7c61a38a141ab9a26c8de0df0c803bf6aa0fabb963
+  metadata.gz: 7dfae02fe9c5876c2a1caa6454231dd33f325e9ddd6814b8a9fbc107e302fb15e44afb8cb6d55a0f09991caf6d6194ad1fc28af26387b9afc0db0d89687b6db7
+  data.tar.gz: 4fa1879643290ddc117d4ff8a3aac6cd0137230dba40953b11f6ab468fdcab9a7e69e407e121e40ab56dc8a74e4ea697484bdaf761ebd8ae4c0c5c26a64af5cf

data/README.md

@@ -1,20 +1,48 @@
 # CircuitBreakage
 
 A simple Circuit Breaker implementation in Ruby with a timeout.  A Circuit
-Breaker wraps potentially troublesome logic and will "trip" the circuit (and
+Breaker wraps potentially troublesome logic and will "trip" the circuit (ie,
 stop trying to run the logic) if it sees too many failures.  After a while, it
 will retry.
 
 ## Usage
 
+### Normal Boring Circuit Breakers
+
 ```ruby
 block = ->(*args) do
   # Some dangerous thing.
 end
 
-breaker = CircuitBreakage.new(block)
+breaker = CircuitBreakage::Breaker.new(block)
 breaker.failure_threshold = 3 # only 3 failures before tripping circuit
 breaker.duration = 10         # 10 seconds before retry
 breaker.timeout = 0.5         # 500 milliseconds allowed before auto-fail
 
-breaker.call(*some_args)      # args are passed through to block
+begin
+  breaker.call(*some_args)    # args are passed through to block
+rescue CircuitBreaker::CircuitOpen
+  puts "Too many recent failures!"
+rescue CircuitBreaker::CircuitTimeout
+  puts "Operation timed out!"
+end
+```
+
+### Redis-backed "Shared" Circuit Breakers
+
+The unique feature of this particular Circuit Breaker gem is that it also
+supports shared state via Redis, using the SETNX and GETSET commands.  This
+allows a number of circuit breakers running in separate processes to trip and
+un-trip in unison.
+
+```ruby
+connection = some_redis_connection
+key = 'my_app/some_operation'
+
+breaker = CircuitBreakage::RedisBackedBreaker.new(connection, key, block)
+# Everything else is the same as above.
+```
+
+So, if you have the same piece of code running on 27 instances across 3
+different servers, as soon as one trips, they all trip, and as soon as one
+resets, they all reset.

data/lib/circuit_breakage/breaker.rb

@@ -16,63 +16,62 @@ module CircuitBreakage
     DEFAULT_TIMEOUT           = 10    # Number of seconds before the call times out
 
     def initialize(block)
-      @block = block
+      self.block              = block
       self.failure_threshold  = DEFAULT_FAILURE_THRESHOLD
       self.duration           = DEFAULT_DURATION
       self.timeout            = DEFAULT_TIMEOUT
-
-      self.failure_count ||= 0
-      self.last_failed   ||= Time.at(0)
-      closed!
+      self.failure_count      ||= 0
+      self.last_failed        ||= Time.at(0)
+      self.state              ||= 'closed'
     end
 
     def call(*args)
-      if open?
+      case(state)
+      when 'open'
         if time_to_retry?
-          half_open!
+          do_retry(*args)
         else
           raise CircuitOpen
         end
+      when 'closed'
+        do_call(*args)
       end
+    end
 
-      begin
-        ret_value = nil
-        Timeout.timeout(self.timeout, CircuitTimeout) do
-          ret_value = @block.call(*args)
-        end
-        handle_success
+    private
 
-        return ret_value
-      rescue Exception => e
-        handle_failure
-      end
+    # Defined independently so that it can be overridden.
+    def do_retry(*args)
+      do_call(*args)
     end
 
-    [:open, :closed, :half_open].each do |state|
-      define_method("#{state}?") {
-        self.state == state
-      }
+    def do_call(*args)
+      ret_value = nil
+      Timeout.timeout(self.timeout, CircuitTimeout) do
+        ret_value = @block.call(*args)
+      end
+      handle_success
 
-      define_method("#{state}!") {
-        self.state = state
-      }
+      return ret_value
+    rescue Exception => e
+      handle_failure
     end
 
-    private
-
     def time_to_retry?
       Time.now >= self.last_failed + self.duration
     end
 
     def handle_success
-      closed!
       self.failure_count = 0
+      self.state = 'closed'
     end
 
     def handle_failure
       self.last_failed = Time.now
       self.failure_count += 1
-      open! if self.failure_count >= self.failure_threshold
+      if self.failure_count >= self.failure_threshold
+        self.state = 'open'
+      end
     end
   end
 end

data/lib/circuit_breakage/redis_backed_breaker.rb

@@ -0,0 +1,61 @@
+module CircuitBreakage
+  # Similar to Breaker, but accepts a Redis connection and an arbitrary key,
+  # and will share state among an arbitrary number of RedisBackedBreakers via
+  # Redis. Relies on the SETNX redis command.
+  #
+  class RedisBackedBreaker < Breaker
+
+    # How long before we decide a lock-holder has crashed, in seconds.
+    LOCK_TIMEOUT = DEFAULT_TIMEOUT + 10
+
+    attr_reader :connection, :key
+
+    def initialize(connection, key, block)
+      raise NotImplementedError.new("Still working on it!")
+
+      @connection = connection
+      @key = key
+      super(block)
+    end
+
+    private
+
+    def do_retry(*args)
+      try_with_mutex('half_open_retry') do
+        super
+      end
+    end
+
+    def try_with_mutex(lock, &block)
+      mutex_key = "#{@key}/locks/#{lock}"
+
+      acquired = @connection.setnx(mutex_key, Time.now.to_i)
+      if acquired = 0   # mutex is already acquired
+        locked_at = @connection.get(mutex_key)
+        return if locked_at + LOCK_TIMEOUT < Time.now.to_i  # unexpired lock
+        locked_at_second_check = @connection.getset(mutex_key, Time.now.to_i)
+        return if locked_at_second_check != locked_at       # expired lock, but somebody beat us to it
+      end
+      # If we get this far, we have successfully acquired the mutex.
+
+      begin
+        block.call
+      ensure
+        @connection.del(mutex_key)
+      end
+    end
+
+    [:state, :failure_count, :last_failed].each do |attr|
+      attr_key = "#{@key}/attrs/#{attr}"
+
+      define_method(attr) do
+        @connection.get(attr_key)
+      end
+
+      define_method("#{attr}=") do |value|
+        @connection.set(attr_key, value)
+      end
+    end
+
+  end
+end

data/lib/circuit_breakage/version.rb

@@ -1,3 +1,3 @@
 module CircuitBreakage
-  VERSION = "0.0.1"
+  VERSION = "0.1.0"
 end

data/spec/{caching_breaker_spec.rb → redis_backed_breaker_spec.rb}

@@ -2,11 +2,11 @@
 # group for this and the vanilla Breaker spec.
 
 module CircuitBreakage
-  describe CachingBreaker do
-    let(:breaker) { CachingBreaker.new(cache, key, block) }
-    let(:cache)   { MockCache.new }
-    let(:key)     { 'test/data' }
-    let(:block)   { ->(x) { return x } }
+  describe RedisBackedBreaker do
+    let(:breaker)    { RedisBackedBreaker.new(connection, key, block) }
+    let(:connection) { MockCache.new }
+    let(:key)        { 'test/data' }
+    let(:block)      { ->(x) { return x } }
 
     describe '#call' do
       subject { -> { breaker.call(arg) } }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: circuit_breakage
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - John Hyland
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-04 00:00:00.000000000 Z
+date: 2014-12-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -83,10 +83,10 @@ files:
 - circuit_breakage.gemspec
 - lib/circuit_breakage.rb
 - lib/circuit_breakage/breaker.rb
-- lib/circuit_breakage/caching_breaker.rb
+- lib/circuit_breakage/redis_backed_breaker.rb
 - lib/circuit_breakage/version.rb
 - spec/breaker_spec.rb
-- spec/caching_breaker_spec.rb
+- spec/redis_backed_breaker_spec.rb
 - spec/spec_helper.rb
 homepage: https://source.datanerd.us/jhyland/circuit_breakage
 licenses:
@@ -114,5 +114,5 @@ specification_version: 4
 summary: Provides a simple circuit breaker pattern.
 test_files:
 - spec/breaker_spec.rb
-- spec/caching_breaker_spec.rb
+- spec/redis_backed_breaker_spec.rb
 - spec/spec_helper.rb

data/lib/circuit_breakage/caching_breaker.rb

@@ -1,31 +0,0 @@
-module CircuitBreakage
-  # Similar to Breaker, but accepts a cache object, and will call #write and
-  # #fetch on that object to store and retrieve all state, instead of keeping
-  # it in memory.
-  #
-  class CachingBreaker < Breaker
-    attr_reader :cache, :key
-
-    def initialize(cache, key, block)
-      @cache = cache
-      @key = key
-      super(block)
-    end
-
-    def self.cached_attr(*attrs)
-      attrs.each do |attr|
-        define_method attr do
-          raise "You must define the cache and key on a CachingBreaker!" unless cache && key
-          cache.fetch "#{key}/#{attr}"
-        end
-
-        define_method "#{attr}=" do |value|
-          raise "You must define the cache and key on a CachingBreaker!" unless cache && key
-          cache.write "#{key}/#{attr}", value
-        end
-      end
-    end
-
-    cached_attr :failure_count, :last_failed, :state
-  end
-end