pause 0.2.1 → 0.4.0

data/lib/pause/analyzer.rb

@@ -10,10 +10,10 @@ module Pause
     # @return [nil]  everything is fine
     # @return [false]  this action is already blocked
     # @return [Pause::RateLimitedEvent]  the action was blocked as a result of this check
-    def check(action)
-      return false if adapter.rate_limited?(action.scope, action.identifier)
+    def check(action, recalculate: false)
+      return false if adapter.rate_limited?(action.scope, action.identifier) && !recalculate
       timestamp = period_marker(Pause.config.resolution, Time.now.to_i)
-      set = adapter.key_history(action.scope, action.identifier)
+      set       = adapter.key_history(action.scope, action.identifier)
       action.checks.each do |period_check|
         start_time = timestamp - period_check.period_seconds
         set.reverse.inject(0) do |sum, element|

data/lib/pause/configuration.rb

@@ -3,7 +3,7 @@ module Pause
     attr_writer :redis_host, :redis_port, :redis_db, :resolution, :history, :sharded
 
     def configure
-      yield self
+      yield self if block_given?
       self
     end
 

data/lib/pause/redis/adapter.rb

@@ -5,28 +5,42 @@ module Pause
 
     # This class encapsulates Redis operations used by Pause
     class Adapter
+      class << self
+        def redis
+          @redis_conn ||= ::Redis.new(redis_connection_opts)
+        end
+
+        def redis_connection_opts
+          { host: Pause.config.redis_host,
+            port: Pause.config.redis_port,
+            db:   Pause.config.redis_db }
+        end
+      end
 
       include Pause::Helper::Timing
       attr_accessor :resolution, :time_blocks_to_keep, :history
 
       def initialize(config)
-        @resolution = config.resolution
+        @resolution          = config.resolution
         @time_blocks_to_keep = config.history / @resolution
-        @history = config.history
+        @history             = config.history
+      end
+
+      # Override in subclasses to disable
+      def with_multi
+        redis.multi do |redis|
+          yield(redis) if block_given?
+        end
       end
 
       def increment(scope, identifier, timestamp, count = 1)
         k = tracked_key(scope, identifier)
-        redis.multi do |redis|
+        with_multi do |redis|
           redis.zincrby k, count, period_marker(resolution, timestamp)
           redis.expire k, history
         end
 
-        if redis.zcard(k) > time_blocks_to_keep
-          list = extract_set_elements(k)
-          to_remove = list.slice(0, (list.size - time_blocks_to_keep))
-          redis.zrem(k, to_remove.map(&:ts))
-        end
+        truncate_set_for(k)
       end
 
       def key_history(scope, identifier)
@@ -78,7 +92,7 @@ module Pause
       end
 
       def disabled?(scope)
-        ! enabled?(scope)
+        !enabled?(scope)
       end
 
       def enabled?(scope)
@@ -91,15 +105,21 @@ module Pause
 
       private
 
-      def delete_tracking_keys(scope, ids)
-        increment_keys = ids.map{ |key| tracked_key(scope, key) }
-        redis.del(increment_keys)
+      def redis
+        self.class.redis
       end
 
-      def redis
-        @redis_conn ||= ::Redis.new(host: Pause.config.redis_host,
-                                    port: Pause.config.redis_port,
-                                    db:   Pause.config.redis_db)
+      def truncate_set_for(k)
+        if redis.zcard(k) > time_blocks_to_keep
+          list      = extract_set_elements(k)
+          to_remove = list.slice(0, (list.size - time_blocks_to_keep)).map(&:ts)
+          redis.zrem(k, to_remove) if k && to_remove && to_remove.size > 0
+        end
+      end
+
+      def delete_tracking_keys(scope, ids)
+        increment_keys = ids.map { |key| tracked_key(scope, key) }
+        redis.del(increment_keys)
       end
 
       def tracked_scope(scope)
@@ -117,7 +137,7 @@ module Pause
 
       def keys(key_scope)
         redis.keys("#{key_scope}:*").map do |key|
-          key.gsub(/^#{key_scope}:/, "").tr('|','')
+          key.gsub(/^#{key_scope}:/, "").tr('|', '')
         end
       end
 

data/lib/pause/redis/sharded_adapter.rb

@@ -7,26 +7,21 @@ module Pause
     # Operations that are not possible when data is sharded
     # raise an error.
     class ShardedAdapter < Adapter
-      def increment(scope, identifier, timestamp, count = 1)
-        k = tracked_key(scope, identifier)
-        redis.zincrby k, count, period_marker(resolution, timestamp)
-        redis.expire k, history
 
-        if redis.zcard(k) > time_blocks_to_keep
-          list = extract_set_elements(k)
-          to_remove = list.slice(0, (list.size - time_blocks_to_keep))
-          redis.zrem(k, to_remove.map(&:ts))
-        end
+      # Overrides real multi which is not possible when sharded.
+      def with_multi
+        yield(redis) if block_given?
       end
 
+      protected
 
-      private
-
-      def redis
-        @redis_conn ||= ::Redis.new(host: Pause.config.redis_host,
-          port: Pause.config.redis_port)
+      def redis_connection_opts
+        { host: Pause.config.redis_host,
+          port: Pause.config.redis_port }
       end
 
+      private
+
       def keys(_key_scope)
         raise OperationNotSupported.new('Can not be executed when Pause is configured in sharded mode')
       end

data/lib/pause/version.rb

@@ -1,3 +1,3 @@
 module Pause
-  VERSION = '0.2.1'
+  VERSION = '0.4.0'
 end

data/pause.gemspec

@@ -4,18 +4,27 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'pause/version'
 
 Gem::Specification.new do |gem|
-  gem.name          = "pause"
+  gem.name          = 'pause'
   gem.version       = Pause::VERSION
-  gem.authors       = ["Atasay Gokkaya", "Paul Henry", "Eric Saxby", "Konstantin Gredeskoul"]
-  gem.email         = %w(atasay@wanelo.com paul@wanelo.com sax@wanelo.com kig@wanelo.com)
-  gem.description   = %q(Real time rate limiting for multi-process ruby environments based on Redis)
-  gem.summary       = %q(RReal time rate limiting for multi-process ruby environments based on Redis)
-  gem.homepage      = "https://github.com/wanelo/pause"
+  gem.authors       = ['Atasay Gokkaya', 'Paul Henry', 'Eric Saxby', 'Konstantin Gredeskoul']
+  gem.email         = %w(atasay@wanelo.com paul@wanelo.com sax@ericsaxby.com kigster@gmail.com)
+  gem.summary       = %q(Fast, scalable, and flexible real time rate limiting library for distributed Ruby environments backed by Redis.)
+  gem.description   = %q(This gem provides highly flexible and easy to use interface to define rate limit checks, register events as they come, and verify if the rate limit is reached. Multiple checks for the same metric are easily supported. This gem is used at very high scale on several popular web sites.)
+  gem.homepage      = 'https://github.com/kigster/pause'
 
   gem.files         = `git ls-files`.split($/)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
-  gem.require_paths = ["lib"]
+  gem.require_paths = ['lib']
 
   gem.add_dependency 'redis'
+  gem.add_dependency 'hiredis'
+
+  gem.add_development_dependency 'simplecov'
+  gem.add_development_dependency 'yard'
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'fakeredis'
+  gem.add_development_dependency 'guard-rspec'
+  gem.add_development_dependency 'timecop'
+  gem.add_development_dependency 'rake'
 end

data/spec/pause/action_spec.rb

@@ -22,86 +22,169 @@ describe Pause::Action do
     allow(Pause).to receive(:adapter).and_return(adapter)
   end
 
-  let(:action) { MyNotification.new('1237612') }
-  let(:other_action) { MyNotification.new('1237613') }
-
-  describe '#increment!' do
-    it 'should increment' do
-      time = Time.now
-      Timecop.freeze time do
-        expect(Pause.adapter).to receive(:increment).with(action.scope, '1237612', time.to_i, 1)
-        action.increment!
+  let(:identifier) { '11112222' }
+  let(:action) { MyNotification.new(identifier) }
+
+  let(:other_identifier) { '8798734' }
+  let(:other_action) { MyNotification.new(other_identifier) }
+
+  RSpec.shared_examples 'an action' do
+    describe '#increment!' do
+      it 'should increment' do
+        time = Time.now
+        Timecop.freeze time do
+          expect(Pause.adapter).to receive(:increment).with(action.scope, identifier, time.to_i, 1)
+          action.increment!
+        end
       end
     end
-  end
 
-  describe '#ok?' do
-    it 'should successfully return if the action is blocked or not' do
-      time = Time.now
-      Timecop.freeze time do
-        4.times do
+    describe '#ok?' do
+      it 'should successfully return if the action is blocked or not' do
+        time = Time.now
+        Timecop.freeze time do
+          4.times do
+            action.increment!
+            expect(action.ok?).to be true
+          end
           action.increment!
+          expect(action.ok?).to be false
+        end
+      end
+
+      it 'should successfully consider different period checks' do
+        time = Time.parse('Sept 22, 11:34:00')
+
+        Timecop.freeze time - 30 do
+          action.increment! 4
+          expect(action.ok?).to be true
+        end
+
+        Timecop.freeze time do
+          action.increment! 2
           expect(action.ok?).to be true
         end
-        action.increment!
+
+        Timecop.freeze time do
+          action.increment! 1
+          expect(action.ok?).to be false
+        end
+      end
+
+      it 'should return false and silently fail if redis is not available' do
+        allow(Pause::Logger).to receive(:fatal)
+        allow_any_instance_of(Redis).to receive(:zrange).and_raise Redis::CannotConnectError
+        time = period_marker(resolution, Time.now.to_i)
+
+        action.increment! 4, time - 25
+
         expect(action.ok?).to be false
       end
     end
 
-    it 'should successfully consider different period checks' do
-      time = Time.parse('Sept 22, 11:34:00')
+    describe '#analyze' do
+      context 'action should not be rate limited' do
+        it 'returns nil' do
+          expect(adapter.rate_limited?(action.scope, action.identifier)).to be false
+          expect(action.analyze).to be nil
+        end
+      end
 
-      Timecop.freeze time - 30 do
-        action.increment! 4
-        expect(action.ok?).to be true
+      context 'action should be rate limited' do
+        it 'returns a RateLimitedEvent object' do
+          time       = Time.now
+          rate_limit = nil
+
+          Timecop.freeze time do
+            7.times { action.increment! }
+            rate_limit = action.analyze
+          end
+
+          expected_rate_limit = Pause::RateLimitedEvent.new(action, action.checks[0], 7, time.to_i)
+
+          expect(rate_limit).to be_a(Pause::RateLimitedEvent)
+          expect(rate_limit.identifier).to eq(expected_rate_limit.identifier)
+          expect(rate_limit.sum).to eq(expected_rate_limit.sum)
+          expect(rate_limit.period_check).to eq(expected_rate_limit.period_check)
+          expect(rate_limit.timestamp).to eq(expected_rate_limit.timestamp)
+        end
       end
+    end
 
-      Timecop.freeze time do
-        action.increment! 2
+    describe '#unblock' do
+      it 'unblocks the specified id' do
+        10.times { action.increment! }
+        expect(action.ok?).to be false
+        action.unblock
         expect(action.ok?).to be true
       end
+    end
 
-      Timecop.freeze time do
-        action.increment! 1
+    describe '#block_for' do
+      it 'blocks the IP for N seconds' do
+        expect(adapter).to receive(:rate_limit!).with(action.scope, action.identifier, 10).and_call_original
+        action.block_for(10)
         expect(action.ok?).to be false
       end
     end
+  end
 
-    it 'should return false and silently fail if redis is not available' do
-      allow(Pause::Logger).to receive(:fatal)
-      allow_any_instance_of(Redis).to receive(:zrange).and_raise Redis::CannotConnectError
-      time = period_marker(resolution, Time.now.to_i)
-
-      action.increment! 4, time - 25
-
-      expect(action.ok?).to be false
+  context 'actions under test' do
+    ['123456', 'hello', 0, 999999].each do |id|
+      let(:identifier) { id }
+      let(:action) { MyNotification.new(identifier) }
+      describe "action with identifier #{id}" do
+        it_behaves_like 'an action'
+      end
     end
   end
 
-  describe '#analyze' do
-    context 'action should not be rate limited' do
-      it 'returns nil' do
-        expect(action.analyze).to be nil
+  context 'DSL usage' do
+    class CowRateLimited < Pause::Action
+      scope 'cow:moo'
+      check period_seconds: 10, max_allowed: 2, block_ttl: 40
+      check period_seconds: 20, max_allowed: 4, block_ttl: 40
+    end
+
+    let(:identifier) { 'cow-moo' }
+    let(:action) { CowRateLimited.new(identifier) }
+    let(:bogus) { Struct.new(:name, :event).new }
+
+    describe '#unless_rate_limited' do
+      before do
+        expect(bogus).to receive(:name).exactly(2).times
+      end
+      it 'should call through the block' do
+        action.unless_rate_limited { bogus.name }
+        action.unless_rate_limited { bogus.name }
+        result = action.unless_rate_limited { bogus.name }
+        expect(result).to be_a_kind_of(::Pause::RateLimitedEvent)
       end
     end
 
-    context 'action should be rate limited' do
-      it 'returns a RateLimitedEvent object' do
-        time = Time.now
-        rate_limit = nil
+    describe '#unless_rate_limited' do
+      before { expect(bogus).to receive(:name).exactly(2).times }
 
-        Timecop.freeze time do
-          7.times { action.increment! }
-          rate_limit = action.analyze
-        end
+      it 'should call through the block' do
+        3.times { action.unless_rate_limited { bogus.name } }
+      end
+
+      describe '#if_rate_limited' do
+        before { 2.times { action.unless_rate_limited { bogus.name } } }
 
-        expected_rate_limit = Pause::RateLimitedEvent.new(action, action.checks[0], 7, time.to_i)
+        it 'it should not analyze during method call' do
+          bogus.event = 1
+          action.if_rate_limited { |event| bogus.event = event }
+          expect(bogus.event).to be_a_kind_of(::Pause::RateLimitedEvent)
+          expect(bogus.event.identifier).to eq(identifier)
+        end
 
-        expect(rate_limit).to be_a(Pause::RateLimitedEvent)
-        expect(rate_limit.identifier).to eq(expected_rate_limit.identifier)
-        expect(rate_limit.sum).to eq(expected_rate_limit.sum)
-        expect(rate_limit.period_check).to eq(expected_rate_limit.period_check)
-        expect(rate_limit.timestamp).to eq(expected_rate_limit.timestamp)
+        it 'should analyze if requested' do
+          action.unless_rate_limited { bogus.name }
+          result = action.if_rate_limited { |event| bogus.event = event }
+          expect(bogus.event).to be_a_kind_of(::Pause::RateLimitedEvent)
+          expect(result).to eq(bogus.event)
+        end
       end
     end
   end
@@ -150,25 +233,6 @@ describe Pause::Action do
     end
   end
 
-  describe '#unblock' do
-    it 'unblocks the specified id' do
-      10.times { action.increment! }
-
-      expect(action.ok?).to be false
-
-      action.unblock
-
-      expect(action.ok?).to be true
-    end
-  end
-
-  describe '#block_for' do
-    it 'blocks the IP for N seconds' do
-      expect(adapter).to receive(:rate_limit!).with(action.scope, action.identifier, 10).and_call_original
-      action.block_for(10)
-      expect(action.ok?).to be false
-    end
-  end
 end
 
 describe Pause::Action, '.check' do
@@ -188,34 +252,34 @@ describe Pause::Action, '.check' do
 
   it 'should define a period check on new instances' do
     expect(ActionWithCheck.new('id').checks).to eq([
-          Pause::PeriodCheck.new(100, 150, 200)
-        ])
+                                                     Pause::PeriodCheck.new(100, 150, 200)
+                                                   ])
   end
 
   it 'should define a period check on new instances' do
-    expect(ActionWithMultipleChecks.new('id').checks).to eq([
-          Pause::PeriodCheck.new(100, 150, 200),
-          Pause::PeriodCheck.new(200, 150, 200),
-          Pause::PeriodCheck.new(300, 150, 200)
-        ])
+    expect(ActionWithMultipleChecks.new('id').checks).to \
+      eq([
+                                                                    Pause::PeriodCheck.new(100, 150, 200),
+                                                                    Pause::PeriodCheck.new(200, 150, 200),
+                                                                    Pause::PeriodCheck.new(300, 150, 200)
+                                                                  ])
   end
 
   it 'should accept hash arguments' do
     expect(ActionWithHashChecks.new('id').checks).to eq([
-          Pause::PeriodCheck.new(50, 100, 60)
-        ])
+                                                          Pause::PeriodCheck.new(50, 100, 60)
+                                                        ])
   end
-
 end
 
 describe Pause::Action, '.scope' do
-  class UndefinedScopeAction < Pause::Action
+  module MyApp
+    class NoScope < ::Pause::Action
+    end
   end
 
   it 'should raise if scope is not defined' do
-    expect {
-      UndefinedScopeAction.new('1.2.3.4').scope
-    }.to raise_error('Should implement scope. (Ex: ipn:follow)')
+    expect(MyApp::NoScope.new('1.2.3.4').scope).to eq 'myapp.noscope'
   end
 
   class DefinedScopeAction < Pause::Action
@@ -236,7 +300,7 @@ describe Pause::Action, 'enabled/disabled states' do
   before do
     Pause.configure do |c|
       c.resolution = 10
-      c.history = 10
+      c.history    = 10
     end
   end