rollie 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1a4104ef316b1bf29321667fa9ea4e44dd642f4e
-  data.tar.gz: c695f97168e7ea2d7f47fe0a118bef23d8f5e24e
+  metadata.gz: e7dca59b24d14a1f68bc1e52eb8de74ca2fdb10c
+  data.tar.gz: c76e6ef7af25f4b7237c51ab794d2bfbe00eeaf5
 SHA512:
-  metadata.gz: 7f9da971ea951921729261a1920e640dfa9b6d1b8fc4580bb87454e6d7657a90583a52b17005a38ee61fdbfaa0dd7d38501e91cea39e9feefd7bbaf73e5c3068
-  data.tar.gz: a3eb75347541a6092ec9a8bfc085dbb5e0368560abf6dd143892d625b6be6f51e3b7693f208b08a7dcc2a287086d4922ad4d2d7afc6ba90d70e5f1924d0a31fe
+  metadata.gz: de6be4c8f9b384cae1755668b217835e92255fc323abbbdf6a257bdc8237942e06c6e6c0714ed2f4a9a33b7598c4d90030f2081fbe66d05d2968dcbad0ef91ae
+  data.tar.gz: 379cb65364722e9182b736e61430d6dd8c50daaa45eee19538040f35b1501195b5c13c0c6c405e70df787399f9eb25ab0e300cd782b97d24d40a9289a92d1b0e

data/.gitignore

@@ -0,0 +1,13 @@
+*.gem
+*.rbc
+Gemfile.lock
+.DS_Store
+dump.rdb
+doc/
+rdoc/
+/spec/reports/
+/.bundle
+tmp/
+.ruby-gemset
+.ruby-version
+.idea/

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+services:
+  - redis-server

data/CHANGELOG.md

@@ -0,0 +1,12 @@
+# Changelog
+
+## 0.1.0
+
+* fix for intervals < 1000 ms
+* return actual count (not capped at limit)
+* option to count blocked executions against total count (defaults to false)
+* add ability to fetch current count
+
+## 0.0.1
+
+* Initial release

data/Gemfile

@@ -0,0 +1,8 @@
+source "https://rubygems.org"
+gemspec :name => "rollie"
+
+gem "rake"
+
+group :test do
+  gem "rspec", "~> 3.4"
+end

data/README.md

@@ -1,5 +1,94 @@
 # Rollie
 
-Rollie is a generic rate limiter backed by Redis for efficient limiting using sliding windows.
+[![Build Status](https://travis-ci.org/zldavis/rollie.svg?branch=master)](https://travis-ci.org/zldavis/rollie)
 
-[![Build Status](https://travis-ci.org/zldavis/rollie.svg?branch=master)](https://travis-ci.org/zldavis/rollie)
+Rollie is a multi-purpose, fast, redis backed rate limiter that can be used to limit requests to external APIs, in Rack
+middleware, etc. Rollie uses a dedicated redis connection pool implemented using `connection_pool` for more efficient
+redis connection management.
+   
+The key implementation detail is that Rollie utilizes a rolling window to bucket invocations in. Meaning, if you set
+a limit of 100 per 30 seconds, Rollie will start the clock in instant it is first executed with a given key.
+     
+For example, first execution:
+```
+rollie = Rollie::RateLimiter.new("api", limit: 10, interval: 30000)
+rollie.within_limit do
+   puts Time.now
+end
+# => 2016-12-03 08:31:23.873
+```
+
+This doesn't mean the count is reset back to 0 at `2016-12-03 08:31:53.873`. Its a continuous rolling count, the count
+is checked with every invocation over the last 30 seconds.
+ 
+If you invoke this rate 9 times at `2016-12-03 08:31:53.500`, you will only be able to make one more call until `2016-12-03 08:32:23.500`. 
+
+## Install
+
+```
+gem install rollie
+```
+
+## Usage
+
+Rollie is simple to use and has only one method, `within_limit`. `within_limit` expects a block and that block will be
+executed only if you are within the limit.
+
+Initialize Rollie with a key used to uniquely identify what you are limiting. Use the options to set the limit and
+interval in milliseconds.
+```
+# limit 30 requests per second.
+twitter_rate = Rollie::RateLimiter.new("twitter_requests", limit: 30, interval: 1000)
+status = twitter_rate.within_limit do
+  twitter.do_something
+end
+```
+
+The status will tell you the current state. You can also see the current count and how long until the bucket resets.
+```
+status.exceeded?
+# => false
+status.count
+# => 1
+status.time_remaining
+# => 987 # milliseconds
+```
+
+Once exceeded:
+```
+status.exceeded?
+# => true
+status.count
+# => 30
+status.time_remaining
+# => 461 # milliseconds
+```  
+
+You can also use a namespace if you want to track multiple entities, for example users.
+```
+Rollie::RateLimiter.new(user_id, namespace: "user_messages", limit: 100, interval: 30000)
+```
+
+### Counting blocked actions
+
+By default, blocked actions are not counted against the callee. This allows for the block to be executed within the
+rate even when there is a continuous flood of action. If you wish to change this behaviour, for example to require the callee to back off before being allowed to excute again, set this option to true.
+
+```
+request_rate = Rollie::RateLimiter.new(ip, namespace: "ip", limit: 30, interval: 1000, count_blocked: true)
+```
+
+## Configuration
+
+By default Rollie will try to connect to redis using `ENV["REDIS_URL"]` if set or fallback to localhost:6379. You can
+set an alternate redis configuration:
+```
+Rollie.redis = {
+    url: CONFIG[:redis_url],
+    pool_size: 5,
+    pool_timeout: 1,
+    driver: :hiredis
+}
+```
+
+If using rails, create an initializer `config/initializers/rollie.rb` with these settings.

data/Rakefile

@@ -0,0 +1,28 @@
+require "rake"
+require "rdoc"
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), *%w[lib]))
+require "rollie/version"
+
+def name
+  "rollie"
+end
+
+def version
+  Rollie::VERSION
+end
+
+begin
+  require "rspec/core/rake_task"
+  RSpec::Core::RakeTask.new(:spec)
+rescue LoadError; end
+
+require "rdoc/task"
+Rake::RDocTask.new do |rdoc|
+  rdoc.rdoc_dir = "rdoc"
+  rdoc.title = "#{name} #{version}"
+  rdoc.rdoc_files.include("README*")
+  rdoc.rdoc_files.include("lib/**/*.rb")
+end
+
+task :default => :spec

data/lib/rollie.rb

@@ -5,6 +5,7 @@ require "rollie/version"
 
 module Rollie
   class << self
+
     def redis
       raise ArgumentError, "requires a block" unless block_given?
       redis_pool.with do |conn|
@@ -12,17 +13,25 @@ module Rollie
       end
     end
 
-    def redis=(hash)
-      @redis_pool = if hash.is_a?(ConnectionPool)
-         hash
+    # Configures the redis connection pool. Options can be a hash of redis connection pool options or a pre-configured
+    # ConnectionPool instance.
+    #
+    # @option options [String] :url The redis connection URL
+    # @option options [String] :driver The redis driver
+    # @option options [Integer] :pool_size Size of the connection pool
+    # @option options [Integer] :pool_timeout Pool timeout in seconds
+    # @option options [String] :namespace Optional namespace for redis keys
+    def redis=(options)
+      @redis_pool = if options.is_a?(ConnectionPool)
+        options
        else
-         Rollie::RedisPool.create(hash)
+        Rollie::RedisPool.create(options)
        end
     end
 
     def redis_pool
       @redis_pool ||= Rollie::RedisPool.create
     end
-  end
 
+  end
 end

data/lib/rollie/rate_limiter.rb

@@ -2,12 +2,25 @@ module Rollie
 
   class RateLimiter
 
+    # Create a new RateLimiter instance.
+    #
+    # @param [String] key A unique name to track this rate limit against.
+    # @option options [Integer] :limit The limit
+    # @option options [Integer] :interval The interval in milliseconds for this rate limit
+    # @option options [String] :namespace Optional namespace for this rate limit
+    # @option options [Boolean] :count_blocked if true, all calls to within_limit will count towards total execution count, even if blocked.
+    #
+    # @return [RateLimiter] RateLimiter instance
     def initialize(key, options = {})
       @key = "#{options[:namespace]}#{key}"
       @limit = options[:limit] || 25
       @interval = (options[:interval] || 1000) * 1000
+      @count_blocked = options.key?(:count_blocked) ? options[:count_blocked] : false
     end
 
+    # Executes a block as long as the current rate is within the limit.
+    #
+    # @return [Status] The current status for this RateLimiter.
     def within_limit
       raise ArgumentError, "requires a block" unless block_given?
 
@@ -20,6 +33,14 @@ module Rollie
       end
     end
 
+    # @return [Integer] The current count of this RateLimiter.
+    def count
+      Rollie.redis do |conn|
+        range = conn.zrange(@key, 0, -1)
+        range.length
+      end
+    end
+
     private
 
     def inc(conn)
@@ -29,12 +50,18 @@ module Rollie
         conn.zremrangebyscore(@key, 0, old)
         conn.zadd(@key, time, time)
         conn.zrange(@key, 0, -1)
-        conn.expire(@key, (@interval / 1000000).ceil)
+        conn.expire(@key, (@interval / 1000000.0).ceil)
       end[2]
 
       exceeded = range.length > @limit
+      current_count = range.length
       time_remaining = range.first.to_i - time + @interval
-      current_count = exceeded ? @limit : range.length
+
+      if exceeded && !@count_blocked
+        conn.zremrangebyscore(@key, time, time)
+        current_count -= 1
+      end
+
       Rollie::Status.new((time_remaining / 1000).floor, current_count, exceeded)
     end
 

data/lib/rollie/redis_pool.rb

@@ -7,7 +7,6 @@ module Rollie
     class << self
 
       def create(options={})
-        puts "initialized with options: #{options}"
         pool_size = options[:pool_size] || 5
         pool_timeout = options[:pool_timeout] || 1
 

data/lib/rollie/version.rb

@@ -1,3 +1,3 @@
 module Rollie
-  VERSION = "0.0.1".freeze
+  VERSION = "0.1.0".freeze
 end

data/rollie.gemspec

@@ -0,0 +1,24 @@
+$LOAD_PATH.push File.expand_path("../lib", __FILE__)
+require "rollie/version"
+
+Gem::Specification.new do |s|
+  s.name = "rollie"
+  s.version = Rollie::VERSION
+  s.license = "MIT"
+
+  s.summary = "Generic rate limiter backed by Redis for efficient limiting using sliding windows."
+  s.description = s.summary
+
+  s.authors = ["Zach Davis"]
+  s.email = "zldavis@gmail.com"
+  s.homepage = "https://github.com/zldavis/rollie"
+
+  s.files = `git ls-files -z`.split("\x0")
+  s.test_files = `git ls-files -- test/*`.split("\n")
+
+  s.require_paths = ["lib"]
+
+  s.add_dependency "redis", "~> 3.2", ">= 3.2.1"
+  s.add_dependency "redis-namespace", "~> 1.5", ">= 1.5.2"
+  s.add_dependency "connection_pool", "~> 2.2", ">= 2.2.0"
+end

data/spec/rollie/rate_limiter_spec.rb

@@ -0,0 +1,83 @@
+require "spec_helper"
+
+module Rollie
+  describe RateLimiter do
+
+    before do
+      @r = RateLimiter.new(SecureRandom.hex(8), count_blocked: true)
+    end
+
+    describe :within_limit do
+
+      it "should require a block" do
+        expect{ @r.within_limit }.to raise_error(ArgumentError)
+      end
+
+      it "should return status" do
+        status = @r.within_limit do; end
+        expect(status.count).to eq(1)
+        expect(status.exceeded?).to be(false)
+        expect(status.time_remaining).to eq(1000)
+      end
+
+      it "should execute block only while within limit" do
+        count = 0
+        status = nil
+        30.times do
+          status = @r.within_limit do
+            count += 1
+          end
+        end
+        expect(count).to eq(25)
+        expect(status.count).to eq(30)
+        expect(status.exceeded?).to be(true)
+      end
+
+      it "should block all actions within the window" do
+        @r = RateLimiter.new(SecureRandom.hex(8), limit: 10, interval: 100, count_blocked: true)
+        count = 0
+        30.times do
+          @r.within_limit do
+            count += 1
+          end
+          sleep 0.004
+        end
+        expect(count).to eq(10)
+      end
+
+      it "should allow blocked actions not to be counted" do
+        @r = RateLimiter.new(SecureRandom.hex(8), limit: 10, interval: 100, count_blocked: false)
+        count = 0
+        30.times do
+          @r.within_limit do
+            count += 1
+          end
+          sleep 0.004
+        end
+        expect(count).to eq(20)
+      end
+
+    end
+
+    describe :count do
+
+      it "should return the current count" do
+        30.times do
+          @r.within_limit do; sleep 0.001; end
+        end
+
+        expect(@r.count).to eq(30)
+
+        @r = RateLimiter.new(SecureRandom.hex(8), limit: 10, count_blocked: false)
+
+        30.times do
+          @r.within_limit do; sleep 0.001; end
+        end
+
+        expect(@r.count).to eq(10)
+      end
+
+    end
+
+  end
+end

data/spec/rollie_spec.rb

@@ -0,0 +1,36 @@
+require "spec_helper"
+
+describe Rollie do
+
+  describe :redis do
+
+    it "should require a block" do
+      expect{ Rollie.redis }.to raise_error(ArgumentError)
+    end
+
+    it "should return a Redis instance from the pool" do
+      Rollie.redis do |conn|
+        expect(conn.class).to eq(Redis::Namespace)
+      end
+    end
+
+  end
+
+  describe :redis= do
+
+    it "should allow hash options to initialize connection pool" do
+      options = {url: "redis://foo"}
+      pool = ConnectionPool.new do; end
+      expect(Rollie::RedisPool).to receive(:create).with(options).and_return(pool)
+      Rollie.redis = options
+    end
+
+    it "should allow a connection pool" do
+      pool = ConnectionPool.new do; end
+      Rollie.redis = pool
+      expect(Rollie.redis_pool).to eq(pool)
+    end
+
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,106 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause
+# this file to always be loaded, without a need to explicitly require it in any
+# files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need
+# it.
+#
+# The `.rspec` file also contains a few flags that are not defaults but that
+# users commonly want.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+
+require "rollie"
+
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    #     be_bigger_than(2).and_smaller_than(4).description
+    #     # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #     # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+  # This option will default to `:apply_to_host_groups` in RSpec 4 (and will
+  # have no way to turn it off -- the option exists only for backwards
+  # compatibility in RSpec 3). It causes shared context metadata to be
+  # inherited by the metadata hash of host groups and examples, rather than
+  # triggering implicit auto-inclusion in groups with matching metadata.
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # This allows you to limit a spec run to individual examples or groups
+  # you care about by tagging them with `:focus` metadata. When nothing
+  # is tagged with `:focus`, all examples get run. RSpec also provides
+  # aliases for `it`, `describe`, and `context` that include `:focus`
+  # metadata: `fit`, `fdescribe` and `fcontext`, respectively.
+  config.filter_run_when_matching :focus
+
+  # Allows RSpec to persist some state between runs in order to support
+  # the `--only-failures` and `--next-failure` CLI options. We recommend
+  # you configure your source control system to ignore this file.
+  config.example_status_persistence_file_path = "spec/examples.txt"
+
+  # Limits the available syntax to the non-monkey patched syntax that is
+  # recommended. For more details, see:
+  #   - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
+  #   - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
+  config.disable_monkey_patching!
+
+  # This setting enables warnings. It's recommended, but in some cases may
+  # be too noisy due to issues in dependencies.
+  config.warnings = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = 'doc'
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+=end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rollie
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Zach Davis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-03 00:00:00.000000000 Z
+date: 2016-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -75,17 +75,25 @@ description: Generic rate limiter backed by Redis for efficient limiting using s
 email: zldavis@gmail.com
 executables: []
 extensions: []
-extra_rdoc_files:
-- README.md
-- LICENSE
+extra_rdoc_files: []
 files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CHANGELOG.md
+- Gemfile
 - LICENSE
 - README.md
+- Rakefile
 - lib/rollie.rb
 - lib/rollie/rate_limiter.rb
 - lib/rollie/redis_pool.rb
 - lib/rollie/status.rb
 - lib/rollie/version.rb
+- rollie.gemspec
+- spec/rollie/rate_limiter_spec.rb
+- spec/rollie_spec.rb
+- spec/spec_helper.rb
 homepage: https://github.com/zldavis/rollie
 licenses:
 - MIT