sidekiq-rate-limiter 0.0.0 → 0.0.1

data/.gitignore

@@ -4,3 +4,4 @@ Gemfile.lock
 pkg/
 coverage/
 *.swp
+spec/support/redis/

data/.travis.yml

@@ -2,4 +2,3 @@ language: ruby
 rvm:
   - 2.0.0
   - 1.9.3
-  - 1.8.7

data/README.md

@@ -1,4 +1,111 @@
 sidekiq-rate-limiter
 ====================
 
-Adds to sidekiq the ability to rate limit job execution on a per-worker basis in a redis-backed fashion.
+[![Build Status](https://secure.travis-ci.org/enova/sidekiq-rate-limiter.png)](http://travis-ci.org/enova/sidekiq-rate-limiter)
+
+Adds to sidekiq the ability to rate limit job execution on a per-worker basis in
+a redis-backed fashion.
+
+## Compatibility
+
+sidekiq-rate-limiter is actively tested against MRI versions 2.0.0 and 1.9.3.
+
+sidekiq-rate-limiter works by using a custom fetch class, the class responsible
+for pulling work from the queue stored in redis. Consequently you'll want to be
+careful about using other gems that use a same strategy ([sidekiq-priority](https://github.com/socialpandas/sidekiq-priority)
+being one example.
+
+I've attempted to support the same options as used by [sidekiq-throttler](/gevans/sidekiq-throttler). So, if
+your worker already looks like this example I lifted from the sidekiq-throttler wiki:
+
+```ruby
+class MyWorker
+  include Sidekiq::Worker
+
+  sidekiq_options throttle: { threshold: 50, period: 1.hour }
+
+  def perform(user_id)
+    # Do some heavy API interactions.
+  end
+end
+```
+
+Then you wouldn't need to change anything. 
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'sidekiq-rate-limiter'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install sidekiq-rate-limiter
+
+## Configuration
+
+See [server.rb](lib/sidekiq-rate-limiter/server.rb) for an example of how to
+configure sidekiq-rate-limiter. Alternatively you can add the following to your
+initializer or what-have-you:
+
+```ruby
+require 'sidekiq-rate-limiter/server'
+```
+
+Or, if you prefer, amend your Gemfile like so:
+
+    gem 'sidekiq-rate-limiter', :require => 'sidekiq-rate-limiter/server'
+
+By default the limiter uses the name 'sidekiq-rate-limiter'. You can define the
+constant ```Sidekiq::RateLimiter::DEFAULT_LIMIT_NAME``` prior to requiring to
+change this. Alternatively, you can include a 'name' parameter in the configuration
+hash included in sidekiq_options
+
+For example, the following:
+
+```ruby
+  class Job
+    include Sidekiq::Worker
+
+    sidekiq_options :queue => 'some_silly_queue',
+                    :rate  => {
+                      :name   => 'my_super_awesome_rate_limit',
+                      :limit  => 50,
+                      :period => 3600, ## An hour
+                    }
+
+    def perform(*args)
+      ## do stuff
+      ## ...
+```
+
+The configuration above would result in any jobs beyond the first 50 in a one
+hour period being delayed. The server will continue to fetch items from redis, &
+will place any items that are beyond the threshold at the back of their queue.
+
+## Motivation
+
+Sidekiq::Throttler is great for smaller quantities of jobs, but falls down a bit
+for larger queues (see [issue #8](/gevans/sidekiq-throttler/issues/8). In addition, jobs that are
+limited multiple times are counted as 'processed' each time, so the stats baloon quickly.
+
+## TODO
+
+* Most or all of the configuration options should support procs
+* While it subclasses instead of monkey patching, setting Sidekiq.options[:fetch]
+is still asking for interaction issues. It would be better for this to be directly
+in sidekiq or to use some other means to accomplish this goal.
+
+## Contributing
+
+1. Fork
+2. Commit
+5. Pull Request
+
+## License
+
+MIT. See [LICENSE](LICENSE) for details.

data/lib/sidekiq-rate-limiter.rb

@@ -1 +1,2 @@
 require 'sidekiq-rate-limiter/version'
+require 'sidekiq-rate-limiter/fetch'

data/lib/sidekiq-rate-limiter/fetch.rb

@@ -0,0 +1,54 @@
+require 'celluloid'
+require 'sidekiq/fetch'
+require 'redis_rate_limiter'
+
+module Sidekiq::RateLimiter
+  DEFAULT_LIMIT_NAME =
+    'sidekiq-rate-limit'.freeze unless defined?(DEFAULT_LIMIT_NAME)
+
+  class Fetch < Sidekiq::BasicFetch
+    def retrieve_work
+      limit(super)
+    end
+
+    def limit(work)
+      message = JSON.parse(work.message) rescue {}
+
+      klass     = message['class']
+      rate      = message['rate'] || message['throttle'] || {}
+      limit     = rate['limit']   || rate['threshold']
+      interval  = rate['period']  || rate['interval']
+      name      = rate['name']    || DEFAULT_LIMIT_NAME
+
+      return work unless !!(klass && limit && interval)
+
+      options = {
+        :limit    => limit,
+        :interval => interval,
+        :name     => name,
+      }
+
+      Sidekiq.redis do |conn|
+        lim = Limit.new(conn, options)
+        if lim.exceeded?(klass)
+          conn.lpush("queue:#{work.queue_name}", work.message)
+          nil
+        else
+          lim.add(klass)
+          work
+        end
+      end
+    end
+  end
+
+  class Limit < RedisRateLimiter
+    def initialize(redis, options = {})
+      options = options.dup
+      name = options.delete('name') ||
+             options.delete(:name)
+
+      super(name, redis, options)
+    end
+  end
+
+end

data/lib/sidekiq-rate-limiter/server.rb

@@ -0,0 +1,6 @@
+require 'sidekiq-rate-limiter/version'
+require 'sidekiq-rate-limiter/fetch'
+
+Sidekiq.configure_server do |config|
+  Sidekiq.options[:fetch] = Sidekiq::RateLimiter::Fetch
+end

data/lib/sidekiq-rate-limiter/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   module RateLimiter
-    VERSION = "0.0.0"
+    VERSION = "0.0.1"
   end
 end

data/sidekiq-rate-limiter.gemspec

@@ -24,6 +24,9 @@ Gem::Specification.new do |s|
   s.add_development_dependency "rake"
   s.add_development_dependency "rspec"
   s.add_development_dependency "simplecov"
+  s.add_development_dependency "simplecov-rcov"
 
+  s.add_dependency "redis"
   s.add_dependency "sidekiq"
+  s.add_dependency "redis_rate_limiter"
 end

data/spec/sidekiq-rate-limiter/fetch_spec.rb

@@ -0,0 +1,64 @@
+require 'spec_helper'
+require 'sidekiq'
+
+describe Sidekiq::RateLimiter::Fetch do
+  before(:all) do
+    class Job
+      include Sidekiq::Worker
+      sidekiq_options 'queue'    => 'basic',
+                      'retry'    => false,
+                      'rate' => {
+                          'limit'  => 1,
+                          'period' => 1
+                      }
+      def perform(*args); end
+    end
+  end
+
+  let(:options)       { { queues: [queue, another_queue, another_queue] } }
+  let(:queue)         { 'basic' }
+  let(:another_queue) { 'some_other_queue' }
+  let(:args)          { ['I am some args'] }
+  let(:worker)        { Job }
+  let(:redis_class)   { Sidekiq.redis { |conn| conn.class } }
+
+  it 'should inherit from Sidekiq::BasicFetch' do
+    described_class.should < Sidekiq::BasicFetch
+  end
+
+  it 'should retrieve work with strict setting' do
+    fetch = described_class.new options.merge(:strict => true)
+    fetch.queues_cmd.should eql(["queue:#{queue}", "queue:#{another_queue}", 1])
+  end
+
+  it 'should retrieve work', queuing: true do
+    worker.perform_async(*args)
+    fetch = described_class.new(options)
+    work    = fetch.retrieve_work
+    parsed  = JSON.parse(work.message)
+
+    work.should_not be_nil
+    work.queue_name.should eql(queue)
+    work.acknowledge.should be_nil
+
+    parsed.should include(worker.get_sidekiq_options)
+    parsed.should include("class" => worker.to_s, "args" => args)
+    parsed.should include("jid", "enqueued_at")
+
+    q = Sidekiq::Queue.new(queue)
+    q.size.should == 0
+  end
+
+  it 'should place rate-limited work at the back of the queue', queuing: true do
+    worker.perform_async(*args)
+    Sidekiq::RateLimiter::Limit.any_instance.should_receive(:exceeded?).and_return(true)
+    redis_class.any_instance.should_receive(:lpush).exactly(:once).and_call_original
+
+    fetch = described_class.new(options)
+    fetch.retrieve_work.should be_nil
+
+    q = Sidekiq::Queue.new(queue)
+    q.size.should == 1
+  end
+
+end

data/spec/sidekiq-rate-limiter/server_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe Sidekiq::RateLimiter, 'server configuration' do
+  before do
+    Sidekiq.stub(:server? => true)
+    require 'sidekiq-rate-limiter/server'
+  end
+
+  it 'should set Sidekiq.options[:fetch] as desired' do
+    Sidekiq.configure_server do |config|
+      Sidekiq.options[:fetch].should eql(Sidekiq::RateLimiter::Fetch)
+    end
+  end
+
+  it 'should inherit from Sidekiq::BasicFetch' do
+    Sidekiq.configure_server do |config|
+      Sidekiq.options[:fetch].should < Sidekiq::BasicFetch
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -1,3 +1,51 @@
+require 'sidekiq'
+require 'sidekiq/testing'
+
+## Confirming presence of redis server executable
+abort "## `redis-server` not in path" if %x(which redis-server).empty?
+redis_dir = "#{File.dirname(__FILE__)}/support/redis"
+
+## Redis configuration
+REDIS_CONFIG = <<-CONF
+  daemonize yes
+  pidfile #{redis_dir}/test.pid
+  port 6380
+  timeout 300
+  save 900 1
+  save 300 10
+  save 60 10000
+  dbfilename test.rdb
+  dir #{redis_dir}
+  loglevel warning
+  logfile stdout
+  databases 1
+CONF
+
+%x(echo '#{REDIS_CONFIG}' > #{redis_dir}/test.conf)
+redis_command = "redis-server #{redis_dir}/test.conf"
+%x[ #{redis_command} ]
+##
+
+## Configuring sidekiq
+options = {
+  logger: nil,
+  redis: { :url => "redis://localhost:6380/0" }
+}
+
+Sidekiq.configure_client do |config|
+  options.each do |option, value|
+    config.send("#{option}=", value)
+  end
+end
+
+Sidekiq.configure_server do |config|
+  options.each do |option, value|
+    config.send("#{option}=", value)
+  end
+end
+##
+
+## Configuring simplecov
 require 'simplecov'
 
 SimpleCov.start do
@@ -6,3 +54,43 @@ SimpleCov.start do
 end
 
 require File.expand_path("../../lib/sidekiq-rate-limiter", __FILE__)
+##
+
+## Hook to set Sidekiq::Testing mode using rspec tags
+RSpec.configure do |config|
+  config.before(:each) do
+    ## Use metadata to determine testing behavior
+    ## for queuing.
+
+    case example.metadata[:queuing].to_s
+    when 'enable', 'enabled', 'on', 'true'
+      Sidekiq::Testing.disable!
+    when 'fake', 'mock'
+      Sidekiq::Testing.fake!
+    when 'inline'
+      Sidekiq::Testing.inline!
+    else
+      defined?(Redis::Connection::Memory) ?
+        Sidekiq::Testing.disable! : Sidekiq::Testing.inline!
+    end
+
+    if Sidekiq::Testing.disabled?
+      Sidekiq.redis { |conn| conn.flushdb }
+    elsif Sidekiq::Testing.fake?
+      Sidekiq::Worker.clear_all
+    end
+
+  end
+
+  config.after(:all) do
+    ## Stopping Redis
+    ps = %x(ps -A -o pid,command | grep '#{redis_command}' | grep -v grep).split($/)
+    pids = ps.map { |p| p.split(/\s+/).reject(&:empty?).first.to_i }
+    pids.each { |pid| Process.kill("TERM", pid) }
+
+    ## Cleaning up
+    sleep 0.1
+    %x(rm -rf #{redis_dir}/*)
+  end
+end
+##

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-rate-limiter
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.0.1
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-01 00:00:00.000000000 Z
+date: 2013-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pry
@@ -76,6 +76,38 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov-rcov
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
@@ -92,6 +124,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis_rate_limiter
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Rate-limit Sidekiq fetches by worker class
 email:
 - bwthomas@gmail.com
@@ -108,9 +156,14 @@ files:
 - README.md
 - Rakefile
 - lib/sidekiq-rate-limiter.rb
+- lib/sidekiq-rate-limiter/fetch.rb
+- lib/sidekiq-rate-limiter/server.rb
 - lib/sidekiq-rate-limiter/version.rb
 - sidekiq-rate-limiter.gemspec
+- spec/sidekiq-rate-limiter/fetch_spec.rb
+- spec/sidekiq-rate-limiter/server_spec.rb
 - spec/spec_helper.rb
+- spec/support/redis/.keep
 homepage: https://github.com/enova/sidekiq-rate-limiter
 licenses:
 - MIT
@@ -137,5 +190,8 @@ signing_key:
 specification_version: 3
 summary: Rate-limit Sidekiq fetches by worker class
 test_files:
+- spec/sidekiq-rate-limiter/fetch_spec.rb
+- spec/sidekiq-rate-limiter/server_spec.rb
 - spec/spec_helper.rb
+- spec/support/redis/.keep
 has_rdoc: