sidekiq-dynamic-throttle 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9b7fbc2235c707b7b851a12b9e03bf1b3db1cbaa
+  data.tar.gz: 080520421f62a90964903da61665b75e5b72c3ac
+SHA512:
+  metadata.gz: e7e73a1a531249a84ab300f9e88d75f27b2c2b4c11b64ee954274bc80ffec007c1c384805440ff77de5e0378de1c7773380424d1943a564739ca95ffc74bbd2e
+  data.tar.gz: 61216980ba1bb3de1e4c640f063e3252888da87c17dbb554cef53220f14498d871550861ebdb8e4b42a9de7d6da270f24bcee6fe28794cfe16d91f96ea29ca26

data/.gitignore

@@ -0,0 +1,7 @@
+.bundle/
+*.gem
+Gemfile.lock
+pkg/
+coverage/
+*.swp
+spec/support/redis/

data/.rspec

@@ -0,0 +1 @@
+--color

data/.simplecov

@@ -0,0 +1,12 @@
+unless ENV['COVERAGE'] == 'false'
+  require 'simplecov-rcov'
+
+  class SimpleCov::Formatter::MergedFormatter
+    def format(result)
+      SimpleCov::Formatter::HTMLFormatter.new.format(result)
+      SimpleCov::Formatter::RcovFormatter.new.format(result)
+    end
+  end
+
+  SimpleCov.formatter = SimpleCov::Formatter::MergedFormatter
+end

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.0.0
+  - 1.9.3

data/CHANGELOG.md

@@ -0,0 +1,11 @@
+## 0.1.0 (May 30, 2014)
+
+* Support callables for all configuration options
+
+  *Kevin Yank*
+
+## 0.0.1 (December 3, 2013)
+
+* Initial release.
+
+  *Blake Thomas*

data/Gemfile

@@ -0,0 +1,2 @@
+source "http://rubygems.org"
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Enova
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,62 @@
+Sidekiq Dynamic Throttle
+====================
+
+Throttle fetching from queues dynamically.
+
+## Compatibility
+
+sidekiq-rate-limiter works by using a custom fetch class, the class responsible
+for pulling work from the queue stored in redis. Consequently you'll want to be
+careful about using other gems that use a same strategy, [sidekiq-priority](https://github.com/socialpandas/sidekiq-priority)
+being one example.
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'sidekiq-dynamic-throttle'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install sidekiq-dynamic-throttle
+
+## Configuration
+
+See [server.rb](lib/sidekiq-dynamic-throttle/server.rb) for an example of how to
+configure sidekiq-rate-limiter. Alternatively you can add the following to your
+initializer or what-have-you:
+
+```ruby
+require 'sidekiq-dynamic-throttle/server'
+```
+
+Or, if you prefer, amend your Gemfile like so:
+
+    gem 'sidekiq-dynamic-throttle', :require => 'sidekiq-dynamic-throttle/server'
+
+## Motivation
+
+Sidekiq::Throttler is great for smaller quantities of jobs, but falls down a bit
+for larger queues (see [issue #8](https://github.com/gevans/sidekiq-throttler/issues/8)). In addition, jobs that are
+limited multiple times are counted as 'processed' each time, so the stats balloon quickly.
+
+[Sidekiq::RateLimiter](https://github.com/enova/sidekiq-rate-limiter) works well when you've got consistent job processing limits, but can be a bit restrictive when throttling needs to be based on something that is more dynamic (i.e. API limits).
+
+## Contributing
+
+1. Fork
+2. Commit
+5. Pull Request
+
+## License
+
+MIT. See [LICENSE](LICENSE) for details.
+
+## Credits
+
+This was originally forked from [sidekiq-rate-limiter](https://github.com/enova/sidekiq-rate-limiter).

data/Rakefile

@@ -0,0 +1,11 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new
+task :default => :spec
+
+desc 'Start IRB with preloaded environment'
+task :console do
+  exec 'irb', "-I#{File.join(File.dirname(__FILE__), 'lib')}", '-rsidekiq-rate-limiter'
+end

data/lib/sidekiq-dynamic-throttle/fetch.rb

@@ -0,0 +1,19 @@
+module Sidekiq::DynamicThrottle
+  class Fetch < Sidekiq::BasicFetch
+    def retrieve_work
+      work = Sidekiq.redis { |conn| conn.brpop(*unthrottled_queues) }
+      UnitOfWork.new(*work) if work
+    end
+
+    private
+
+    def unthrottled_queues
+      queues_cmd.reject do |queue|
+        throttles = Sidekiq::DynamicThrottle.throttles[queue]
+        next if throttles.blank?
+
+        throttles.find { |throttle| throttle.throttled? }
+      end
+    end
+  end
+end

data/lib/sidekiq-dynamic-throttle/server.rb

@@ -0,0 +1,6 @@
+require 'sidekiq-rate-limiter/version'
+require 'sidekiq-rate-limiter/fetch'
+
+Sidekiq.configure_server do |config|
+  Sidekiq.options[:fetch] = Sidekiq::DynamicThrottle::Fetch
+end

data/lib/sidekiq-dynamic-throttle/version.rb

@@ -0,0 +1,5 @@
+module Sidekiq
+  module DynamicThrottle
+    VERSION = '0.1.0'
+  end
+end

data/lib/sidekiq-dynamic-throttle.rb

@@ -0,0 +1,15 @@
+require 'celluloid'
+require 'sidekiq/fetch'
+
+require 'sidekiq-dynamic-throttle/version'
+require 'sidekiq-dynamic-throttle/fetch'
+
+module Sidekiq::DynamicThrottle
+  def self.throttles
+    @throttles ||= Hash.new { |hash, key| hash[key] = [] }
+  end
+
+  def self.register_throttle(queue, throttle)
+    throttles['queue:' + queue.to_s] << throttle
+  end
+end

data/sidekiq-dynamic-throttle.gemspec

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+
+$:.push File.expand_path '../lib', __FILE__
+require 'sidekiq-dynamic-throttle/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'sidekiq-dynamic-throttle'
+  s.license     = 'MIT'
+  s.version     = Sidekiq::DynamicThrottle::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ['Bob Breznak', 'Blake Thomas', 'Enova']
+  s.email       = ['bob.breznak@gmail.com', 'bwthomas@gmail.com']
+  s.homepage    = 'https://github.com/bobbrez/sidekiq-dynamic-throttle'
+  s.summary     = 'Worker backed throttling for job processing'
+  s.description = 'Worker backed throttling for job processing'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ['lib']
+
+  s.add_development_dependency 'pry'
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'rspec'
+  s.add_development_dependency 'simplecov'
+  s.add_development_dependency 'simplecov-rcov'
+
+  s.add_dependency 'redis'
+  s.add_dependency 'sidekiq', '>= 2.0', '< 4.0'
+end

data/spec/sidekiq-rate-limiter/fetch_spec.rb

@@ -0,0 +1,86 @@
+require 'spec_helper'
+require 'sidekiq'
+require 'sidekiq/api'
+
+describe Sidekiq::RateLimiter::Fetch do
+  before(:all) do
+    class Job
+      include Sidekiq::Worker
+      sidekiq_options 'queue'    => 'basic',
+                      'retry'    => false,
+                      'rate' => {
+                          'limit'  => 1,
+                          'period' => 1
+                      }
+      def perform(*args); end
+    end
+    class ProcJob
+      include Sidekiq::Worker
+      sidekiq_options 'queue'    => 'basic',
+                      'retry'    => false,
+                      'rate' => {
+                          'limit'  => ->(arg1, arg2) { arg2 },
+                          'name'   => ->(arg1, arg2) { arg2 },
+                          'period' => ->(arg1, arg2) { arg2 }
+                      }
+      def perform(arg1, arg2); end
+    end
+  end
+
+  let(:options)       { { queues: [queue, another_queue, another_queue] } }
+  let(:queue)         { 'basic' }
+  let(:another_queue) { 'some_other_queue' }
+  let(:args)          { ['I am some args'] }
+  let(:worker)        { Job }
+  let(:proc_worker)   { ProcJob }
+  let(:redis_class)   { Sidekiq.redis { |conn| conn.class } }
+
+  it 'should inherit from Sidekiq::BasicFetch' do
+    described_class.should < Sidekiq::BasicFetch
+  end
+
+  it 'should retrieve work with strict setting' do
+    fetch = described_class.new options.merge(:strict => true)
+    fetch.queues_cmd.should eql(["queue:#{queue}", "queue:#{another_queue}", 1])
+  end
+
+  it 'should retrieve work', queuing: true do
+    worker.perform_async(*args)
+    fetch = described_class.new(options)
+    work    = fetch.retrieve_work
+    parsed  = JSON.parse(work.message)
+
+    work.should_not be_nil
+    work.queue_name.should eql(queue)
+    work.acknowledge.should be_nil
+
+    parsed.should include(worker.get_sidekiq_options)
+    parsed.should include("class" => worker.to_s, "args" => args)
+    parsed.should include("jid", "enqueued_at")
+
+    q = Sidekiq::Queue.new(queue)
+    q.size.should == 0
+  end
+
+  it 'should place rate-limited work at the back of the queue', queuing: true do
+    worker.perform_async(*args)
+    Sidekiq::RateLimiter::Limit.any_instance.should_receive(:exceeded?).and_return(true)
+    redis_class.any_instance.should_receive(:lpush).exactly(:once).and_call_original
+
+    fetch = described_class.new(options)
+    fetch.retrieve_work.should be_nil
+
+    q = Sidekiq::Queue.new(queue)
+    q.size.should == 1
+  end
+
+  it 'should accept procs for limit, name, and period config keys', queuing: true do
+    proc_worker.perform_async(1,2)
+
+    Sidekiq::RateLimiter::Limit.should_receive(:new).with(anything(), {:limit => 2, :interval => 2, :name => "2"}).and_call_original
+
+    fetch = described_class.new(options)
+    work   = fetch.retrieve_work
+  end
+
+end

data/spec/sidekiq-rate-limiter/server_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe Sidekiq::RateLimiter, 'server configuration' do
+  before do
+    Sidekiq.stub(:server? => true)
+    require 'sidekiq-rate-limiter/server'
+  end
+
+  it 'should set Sidekiq.options[:fetch] as desired' do
+    Sidekiq.configure_server do |config|
+      Sidekiq.options[:fetch].should eql(Sidekiq::RateLimiter::Fetch)
+    end
+  end
+
+  it 'should inherit from Sidekiq::BasicFetch' do
+    Sidekiq.configure_server do |config|
+      Sidekiq.options[:fetch].should < Sidekiq::BasicFetch
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,96 @@
+require 'sidekiq'
+require 'sidekiq/testing'
+
+## Confirming presence of redis server executable
+abort "## `redis-server` not in path" if %x(which redis-server).empty?
+redis_dir = "#{File.dirname(__FILE__)}/support/redis"
+
+## Redis configuration
+REDIS_CONFIG = <<-CONF
+  daemonize yes
+  pidfile #{redis_dir}/test.pid
+  port 6380
+  timeout 300
+  save 900 1
+  save 300 10
+  save 60 10000
+  dbfilename test.rdb
+  dir #{redis_dir}
+  loglevel warning
+  logfile stdout
+  databases 1
+CONF
+
+%x(echo '#{REDIS_CONFIG}' > #{redis_dir}/test.conf)
+redis_command = "redis-server #{redis_dir}/test.conf"
+%x[ #{redis_command} ]
+##
+
+## Configuring sidekiq
+options = {
+  logger: nil,
+  redis: { :url => "redis://localhost:6380/0" }
+}
+
+Sidekiq.configure_client do |config|
+  options.each do |option, value|
+    config.send("#{option}=", value)
+  end
+end
+
+Sidekiq.configure_server do |config|
+  options.each do |option, value|
+    config.send("#{option}=", value)
+  end
+end
+##
+
+## Configuring simplecov
+require 'simplecov'
+
+SimpleCov.start do
+  add_filter "vendor"
+  add_filter "spec"
+end
+
+require File.expand_path("../../lib/sidekiq-rate-limiter", __FILE__)
+##
+
+## Hook to set Sidekiq::Testing mode using rspec tags
+RSpec.configure do |config|
+  config.before(:each) do
+    ## Use metadata to determine testing behavior
+    ## for queuing.
+
+    case example.metadata[:queuing].to_s
+    when 'enable', 'enabled', 'on', 'true'
+      Sidekiq::Testing.disable!
+    when 'fake', 'mock'
+      Sidekiq::Testing.fake!
+    when 'inline'
+      Sidekiq::Testing.inline!
+    else
+      defined?(Redis::Connection::Memory) ?
+        Sidekiq::Testing.disable! : Sidekiq::Testing.inline!
+    end
+
+    if Sidekiq::Testing.disabled?
+      Sidekiq.redis { |conn| conn.flushdb }
+    elsif Sidekiq::Testing.fake?
+      Sidekiq::Worker.clear_all
+    end
+
+  end
+
+  config.after(:all) do
+    ## Stopping Redis
+    ps = %x(ps -A -o pid,command | grep '#{redis_command}' | grep -v grep).split($/)
+    pids = ps.map { |p| p.split(/\s+/).reject(&:empty?).first.to_i }
+    pids.each { |pid| Process.kill("TERM", pid) }
+
+    ## Cleaning up
+    sleep 0.1
+    %x(rm -rf #{redis_dir}/*)
+  end
+end
+##

metadata

@@ -0,0 +1,174 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-dynamic-throttle
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Bob Breznak
+- Blake Thomas
+- Enova
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-01-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov-rcov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+description: Worker backed throttling for job processing
+email:
+- bob.breznak@gmail.com
+- bwthomas@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".simplecov"
+- ".travis.yml"
+- CHANGELOG.md
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/sidekiq-dynamic-throttle.rb
+- lib/sidekiq-dynamic-throttle/fetch.rb
+- lib/sidekiq-dynamic-throttle/server.rb
+- lib/sidekiq-dynamic-throttle/version.rb
+- sidekiq-dynamic-throttle.gemspec
+- spec/sidekiq-rate-limiter/fetch_spec.rb
+- spec/sidekiq-rate-limiter/server_spec.rb
+- spec/spec_helper.rb
+- spec/support/redis/.keep
+homepage: https://github.com/bobbrez/sidekiq-dynamic-throttle
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Worker backed throttling for job processing
+test_files:
+- spec/sidekiq-rate-limiter/fetch_spec.rb
+- spec/sidekiq-rate-limiter/server_spec.rb
+- spec/spec_helper.rb
+- spec/support/redis/.keep
+has_rdoc: