resque-throttler 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6cc4d7ea16ce3914cd16f8c6bdc42bb43d391d3d
+  data.tar.gz: 1315df0567f36ee68e14b4daf16f756552c871bf
+SHA512:
+  metadata.gz: 03d5c6177b891b30cab5b4ebe9b5e31de08db32790aa2c9c132fcac835c55a8734a843b31720f915536142d73e6ed620e027fa08138d3e428188df7f7677c0e1
+  data.tar.gz: c1b0db2f23672b7e7566c64199dca70364ffc48342ed1922984c1739993a77940e79062e416c581f0ecc9f24d4913287045baea7f26f384097f3ab408751fc61

data/.gitignore

@@ -0,0 +1,34 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in sunstone.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,66 @@
+PATH
+  remote: .
+  specs:
+    resque-throttler (0.1.0)
+      resque (~> 1.25)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.1.4)
+      i18n (~> 0.6, >= 0.6.9)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.1)
+      tzinfo (~> 1.1)
+    ansi (1.4.3)
+    builder (3.2.2)
+    i18n (0.6.11)
+    json (1.8.1)
+    metaclass (0.0.4)
+    minitest (5.4.0)
+    minitest-reporters (1.0.4)
+      ansi
+      builder
+      minitest (>= 5.0)
+      ruby-progressbar
+    mocha (1.1.0)
+      metaclass (~> 0.0.1)
+    mono_logger (1.1.0)
+    multi_json (1.10.1)
+    rack (1.5.2)
+    rack-protection (1.5.3)
+      rack
+    rake (10.3.2)
+    redis (3.0.7)
+    redis-namespace (1.5.0)
+      redis (~> 3.0, >= 3.0.4)
+    resque (1.25.2)
+      mono_logger (~> 1.0)
+      multi_json (~> 1.0)
+      redis-namespace (~> 1.3)
+      sinatra (>= 0.9.2)
+      vegas (~> 0.1.2)
+    ruby-progressbar (1.5.1)
+    sinatra (1.4.5)
+      rack (~> 1.4)
+      rack-protection (~> 1.4)
+      tilt (~> 1.3, >= 1.3.4)
+    thread_safe (0.3.4)
+    tilt (1.4.1)
+    tzinfo (1.2.1)
+      thread_safe (~> 0.1)
+    vegas (0.1.11)
+      rack (>= 1.0.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport
+  bundler
+  minitest
+  minitest-reporters
+  mocha
+  rake
+  resque-throttler!

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Jon Bracy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,30 @@
+Resque Throttler
+================
+
+Resque Throttler allows you to throttle the rate at which jobs are performed
+on a specific queue.
+
+
+Installation
+============
+
+```ruby
+require 'resque/throttler'
+```
+
+Or in a Gemfile:
+
+```ruby
+require 'resque-throttler', :require => 'resque/throttler'
+```
+
+Usage
+=====
+
+```ruby
+require 'resque'
+require 'resque/throttler'
+
+# Rate limit at 10 jobs from `my_queue` per minute
+Resque.rate_limit(:my_queue, :at => 10, :per => 60)
+```

data/Rakefile

@@ -0,0 +1,32 @@
+require 'bundler/setup'
+require "bundler/gem_tasks"
+Bundler.require(:development)
+require 'rake/testtask'
+require 'rdoc/task'
+
+Rake::TestTask.new do |t|
+    t.libs << 'lib' << 'test'
+    t.test_files = FileList['test/**/*_test.rb']
+    #t.warning = true
+    #t.verbose = true
+end
+
+Rake::RDocTask.new do |rd|
+  rd.main = 'README.md'
+  rd.title = 'Sunstone Documentation'
+  rd.rdoc_dir = 'doc'
+  
+  rd.options << '-f' << 'sdoc'
+  rd.options << '-T' << '42floors'
+  rd.options << '-g' # Generate github links
+  
+  rd.rdoc_files.include('README.rdoc')
+  rd.rdoc_files.include('lib/**/*.rb')
+end
+
+desc "Run tests"
+task :default => :test
+
+namespace :pages do
+  #TODO: https://github.com/defunkt/sdoc-helpers/blob/master/lib/sdoc_helpers/pages.rb
+end

data/lib/resque/throttler.rb

@@ -0,0 +1,86 @@
+require 'resque'
+require 'securerandom'
+
+module Resque::Plugins
+  module Throttler
+    extend self
+  
+    def self.extended(other)
+      other.instance_variable_set(:@rate_limits, {})
+    end
+  
+    def pop(queue)
+      if queue_at_or_over_rate_limit?(queue)
+        gc_rate_limit_data_for_queue(queue)
+        nil
+      else
+        super
+      end
+    end
+
+    def rate_limit(queue, options={})
+      if options.keys.sort != [:at, :per]
+        raise ArgumentError.new("Mising either :at or :per in options") 
+      end
+    
+      @rate_limits[queue.to_s] = options
+    end
+  
+    def rate_limit_for(queue)
+      @rate_limits[queue.to_s]
+    end
+  
+    def queue_rate_limited?(queue)
+      @rate_limits[queue.to_s]
+    end
+  
+    def queue_at_or_over_rate_limit?(queue)
+      if queue_rate_limited?(queue)
+        redis.scard("throttler:#{queue}_uuids") >= rate_limit_for(queue)[:at]
+      else
+        false
+      end
+    end
+  
+    def gc_rate_limit_data_for_queue(queue)
+      return unless queue_rate_limited?(queue)
+    
+      limit = rate_limit_for(queue)
+      queue_key = "throttler:#{queue}_uuids"
+      uuids = redis.smembers(queue_key)
+    
+      uuids.each do |uuid|
+        job_ended_at = redis.hmget("throttler:jobs:#{uuid}", "ended_at")[0]
+        if job_ended_at && Time.at(job_ended_at.to_i) < Time.now - limit[:per]
+          redis.srem(queue_key, uuid)
+          redis.del("throttler:jobs:#{uuid}")
+        end
+      end
+    end
+    
+  end
+end
+
+Resque.extend(Resque::Plugins::Throttler)
+
+class Resque::Job
+  
+  def perform_with_throttler
+    if Resque.queue_rate_limited?(self.queue)
+      uuid = SecureRandom.uuid
+      begin
+        # TODO this needs to be wrapped in a transcation
+        redis.hmset("throttler:jobs:#{uuid}", "started_at", Time.now.to_i)
+        redis.sadd("throttler:#{queue}_uuids", uuid)
+        perform_without_throttler
+      ensure
+        redis.hmset("throttler:jobs:#{uuid}", "ended_at", Time.now.to_i)
+      end
+    else
+      perform_without_throttler
+    end
+  end
+  alias_method :perform_without_throttler, :perform
+  alias_method :perform, :perform_with_throttler
+  
+end

data/resque-throttler.gemspec

@@ -0,0 +1,31 @@
+Gem::Specification.new do |s|
+  s.name        = "resque-throttler"
+  s.version     = '0.1.1'
+  s.licenses    = ['MIT']
+  s.authors     = ["Jon Bracy"]
+  s.email       = ["jonbracy@gmail.com"]
+  s.homepage    = "https://github.com/malomalo/resque-throttler"
+  s.summary     = %q{Rate limit Resque Jobs}
+  s.description = %q{Rate limit how many times a job can be run from a queue}
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.extensions    = []
+  s.require_paths = ["lib"]
+  #s.extra_rdoc_files = ["LICENSE", "README.md"]
+  
+  # Developoment 
+  s.add_development_dependency 'rake'
+  #s.add_development_dependency 'rdoc'
+  #s.add_development_dependency 'sdoc'
+  s.add_development_dependency 'bundler'
+  s.add_development_dependency 'activesupport'
+  s.add_development_dependency 'minitest'
+  s.add_development_dependency 'minitest-reporters'
+  s.add_development_dependency 'mocha'
+  #s.add_development_dependency 'sdoc-templates-42floors'
+
+  # Runtime
+  s.add_runtime_dependency 'resque', '~> 1.25'
+end

data/test/resque/job_test.rb

@@ -0,0 +1,77 @@
+require 'test_helper'
+
+class MyJob
+  def self.perform
+  end
+end
+
+class MyErrorJob
+  def self.perform
+    raise ArgumentError
+  end
+end
+
+class Resque::JobTest < Minitest::Test
+
+  def setup
+    Resque.instance_variable_set(:@rate_limits, {})
+  end
+
+  test "Resque::Job::perform on unthrottled job" do
+    Resque.rate_limit(:myqueue, :at => 10, :per => 1)
+
+    job = Resque::Job.new(:other_queue, {
+      'class' => 'MyJob',
+      'args'  => []
+    })
+    
+    travel_to Time.now do
+      SecureRandom.expects(:uuid).returns("jobuuid").never
+      Resque.redis.expects(:hmset).with("throttler:jobs:jobuuid", "started_at", Time.now.to_i).never
+      Resque.redis.expects(:sadd).with("throttler:myqueue_uuids", "jobuuid").never
+      Resque.redis.expects(:hmset).with("throttler:jobs:jobuuid", "ended_at", Time.now.to_i).never
+      
+      job.perform
+    end
+  end
+    
+  test "Resque::Job::perform on throttled job" do
+    Resque.rate_limit(:myqueue, :at => 10, :per => 1)
+
+    job = Resque::Job.new(:myqueue, {
+      'class' => 'MyJob',
+      'args'  => []
+    })
+    
+    travel_to Time.now do
+      SecureRandom.expects(:uuid).returns("jobuuid")
+      Resque.redis.expects(:hmset).with("throttler:jobs:jobuuid", "started_at", Time.now.to_i).once
+      Resque.redis.expects(:sadd).with("throttler:myqueue_uuids", "jobuuid").once
+      Resque.redis.expects(:hmset).with("throttler:jobs:jobuuid", "ended_at", Time.now.to_i).once
+      
+      job.perform
+    end
+  end
+  
+  test "Resque::Job::perform on throttled job with job that throws error" do
+    Resque.rate_limit(:myqueue, :at => 10, :per => 1)
+
+    job = Resque::Job.new('myqueue', {
+      'class' => 'MyErrorJob',
+      'args'  => []
+    })
+
+    travel_to Time.now do
+      SecureRandom.expects(:uuid).returns("jobuuid")
+      Resque.redis.expects(:hmset).with("throttler:jobs:jobuuid", "started_at", Time.now.to_i).once
+      Resque.redis.expects(:sadd).with("throttler:myqueue_uuids", "jobuuid").once
+      Resque.redis.expects(:hmset).with("throttler:jobs:jobuuid", "ended_at", Time.now.to_i).once
+
+      assert_raises(ArgumentError) {
+        job.perform
+      }
+    end
+  end
+  
+end
+

data/test/resque_test.rb

@@ -0,0 +1,78 @@
+require 'test_helper'
+
+class ResqueTest < Minitest::Test
+
+  def setup
+    Resque.instance_variable_set(:@rate_limits, {})
+  end
+  
+  test "Resque::rate_limit" do
+    Resque.rate_limit(:myqueue, :at => 10, :per => 1)
+
+    assert_equal Resque.instance_variable_get(:@rate_limits), {
+      'myqueue' => {:at => 10, :per => 1}
+    }
+  end
+  
+  test "Resque::queue_rate_limited?" do
+    Resque.rate_limit(:myqueue, :at => 10, :per => 1)
+    
+    assert Resque.queue_rate_limited?(:myqueue)
+    assert Resque.queue_rate_limited?("myqueue")
+  end
+  
+  test "Resque::queue_at_or_over_rate_limit?" do
+    Resque.rate_limit(:myqueue, :at => 10, :per => 1)
+
+    Resque.redis.expects(:scard).with("throttler:myqueue_uuids").returns(5).twice
+    assert !Resque.queue_at_or_over_rate_limit?(:myqueue)
+    assert !Resque.queue_at_or_over_rate_limit?("myqueue")
+        
+    Resque.redis.expects(:scard).with("throttler:myqueue_uuids").returns(10).twice
+    assert Resque.queue_at_or_over_rate_limit?(:myqueue)
+    assert Resque.queue_at_or_over_rate_limit?("myqueue")
+  end
+
+  test "Resque::pop pops on unthrottled queues" do
+    Resque.redis.expects(:lpop).returns(nil)
+    
+    Resque.pop('myqueue')
+  end
+  
+  test "Resque::pop skips over queues that are at or over their limit" do
+    Resque.rate_limit(:myqueue, :at => 10, :per => 1)
+    Resque.expects(:queue_at_or_over_rate_limit?).with("myqueue").returns(true)
+    Resque.redis.expects(:lpop).never
+    
+    Resque.pop('myqueue')
+  end
+
+  
+  test "Resque::pop gc's the limit data after skipping over a throttled queue" do
+    Resque.rate_limit(:myqueue, :at => 10, :per => 1)
+    Resque.expects(:queue_at_or_over_rate_limit?).with("myqueue").returns(true)
+    Resque.expects(:gc_rate_limit_data_for_queue).with("myqueue").once
+    
+    Resque.pop('myqueue')
+  end
+  
+  test "Resque::gc_rate_limit_data_for_queue" do
+    Resque.rate_limit(:myqueue, :at => 10, :per => 5)
+    Resque.redis.expects(:smembers).with("throttler:myqueue_uuids").returns(["1","2","3"]).once
+    Resque.redis.expects(:srem).with("throttler:myqueue_uuids", "1").once
+    Resque.redis.expects(:del).with("throttler:jobs:1").once
+    
+    travel_to Time.now do
+      Resque.redis.expects(:hmget).with("throttler:jobs:1", "ended_at").returns([(Time.now - 10).to_i])
+      Resque.redis.expects(:hmget).with("throttler:jobs:2", "ended_at").returns([(Time.now - 3).to_i])
+      Resque.redis.expects(:hmget).with("throttler:jobs:3", "ended_at").returns([nil])
+    
+      Resque.gc_rate_limit_data_for_queue('myqueue')
+    end
+  end
+  
+  test "Resque::gc_rate_limit_data_for_queue for unthrottled queue" do
+    Resque.gc_rate_limit_data_for_queue('myqueue')
+  end
+  
+end

data/test/test_helper.rb

@@ -0,0 +1,54 @@
+# To make testing/debugging easier, test within this source tree versus an
+# installed gem
+dir = File.dirname(__FILE__)
+root = File.expand_path(File.join(dir, '..'))
+lib = File.expand_path(File.join(root, 'lib'))
+
+$LOAD_PATH << lib
+
+require 'resque'
+require 'resque/throttler'
+require "minitest/autorun"
+require 'minitest/unit'
+require 'minitest/reporters'
+require "mocha"
+require "mocha/mini_test"
+require 'active_support/testing/time_helpers'
+
+Minitest::Reporters.use! Minitest::Reporters::SpecReporter.new
+
+# File 'lib/active_support/testing/declarative.rb', somewhere in rails....
+class Minitest::Test
+  
+  include ActiveSupport::Testing::TimeHelpers
+  
+  def self.test(name, &block)
+    test_name = "test_#{name.gsub(/\s+/,'_')}".to_sym
+    defined = instance_method(test_name) rescue false
+    raise "#{test_name} is already defined in #{self}" if defined
+    if block_given?
+      define_method(test_name, &block)
+    else
+      define_method(test_name) do
+        flunk "No implementation provided for #{name}"
+      end
+    end
+  end
+  
+  # test/unit backwards compatibility methods
+  alias :assert_raise :assert_raises
+  alias :assert_not_empty :refute_empty
+  alias :assert_not_equal :refute_equal
+  alias :assert_not_in_delta :refute_in_delta
+  alias :assert_not_in_epsilon :refute_in_epsilon
+  alias :assert_not_includes :refute_includes
+  alias :assert_not_instance_of :refute_instance_of
+  alias :assert_not_kind_of :refute_kind_of
+  alias :assert_no_match :refute_match
+  alias :assert_not_nil :refute_nil
+  alias :assert_not_operator :refute_operator
+  alias :assert_not_predicate :refute_predicate
+  alias :assert_not_respond_to :refute_respond_to
+  alias :assert_not_same :refute_same
+  
+end

metadata

@@ -0,0 +1,157 @@
+--- !ruby/object:Gem::Specification
+name: resque-throttler
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Jon Bracy
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-08-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest-reporters
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: resque
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.25'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.25'
+description: Rate limit how many times a job can be run from a queue
+email:
+- jonbracy@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- lib/resque/throttler.rb
+- resque-throttler.gemspec
+- test/resque/job_test.rb
+- test/resque_test.rb
+- test/test_helper.rb
+homepage: https://github.com/malomalo/resque-throttler
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Rate limit Resque Jobs
+test_files:
+- test/resque/job_test.rb
+- test/resque_test.rb
+- test/test_helper.rb
+has_rdoc: