resque-rate_limited 1.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1d7f4298346cb2c327ba603fd644953c89397d0d
+  data.tar.gz: b860c30f7289da40aa619cbb66ec59ea3f6afece
+SHA512:
+  metadata.gz: c5ce4337150d6b28bfe5ff6c202d187eaa0e34323b66a9d6789be3c8834b220839f49aa51486d243d21a73c42f11f085270ebe6a25d0628e5f5a1ece1d160da1
+  data.tar.gz: a2cb553a6a1856f3cece469a99451b4faa923e0ef5278cc03881b07479f513b196e964cfb7df12bf436a9c2c3b5dd19563c49ff78276cf8e655f7376dfc476bf

data/.gitignore

@@ -0,0 +1,16 @@
+.ruby-gemset
+.ruby-version
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.hound.yml

@@ -0,0 +1,4 @@
+---
+ruby:
+  enabled: true
+  config_file: .rubocop.yml

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/.rubocop.yml

@@ -0,0 +1,46 @@
+---
+AllCops:
+  Exclude:
+    - 'tmp/**/*'
+    - 'coverage/**/*'
+    - 'spec/dummy/**/*'
+
+#-------------------------------------------------------------------------------
+# Project standards
+#-------------------------------------------------------------------------------
+StringLiterals:
+  EnforcedStyle: single_quotes
+  Enabled: true
+
+DotPosition:
+  Description: 'Checks the position of the dot in multi-line method calls.'
+  EnforcedStyle: leading
+  Enabled: true
+
+Documentation:
+  Description: 'Document classes and non-namespace modules.'
+  Enabled: false
+
+FileName:
+  Description: 'Use snake_case for source file names.'
+  Enabled: true
+
+Style/SymbolArray:
+  Description: 'Use %i or %I for arrays of symbols.'
+  StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#percent-i'
+  Enabled: false # Only available in Ruby 2.0+
+
+Style/ExtraSpacing:
+  Description: 'Do not use unnecessary spacing.'
+  Enabled: true
+
+Lint/LiteralInInterpolation:
+  Description: 'Avoid interpolating literals in strings'
+  AutoCorrect: true
+
+#-------------------------------------------------------------------------------
+# These rules have been relaxed because of existing code
+# We should tighten these up over time
+#-------------------------------------------------------------------------------
+LineLength:
+  Max: 166 # project standard is 120

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in resque_rate_limited.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,16 @@
+guard :rubocop do
+  watch(/.+\.rb$/)
+  watch(%r{(?:.+/)?\.rubocop\.yml$}) { |m| File.dirname(m[0]) }
+end
+
+guard(
+  :rspec,
+  all_after_pass: true,
+  all_on_start: true,
+  cmd: 'NO_SIMPLECOV=true bundle exec rspec --fail-fast --format documentation'
+) do
+  watch(%r{spec/.+_spec\.rb$})
+  watch(%r{lib/(.+)\.rb$}) { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb') { 'spec' }
+  watch(%r{^spec/support/.+\.rb$}) { 'spec' }
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 pavoni
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,215 @@
+# Resque Rate Limited
+
+A Resque plugin which makes handling jobs that use rate limited apis easier
+
+If you have a series of jobs in a queue, this gem will pause the queue when one of the jobs hits a rate limit, and re-start the queue when the rate limit has expired.
+
+There are two ways to use the gem.
+
+If the api you are using has a dedicated queue included in the gem (currently Twitter, Angellist and Evernote) then you just need to make some very minor changes to how you queue jobs, and the gem will do the rest.
+
+If you are using another API, then you need to write a little code that catches the rate limit signal.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'resque-rate_limited'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install resque-rate_limited
+
+## Usage
+
+### Configuration
+#### Redis
+The gem uses [redis-mutex](https://github.com/kenn/redis-mutex ) which requires you to register the Redis server: (e.g. in `config/initializers/redis_mutex.rb` for Rails)
+
+```ruby
+RedisClassy.redis = Redis.new
+```
+Note that Redis Mutex uses the `redis-classy` gem internally.
+
+#### Un Pause
+Queues can be unpaused in two ways.
+
+The most elegant is using [resque-scheduler](https://github.com/resque/resque-scheduler), this works well as long as you aren't running on a platform like heroku which requires a dedicated worker to run the resque-scheduler.
+
+To tell the gem to use `resque-scheduler` you need to include it in your Gemfile - and also let the gem know which queue to use to schedule the unpause job (make sure this isn't a queue that could get paused). Put this in an initializer.
+
+```ruby
+Resque::Plugins::RateLimited::UnPause.queue = :my_queue
+```
+
+Please see the section below on how to unpause on heroku as an alternative. If you don't install `resque-scheduler` AND configure the queue, then the gem will not schedule unpause jobs this way.
+
+#### Workers
+Queues are paused by renaming them, so a resque queue called 'twitter\_api' will be renamed 'twitter\_api\_paused' when it hits a rate limit. Of course this will only work if your resque workers are not also taking jobs from the 'twitter\_api\_paused' queue. So your worker commands need to look like:
+
+Either
+```ruby
+bin/resque work --queues=high,low,twitter_api
+```
+or
+```ruby
+env QUEUES=high,low,twitter_api bundle exec rake jobs:work
+```
+
+NOT
+```ruby
+bin/resque work --queues=*
+```
+or NOT
+```ruby
+env QUEUES=* bundle exec rake jobs:work
+```
+
+#### Unpausing on heroku
+The built in schedler on heroku doesn't support dynamic scheduling from an API, so unless you want to provision an extra worker to run resque-scheduler - the best option is just to unpause all your queues on a regular basis. If they aren't paused this is a harmless no-op. If not enough time has elapsed the jobs will just hit the rate_limit and get paused again. We've found that a hourly 'rake unpause' job seems to work well. The rake task would need to call:
+
+```ruby
+Resque::Plugins::RateLimited.TwitterQueue.un_pause
+Resque::Plugins::RateLimited.AngellistQueue.un_pause
+MyQueue.un_pause
+MyJob.un_pause
+```
+### A pausable job using one of the build-in queues (Twitter, Angellist, Evernote)
+If you're using the [twitter gem](https://github.com/sferik/twitter), this is really simple. Instead of queuing using Resque.enqueue, you just use Resque::Plugins::RateLimited:TwitterQueue.enqueue.
+
+Make sure your code in perform doesn't catch the rate_limit exception.
+
+The TwitterQueue will catch the exception and pause the queue (as well as re-scheduling the jobs and scheduling an un pause (if you are using resque-scheduler)). Any jobs you add while the queue is paused will be added to the paused queue
+
+```ruby
+class TwitterJob
+  class << self
+    def refresh(handle)
+      Resque::Plugins::RateLimited:TwitterQueue.enqueue(TwitterJob, handle)
+    end
+
+    def perform(handle)
+      do_something_with_the_twitter_gem(handle)
+    end
+  end
+end
+```
+
+### A single class of pausable job using a new api
+If you only have one class of job you want to queue using the api, then you can use the PauseQueue module directly
+
+```ruby
+class MyApiJob
+  extend Resque::Plugins::RateLimited
+  @queue = :my_api
+  WAIT_TIME = 60*60
+
+  def self.perform(*params)
+    do_api_stuff
+  rescue MyApiRateLimit
+    pause_until(Time.now + WAIT_TIME, name)
+    rate_limited_requeue(self, *params)
+  end
+
+  def self.enqueue(*params)
+    rate_limited_enqueue(self, *params)
+  end
+end
+````
+
+### Multiple classes of pausable job using a new api
+If you have more than one class of job you want to queue to the api, then you need to add another Queue class. This isn't hard
+
+```ruby
+class MyApiQueue < Resque::Plugins::RateLimited::BaseApiQueue
+  @queue = :my_api
+  WAIT_TIME = 60*60
+
+  def self.perform(klass, *params)
+    super
+  rescue MyApiRateLimit
+    pause_until(Time.now + WAIT_TIME, name)
+    rate_limited_requeue(self, klass, *params)
+  end
+end
+````
+If you do this - please contribute - and I'll add to the gem.
+
+## Development Documentation
+All the functions are class methods
+
+```ruby
+rate_limited_enqueue(klass, *params)
+rate_limited_requeue(klass, *params)
+````
+Queue the job specified to the resque queue specified by `@queue`. `rate_limited_requeue` is intended for use when you need the job to be pushed back to the queue; there are two reasons to split this from `rate_limited_enqueue`. Firstly it makes testing with stubs easier - secondly there is a boundary condition when you need to requeue the last job in the queue.
+
+```ruby
+pause
+````
+Pauses the queue specified by `@queue`, if it is not already paused.
+In most cases you should call `pause_until` to pause a queue when you hit a rate limit.
+
+```ruby
+un_pause
+````
+Un-pauses the queue specified by `@queue`, if it is paused.
+
+```ruby
+pause_until(timestamp)
+````
+Pauses the queue (specified by `@queue`) and then queues a job to unpause the queue specified by `@queue`, using resque-scheduler to the queue specified by `Resque::Plugins::RateLimited::UnPause.queue` at the timestamp specified.
+If `resque-schedule` is not included, or `UnPause.queue` isn't specified this will just pause the queue.
+
+This is the prefered function to call when you hit a rate limit, since it with work regardless of the unpause method used by the application.
+
+```ruby
+paused?
+````
+This returns true or false to indicate wheher the queue is paused. Be aware that the queue state could change get after the call returns, but before your code executes. Use `with_lock` if you need to avoid this.
+
+```ruby
+paused_queue_name
+````
+Returns the name of the queue when it is paused.
+
+```ruby
+with_lock(&block)
+````
+Takes ownership of the PauseQueue semaphor before executing the block passed. Useful if you need to test the state of the queue and take some action without the state changing.
+
+```ruby
+find_class(klass)
+````
+Takes the parameter passed, and if it's a string class name, tries to turn it into a class.
+
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/resque_rate_limited/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+
+## Version history
+
+0.0.x Mostly pre-release versions
+
+1.0.0 First release version. Breaking change - renamed  `pause_for` to be `pause_until` to better reflect function
+
+1.1.0 Enqueues underlying job instead of directly performing it
+
+
+
+## Final thoughts
+Thanks to [Dominic](https://github.com/dominicsayers) for idea of renaming the redis key - and the sample code that does this.
+
+This is my first gem - so please forgive any errors - and feedback very welcome

data/Rakefile

@@ -0,0 +1 @@
+require 'bundler/gem_tasks'

data/lib/resque-rate_limited/version.rb

@@ -0,0 +1,3 @@
+module RateLimited
+  VERSION = '1.1.0'.freeze
+end

data/lib/resque/plugins/rate_limited/apis/angellist_queue.rb

@@ -0,0 +1,19 @@
+require 'angellist_api'
+
+module Resque
+  module Plugins
+    module RateLimited
+      class AngellistQueue < BaseApiQueue
+        WAIT_TIME = 60
+        @queue = :angellist_api
+
+        def self.perform(klass, *params)
+          super
+        rescue AngellistApi::Error::TooManyRequests
+          pause_until(Time.now + (60 * 60))
+          rate_limited_requeue(self, klass, *params)
+        end
+      end
+    end
+  end
+end

data/lib/resque/plugins/rate_limited/apis/base_api_queue.rb

@@ -0,0 +1,17 @@
+module Resque
+  module Plugins
+    module RateLimited
+      class BaseApiQueue
+        extend Resque::Plugins::RateLimited
+        def self.perform(klass, *params)
+          # find_class(klass).perform(*params)
+          Resque.enqueue_to @queue, klass, *params # Enqueue so that ResqueSolo can take effect
+        end
+
+        def self.enqueue(klass, *params)
+          rate_limited_enqueue(self, klass.to_s, *params)
+        end
+      end
+    end
+  end
+end

data/lib/resque/plugins/rate_limited/apis/evernote_queue.rb

@@ -0,0 +1,19 @@
+require 'evernote-thrift'
+
+module Resque
+  module Plugins
+    module RateLimited
+      class EvernoteQueue < BaseApiQueue
+        @queue = :evernote_api
+
+        def self.perform(klass, *params)
+          super
+        rescue Evernote::EDAM::Error::EDAMSystemException => e
+          raise unless e.errorCode == Evernote::EDAM::Error::EDAMErrorCode::RATE_LIMIT_REACHED
+          pause_until(Time.now + 60 * e.rateLimitDuration.seconds)
+          rate_limited_requeue(self, klass, *params)
+        end
+      end
+    end
+  end
+end

data/lib/resque/plugins/rate_limited/apis/twitter_queue.rb

@@ -0,0 +1,19 @@
+require 'twitter'
+
+module Resque
+  module Plugins
+    module RateLimited
+      class TwitterQueue < BaseApiQueue
+        @queue = :twitter_api
+
+        def self.perform(klass, *params)
+          super
+        rescue Twitter::Error::TooManyRequests,
+               Twitter::Error::EnhanceYourCalm => e
+          pause_until(Time.now + e.rate_limit.reset_in)
+          rate_limited_requeue(self, klass, *params)
+        end
+      end
+    end
+  end
+end

data/lib/resque/plugins/rate_limited/rate_limited.rb

@@ -0,0 +1,92 @@
+module Resque
+  module Plugins
+    module RateLimited
+      RESQUE_PREFIX = 'queue:'.freeze
+      MUTEX = 'Resque::Plugins::RateLimited'.freeze
+
+      def around_perform_with_check_and_requeue(*params)
+        paused = false
+        with_lock do
+          paused = paused?
+          Resque.enqueue_to(paused_queue_name, self, *params) if paused
+        end
+        return if paused
+        yield
+      end
+
+      def rate_limited_enqueue(klass, *params)
+        with_lock do
+          if paused?
+            Resque.enqueue_to(paused_queue_name, klass, *params)
+          else
+            Resque.enqueue_to(@queue, klass, *params)
+          end
+        end
+      end
+
+      def rate_limited_requeue(klass, *params)
+        # if the queue is empty, this was the last job - so queue to the paused queue
+        with_lock do
+          if paused?(true)
+            Resque.enqueue_to(paused_queue_name, klass, *params)
+          else
+            Resque.enqueue_to(@queue, klass, *params)
+          end
+        end
+      end
+
+      def pause_until(timestamp)
+        UnPause.enqueue(timestamp, name) if pause
+      end
+
+      def un_pause
+        Resque.redis.renamenx(RESQUE_PREFIX + paused_queue_name, RESQUE_PREFIX + @queue.to_s)
+        true
+      rescue Redis::CommandError => e
+        raise unless e.message == 'ERR no such key'
+        false
+      end
+
+      def pause
+        Resque.redis.renamenx(RESQUE_PREFIX + @queue.to_s, RESQUE_PREFIX + paused_queue_name)
+        true
+      rescue Redis::CommandError => e
+        raise unless e.message == 'ERR no such key'
+        false
+      end
+
+      def paused?(unknown = false)
+        # parameter is what to return if the queue is empty, and so the state is unknown
+        if Resque.inline
+          false
+        elsif Resque.redis.exists(RESQUE_PREFIX + @queue.to_s)
+          false
+        elsif Resque.redis.exists(RESQUE_PREFIX + paused_queue_name)
+          true
+        else
+          unknown
+        end
+      end
+
+      def paused_queue_name
+        @queue.to_s + '_paused'
+      end
+
+      def with_lock
+        if Resque.inline
+          yield
+        else
+          RedisMutex.with_lock(MUTEX, block: 60, expire: 120) { yield }
+        end
+      end
+
+      def find_class(klass)
+        return klass if klass.is_a? Class
+        return Object.const_get(klass) unless klass.include?('::')
+        klass.split('::').reduce(Object) do |mod, class_name|
+          mod.const_get(class_name)
+        end
+      end
+    end
+  end
+end