resque-rate_limited 1.1.0

data/lib/resque/plugins/rate_limited/rate_limited_un_pause.rb

@@ -0,0 +1,39 @@
+module Resque
+  module Plugins
+    module RateLimited
+      class UnPause
+        @queue = nil
+
+        class << self
+          attr_writer(:queue)
+
+          def use?
+            Resque.respond_to?(:enqueue_at_with_queue) && @queue
+          end
+
+          def enqueue(timestamp, klass)
+            # If Resque scheduler is installed and queue is set - use it to queue a wake up job
+            return unless use?
+            Resque.enqueue_at_with_queue(
+              @queue,
+              timestamp,
+              Resque::Plugins::RateLimited::UnPause,
+              klass
+            )
+          end
+
+          def perform(klass)
+            class_from_string(klass.to_s).un_pause
+          end
+
+          def class_from_string(str)
+            return Object.const_get(str) unless str.include?('::')
+            str.split('::').reduce(Object) do |mod, class_name|
+              mod.const_get(class_name)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/resque/rate_limited.rb

@@ -0,0 +1,9 @@
+require 'resque'
+require 'redis-mutex'
+require 'resque/version'
+require 'resque/plugins/rate_limited/rate_limited'
+require 'resque/plugins/rate_limited/rate_limited_un_pause'
+require 'resque/plugins/rate_limited/apis/base_api_queue'
+require 'resque/plugins/rate_limited/apis/angellist_queue'
+require 'resque/plugins/rate_limited/apis/evernote_queue'
+require 'resque/plugins/rate_limited/apis/twitter_queue'

data/resque-rate_limited.gemspec

@@ -0,0 +1,43 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'resque-rate_limited/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'resque-rate_limited'
+  spec.version       = RateLimited::VERSION
+  spec.authors       = ['Greg Dowling']
+  spec.email         = ['mail@greddowling.com']
+  spec.summary     = 'A Resque plugin to help manage jobs that use rate limited apis, pausing when you hit the limits and restarting later.'
+  spec.description = 'A Resque plugin which allows you to create dedicated queues for jobs that use rate limited apis.
+These queues will pause when one of the jobs hits a rate limit, and unpause after a suitable time period.
+The rate_limited can be used directly, and just requires catching the rate limit exception and pausing the
+queue. There are also additional queues provided that already include the pause/rety logic for twitter, angelist
+and evernote; these allow you to support rate limited apis with minimal changes.'
+
+  spec.homepage      = 'http://github.com/Xenapto/resque-rate_limited'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'resque', '~> 1.9', '>= 1.9.10'
+  spec.add_dependency 'redis-mutex', '~> 4.0', '>= 4.0.0'
+
+  spec.add_dependency 'angellist_api', '~> 1.0', '>= 1.0.7'
+  spec.add_dependency 'evernote-thrift', '~> 1.25', '>= 1.25.1'
+  spec.add_dependency 'twitter', '~> 5.11', '>= 5.11.0'
+
+  spec.add_development_dependency 'rake', '~> 10'
+  spec.add_development_dependency 'rspec', '~> 2'
+  spec.add_development_dependency 'simplecov', '~> 0'
+  spec.add_development_dependency 'rubocop', '~> 0'
+  spec.add_development_dependency 'reek', '~> 4'
+  spec.add_development_dependency 'listen', '~> 3.0', '< 3.1' # Dependency of guard, 3.1 requires Ruby 2.2+
+  spec.add_development_dependency 'guard', '~> 2'
+  spec.add_development_dependency 'guard-rspec', '~> 4'
+  spec.add_development_dependency 'guard-rubocop', '~> 1'
+  spec.add_development_dependency 'gem-release', '~> 0'
+end

data/spec/apis/angellist_queue_spec.rb

@@ -0,0 +1,51 @@
+require 'spec_helper'
+require 'resque/rate_limited'
+
+class RateLimitedTestQueueAL
+  def self.perform(succeed)
+    raise(AngellistApi::Error::TooManyRequests, 'error') unless succeed
+  end
+end
+
+describe Resque::Plugins::RateLimited::AngellistQueue do
+  before do
+    Resque::Plugins::RateLimited::AngellistQueue.stub(:paused?).and_return(false)
+  end
+
+  describe 'enqueue' do
+    it 'enqueues to the correct queue with the correct parameters' do
+      Resque.should_receive(:enqueue_to).with(
+        :angellist_api,
+        Resque::Plugins::RateLimited::AngellistQueue,
+        RateLimitedTestQueueAL.to_s,
+        true
+      )
+      Resque::Plugins::RateLimited::AngellistQueue
+        .enqueue(RateLimitedTestQueueAL, true)
+    end
+  end
+
+  describe 'perform' do
+    before do
+      Resque.inline = true
+    end
+    context 'with everything' do
+      it 'calls the class with the right parameters' do
+        RateLimitedTestQueueAL.should_receive(:perform).with('test_param')
+        Resque::Plugins::RateLimited::AngellistQueue
+          .enqueue(RateLimitedTestQueueAL, 'test_param')
+      end
+    end
+
+    context 'with rate limit exception' do
+      before do
+        Resque::Plugins::RateLimited::AngellistQueue.stub(:rate_limited_requeue)
+      end
+      it 'pauses queue when request fails' do
+        Resque::Plugins::RateLimited::AngellistQueue.should_receive(:pause_until)
+        Resque::Plugins::RateLimited::AngellistQueue
+          .enqueue(RateLimitedTestQueueAL, false)
+      end
+    end
+  end
+end

data/spec/apis/evernote_queue_spec.rb

@@ -0,0 +1,77 @@
+require 'spec_helper'
+require 'resque/rate_limited'
+
+class RateLimitDuration
+  def self.seconds
+    60
+  end
+end
+
+class RateLimitedTestQueueEn
+  def self.perform(succeed)
+    raise(
+      Evernote::EDAM::Error::EDAMSystemException,
+      errorCode: Evernote::EDAM::Error::EDAMErrorCode::RATE_LIMIT_REACHED,
+      rateLimitDuration: RateLimitDuration
+    ) unless succeed
+  end
+end
+
+class RateLimitedTestQueueOther
+  def self.perform
+    raise(Evernote::EDAM::Error::EDAMSystemException)
+  end
+end
+
+describe Resque::Plugins::RateLimited::EvernoteQueue do
+  before do
+    Resque::Plugins::RateLimited::EvernoteQueue.stub(:paused?).and_return(false)
+  end
+  describe 'enqueue' do
+    it 'enqueues to the correct queue with the correct parameters' do
+      Resque.should_receive(:enqueue_to).with(
+        :evernote_api,
+        Resque::Plugins::RateLimited::EvernoteQueue,
+        RateLimitedTestQueueEn.to_s,
+        true
+      )
+      Resque::Plugins::RateLimited::EvernoteQueue
+        .enqueue(RateLimitedTestQueueEn, true)
+    end
+  end
+
+  describe 'perform' do
+    before do
+      Resque.inline = true
+    end
+    context 'with everything' do
+      it 'calls the class with the right parameters' do
+        RateLimitedTestQueueEn.should_receive(:perform).with('test_param')
+        Resque::Plugins::RateLimited::EvernoteQueue
+          .enqueue(RateLimitedTestQueueEn, 'test_param')
+      end
+    end
+
+    context 'with rate limit exception' do
+      before do
+        Resque::Plugins::RateLimited::EvernoteQueue.stub(:rate_limited_requeue)
+      end
+      it 'pauses queue when request fails' do
+        Resque::Plugins::RateLimited::EvernoteQueue.should_receive(:pause_until)
+        Resque::Plugins::RateLimited::EvernoteQueue
+          .enqueue(RateLimitedTestQueueEn, false)
+      end
+    end
+
+    context 'with exception that is not rate limit' do
+      before do
+        Resque::Plugins::RateLimited::EvernoteQueue.stub(:rate_limited_requeue)
+      end
+      it 'raises the exception when request fails' do
+        expect do
+          Resque::Plugins::RateLimited::EvernoteQueue.enqueue(RateLimitedTestQueueOther)
+        end.to raise_error Evernote::EDAM::Error::EDAMSystemException
+      end
+    end
+  end
+end

data/spec/apis/twitter_queue_spec.rb

@@ -0,0 +1,52 @@
+require 'spec_helper'
+require 'resque/rate_limited'
+
+class RateLimitedTestQueueTw
+  def self.perform(succeed)
+    raise(Twitter::Error::TooManyRequests
+      .new('', 'x-rate-limit-reset' => (Time.now + 60).to_i)) unless succeed
+  end
+end
+
+describe Resque::Plugins::RateLimited::TwitterQueue do
+  before do
+    Resque::Plugins::RateLimited::TwitterQueue.stub(:paused?).and_return(false)
+  end
+
+  describe 'enqueue' do
+    it 'enqueues to the correct queue with the correct parameters' do
+      Resque.should_receive(:enqueue_to).with(
+        :twitter_api,
+        Resque::Plugins::RateLimited::TwitterQueue,
+        RateLimitedTestQueueTw.to_s,
+        true
+      )
+      Resque::Plugins::RateLimited::TwitterQueue
+        .enqueue(RateLimitedTestQueueTw, true)
+    end
+  end
+
+  describe 'perform' do
+    before do
+      Resque.inline = true
+    end
+    context 'with everything' do
+      it 'calls the class with the right parameters' do
+        RateLimitedTestQueueTw.should_receive(:perform).with('test_param')
+        Resque::Plugins::RateLimited::TwitterQueue
+          .enqueue(RateLimitedTestQueueTw, 'test_param')
+      end
+    end
+
+    context 'with rate limit exception' do
+      before do
+        Resque::Plugins::RateLimited::TwitterQueue.stub(:rate_limited_requeue)
+      end
+      it 'pauses queue when request fails' do
+        Resque::Plugins::RateLimited::TwitterQueue.should_receive(:pause_until)
+        Resque::Plugins::RateLimited::TwitterQueue
+          .enqueue(RateLimitedTestQueueTw, false)
+      end
+    end
+  end
+end

data/spec/rate_limited_spec.rb

@@ -0,0 +1,243 @@
+require 'spec_helper'
+require 'resque/rate_limited'
+
+class RateLimitedTestQueue
+  extend Resque::Plugins::RateLimited
+
+  @queue = :test
+
+  def self.perform(succeed)
+    rate_limited_requeue(self, succeed) unless succeed
+  end
+
+  def self.queue_name_private
+    @queue.to_s
+  end
+
+  def self.queue_private
+    @queue
+  end
+end
+
+describe Resque::Plugins::RateLimited do
+  it 'should be compliance with Resque::Plugin document' do
+    expect { Resque::Plugin.lint(Resque::Plugins::RateLimited) }.to_not raise_error
+  end
+
+  shared_examples_for 'queue' do |queue_suffix|
+    it 'should queue to the correct queue' do
+      queue_param = queue_suffix.empty? ? RateLimitedTestQueue.queue_private : "#{RateLimitedTestQueue.queue_name_private}#{queue_suffix}"
+      Resque.should_receive(:enqueue_to).with(queue_param, nil, nil)
+      RateLimitedTestQueue.rate_limited_enqueue(nil, nil)
+    end
+  end
+
+  context 'when queue is not paused' do
+    before do
+      RateLimitedTestQueue.stub(:paused?).and_return(false)
+    end
+
+    describe 'enqueue' do
+      include_examples 'queue', ''
+    end
+
+    describe 'paused?' do
+      it { RateLimitedTestQueue.paused?.should be false }
+    end
+
+    describe 'perform' do
+      it 'should requeue the job on failure' do
+        Resque.should_receive(:enqueue_to)
+        RateLimitedTestQueue.perform(false)
+      end
+
+      it 'should not requeue the job on success' do
+        Resque.should_not_receive(:enqueue_to)
+        RateLimitedTestQueue.perform(true)
+      end
+    end
+
+    describe 'pause' do
+      it 'should rename the queue to paused' do
+        Resque.redis.should_receive(:renamenx).with("queue:#{RateLimitedTestQueue.queue_name_private}", "queue:#{RateLimitedTestQueue.queue_name_private}_paused")
+        RateLimitedTestQueue.pause
+      end
+    end
+
+    describe 'un_pause' do
+      it 'should not unpause the queue' do
+        Resque.redis.should_not_receive(:renamenx).with("queue:#{RateLimitedTestQueue.queue_name_private}", "queue:#{RateLimitedTestQueue.queue_name_private}_paused")
+        RateLimitedTestQueue.un_pause
+      end
+    end
+
+    describe 'pause_until' do
+      before do
+        Resque.redis.stub(:renamenx).and_return(true)
+      end
+
+      it 'should pause the queue' do
+        RateLimitedTestQueue.should_receive(:pause)
+        RateLimitedTestQueue.pause_until(Time.now + (5 * 60 * 60))
+      end
+
+      it 'should schedule an unpause job' do
+        Resque::Plugins::RateLimited::UnPause.should_receive(:enqueue)
+                                                  .with(nil, 'RateLimitedTestQueue')
+        RateLimitedTestQueue.pause_until(nil)
+      end
+    end
+  end
+
+  context 'when queue is paused' do
+    before do
+      RateLimitedTestQueue.stub(:paused?).and_return(true)
+    end
+
+    describe 'enqueue' do
+      include_examples 'queue', '_paused'
+    end
+
+    describe 'paused?' do
+      it { RateLimitedTestQueue.paused?.should be true }
+    end
+
+    describe 'perform' do
+      it 'should not execute the block' do
+        Resque.should_receive(:enqueue_to).with("#{RateLimitedTestQueue.queue_name_private}_paused", RateLimitedTestQueue, true)
+        RateLimitedTestQueue.should_not_receive(:perform)
+        RateLimitedTestQueue.around_perform_with_check_and_requeue(true)
+      end
+    end
+
+    describe 'un_pause' do
+      it 'should rename the queue to live' do
+        Resque.redis.should_receive(:renamenx).with("queue:#{RateLimitedTestQueue.queue_name_private}_paused", "queue:#{RateLimitedTestQueue.queue_name_private}")
+        RateLimitedTestQueue.un_pause
+      end
+    end
+  end
+
+  describe 'when queue is paused and Resque is in inline mode' do
+    let(:resque_prefix) { Resque::Plugins::RateLimited::RESQUE_PREFIX }
+    let(:queue) { resque_prefix + RateLimitedTestQueue.queue_name_private }
+    let(:paused_queue) { resque_prefix + RateLimitedTestQueue.paused_queue_name }
+
+    before do
+      Resque.redis.stub(:exists).with(queue).and_return(false)
+      Resque.redis.stub(:exists).with(paused_queue).and_return(true)
+      Resque.inline = true
+    end
+
+    after do
+      Resque.inline = false
+    end
+
+    it 'would be paused' do
+      expect(Resque.redis.exists(queue)).to eq false
+      expect(Resque.redis.exists(paused_queue)).to eq true
+    end
+
+    it 'says it is not paused' do
+      expect(RateLimitedTestQueue.paused?).to eq false
+    end
+
+    it 'performs the job' do
+      expect do
+        # Stack overflow unless handled
+        RateLimitedTestQueue.rate_limited_enqueue(RateLimitedTestQueue, true)
+      end.not_to raise_error
+    end
+  end
+
+  describe 'find_class' do
+    it 'works with symbol' do
+      RateLimitedTestQueue.find_class(RateLimitedTestQueue).should eq RateLimitedTestQueue
+    end
+
+    it 'works with simple string' do
+      RateLimitedTestQueue.find_class('RateLimitedTestQueue').should eq RateLimitedTestQueue
+    end
+
+    it 'works with complex string' do
+      RateLimitedTestQueue.find_class('Resque::Plugins::RateLimited').should eq Resque::Plugins::RateLimited
+    end
+  end
+
+  context 'with redis errors' do
+    before do
+      RateLimitedTestQueue.stub(:paused?).and_return(true)
+    end
+    context 'with not found error' do
+      before do
+        Resque.redis.stub(:renamenx).and_raise(Redis::CommandError.new('ERR no such key'))
+      end
+
+      describe 'pause' do
+        it 'should not throw exception' do
+          expect { RateLimitedTestQueue.pause }.to_not raise_error
+        end
+      end
+
+      describe 'un_pause' do
+        it 'should not throw exception' do
+          expect { RateLimitedTestQueue.un_pause }.to_not raise_error
+        end
+      end
+    end
+
+    context 'with other errror' do
+      before do
+        Resque.redis.stub(:renamenx).and_raise(Redis::CommandError.new('ERR something else'))
+      end
+
+      describe 'pause' do
+        it 'should throw exception' do
+          expect { RateLimitedTestQueue.pause }.to raise_error(Redis::CommandError)
+        end
+      end
+
+      describe 'un_pause' do
+        it 'should throw exception' do
+          expect { RateLimitedTestQueue.un_pause }.to raise_error(Redis::CommandError)
+        end
+      end
+    end
+  end
+
+  describe 'paused?' do
+    context 'with paused queue' do
+      before do
+        Resque.redis.stub(:exists).with("queue:#{RateLimitedTestQueue.queue_name_private}_paused").and_return(true)
+        Resque.redis.stub(:exists).with("queue:#{RateLimitedTestQueue.queue_name_private}").and_return(false)
+      end
+
+      it 'should return the true if the paused queue exists' do
+        expect(RateLimitedTestQueue.paused?).to eq(true)
+      end
+    end
+
+    context 'with un paused queue' do
+      before do
+        Resque.redis.stub(:exists).with("queue:#{RateLimitedTestQueue.queue_name_private}_paused").and_return(false)
+        Resque.redis.stub(:exists).with("queue:#{RateLimitedTestQueue.queue_name_private}").and_return(true)
+      end
+
+      it 'should return the false if the main queue exists exist' do
+        expect(RateLimitedTestQueue.paused?).to eq(false)
+      end
+    end
+
+    context 'with unknown queue state' do
+      before do
+        Resque.redis.stub(:exists).with("queue:#{RateLimitedTestQueue.queue_name_private}_paused").and_return(false)
+        Resque.redis.stub(:exists).with("queue:#{RateLimitedTestQueue.queue_name_private}").and_return(false)
+      end
+
+      it 'should return the default' do
+        expect(RateLimitedTestQueue.paused?(true)).to eq(true)
+        expect(RateLimitedTestQueue.paused?(false)).to eq(false)
+      end
+    end
+  end
+end