resque-integration 3.4.1

data/spec/resque/integration/queues_info_spec.rb

@@ -0,0 +1,402 @@
+require "spec_helper"
+
+describe Resque::Integration::QueuesInfo do
+  let(:queue_info) { described_class.new(config: 'spec/fixtures/resque_queues.yml') }
+
+  before do
+    Timecop.freeze Time.current
+  end
+
+  after do
+    Timecop.return
+  end
+
+  context 'age' do
+    before do
+      allow(Resque).to receive(:workers).and_return(workers)
+    end
+
+    context 'age_for_queue' do
+      context 'when old job running' do
+        let(:workers) do
+          [double(job: {'run_at' => 4.hours.ago.utc.iso8601, 'queue' => 'first'}, 'idle?' => false)]
+        end
+
+        it 'returns time for job ' do
+          expect(queue_info.age_for_queue('first')).to eq 4.hours.to_i
+        end
+      end
+
+      context 'when job running normal time' do
+        let(:workers) do
+          [double(job: {'run_at' => 4.seconds.ago.utc.iso8601, 'queue' => 'first'}, 'idle?' => false)]
+        end
+
+        it 'returns time for job ' do
+          expect(queue_info.age_for_queue('first')).to eq 4
+        end
+      end
+
+      context 'when queue is empty' do
+        let(:workers) { [] }
+
+        it 'returns 0' do
+          expect(queue_info.age_for_queue('first')).to eq 0
+        end
+      end
+
+      context 'when worker is iddling' do
+        let(:workers) do
+          [double('idle?' => true, job: nil)]
+        end
+
+        it 'returns 0' do
+          expect(queue_info.age_for_queue('first')).to eq 0
+        end
+      end
+    end
+
+    describe '#age_overall' do
+      context 'when there is old job for its queue' do
+        let(:workers) do
+          [
+            double(job: {'run_at' => 4.seconds.ago.utc.iso8601, 'queue' => 'second'}, 'idle?' => false),
+            double(job: {'run_at' => 20.seconds.ago.utc.iso8601, 'queue' => 'first'}, 'idle?' => false)
+          ]
+        end
+
+        it 'returns time for job ' do
+          expect(queue_info.age_overall).to eq 20
+        end
+      end
+
+      context 'when there is a several old jobs' do
+        let(:workers) do
+          [
+            double(job: {'run_at' => 100.seconds.ago.utc.iso8601, 'queue' => 'first'}, 'idle?' => false),
+            double(job: {'run_at' => 20.seconds.ago.utc.iso8601, 'queue' => 'second'}, 'idle?' => false)
+          ]
+        end
+
+        it 'returns time for the oldest job ' do
+          expect(queue_info.age_overall).to eq 100
+        end
+      end
+
+      context 'when the is no old job' do
+        let(:workers) do
+          [
+            double(job: {'run_at' => 4.seconds.ago.utc.iso8601, 'queue' => 'first'}, 'idle?' => false),
+            double(job: {'run_at' => 2.seconds.ago.utc.iso8601, 'queue' => 'second'}, 'idle?' => false)
+          ]
+        end
+
+        it 'returns 0' do
+          expect(queue_info.age_overall).to eq 0
+        end
+      end
+
+      context 'when old job running in unknown queue' do
+        let(:workers) do
+          [double(job: {'run_at' => 11.seconds.ago.utc.iso8601, 'queue' => 'unknown'}, 'idle?' => false)]
+        end
+
+        it 'checks time of job with defaults thresholds' do
+          expect(queue_info.age_overall).to eq 11
+        end
+      end
+
+      context "when one job have problem and other with older age doesn't" do
+        let(:workers) do
+          [
+            double(job: {'run_at' => 16.seconds.ago.utc.iso8601, 'queue' => 'first'}, 'idle?' => false),
+            double(job: {'run_at' => 14.seconds.ago.utc.iso8601, 'queue' => 'second'}, 'idle?' => false)
+          ]
+        end
+
+        it 'returns size for queue with problem' do
+          expect(queue_info.age_overall).to eq 14
+        end
+      end
+
+      context 'when there is no job running' do
+        let(:workers) do
+          [
+            double(job: nil, 'idle?' => true)
+          ]
+        end
+
+        it 'returns 0' do
+          expect(queue_info.age_overall).to eq 0
+        end
+      end
+    end
+  end
+
+  context 'size' do
+    describe '#size_for_queue' do
+      before do
+        allow(Resque).to receive(:size).with('first').and_return(size)
+      end
+
+      context 'when too much jobs in queue' do
+        let(:size) { 100 }
+
+        it 'returns queue size' do
+          expect(queue_info.size_for_queue('first')).to eq 100
+        end
+      end
+
+      context 'when queue have normal size' do
+        let(:size) { 1 }
+
+        it 'returns time for job ' do
+          expect(queue_info.size_for_queue('first')).to eq 1
+        end
+      end
+
+      context 'when queue is empty' do
+        let(:size) { nil }
+
+        it 'returns 0' do
+          expect(queue_info.size_for_queue('first')).to eq 0
+        end
+      end
+    end
+
+    describe '#size_overall' do
+      before do
+        allow(Resque).to receive(:queues).and_return(%w(first second))
+        allow(Resque).to receive(:size).with('first').and_return(size_first)
+        allow(Resque).to receive(:size).with('second').and_return(size_second)
+      end
+
+      context 'when there is one big queue' do
+        let(:size_first) { 100 }
+        let(:size_second) { nil }
+
+        it 'returns size for queue with problem' do
+          expect(queue_info.size_overall).to eq 100
+        end
+      end
+
+      context 'when there is a several problem queues' do
+        let(:size_first) { 1000 }
+        let(:size_second) { 200 }
+
+        it 'returns size for the lagrest queue ' do
+          expect(queue_info.size_overall).to eq 1000
+        end
+      end
+
+      context 'when the is no large queus' do
+        let(:size_first) { 1 }
+        let(:size_second) { 2 }
+
+        it 'returns 0' do
+          expect(queue_info.size_overall).to eq 0
+        end
+      end
+
+      context 'when lagre queue is unknown' do
+        let(:size_first) { nil }
+        let(:size_second) { 11 }
+
+        it 'checks size of queue with defaults thresholds' do
+          expect(queue_info.size_overall).to eq 11
+        end
+      end
+
+      context "when one queue have problem and other with bigger size doesn't" do
+        let(:size_first) { 30 }
+        let(:size_second) { 20 }
+
+        it 'returns size for queue with problem' do
+          expect(queue_info.size_overall).to eq 20
+        end
+      end
+    end
+  end
+
+  describe '#threshold_age' do
+    context 'when queue defined in config' do
+      let(:queue_name) { 'first' }
+
+      it 'returns threshold for age' do
+        expect(queue_info.threshold_age(queue_name)).to eq 20
+      end
+    end
+
+    context 'when queue not defined in config' do
+      let(:queue_name) { 'second' }
+
+      it 'returns default threshold' do
+        expect(queue_info.threshold_age(queue_name)).to eq 10
+      end
+    end
+  end
+
+  describe '#threshold_size' do
+    context 'when queue defined in config' do
+      let(:queue_name) { 'first' }
+
+      it 'returns threshold for size' do
+        expect(queue_info.threshold_size(queue_name)).to eq 100
+      end
+    end
+
+    context 'when queue not defined in config' do
+      let(:queue_name) { 'second' }
+
+      it 'returns default threshold' do
+        expect(queue_info.threshold_size(queue_name)).to eq 10
+      end
+    end
+  end
+
+  describe '#channel' do
+    context 'when queue defined in config' do
+      context 'when returns the one channel' do
+        let(:queue_name) { 'first' }
+
+        it { expect(queue_info.channel(queue_name)).to eq 'first' }
+      end
+
+      context 'when returns several channels' do
+        let(:queue_name) { 'second_queue' }
+
+        it do
+          expect(queue_info.channel(queue_name)).to eq 'first second'
+        end
+      end
+
+      context 'when channel not specified' do
+        let(:queue_name) { 'without_channel' }
+
+        it { expect(queue_info.channel(queue_name)).to eq 'default' }
+      end
+    end
+
+    context 'when queue not defined in config' do
+      let(:queue_name) { 'other_queue' }
+
+      it 'returns channel' do
+        expect(queue_info.channel(queue_name)).to eq 'default'
+      end
+    end
+  end
+
+  describe '#threshold_failures_count' do
+    context 'when queue is defined in config' do
+      let(:queue_name) { 'first' }
+
+      it 'returns failures count threshold for specified queue and time period' do
+        expect(queue_info.threshold_failures_count(queue_name, '5m')).to eq 15
+        expect(queue_info.threshold_failures_count(queue_name, '1h')).to eq 90
+      end
+    end
+
+    context 'when queue is not defined in config' do
+      let(:queue_name) { 'second' }
+
+      it 'returns default failures count threshold for specified time period' do
+        expect(queue_info.threshold_failures_count(queue_name, '5m')).to eq 5
+        expect(queue_info.threshold_failures_count(queue_name, '1h')).to eq 60
+      end
+    end
+  end
+
+  describe '#failures_count_for_queue' do
+    before do
+      allow(Resque::Integration::FailureBackends::QueuesTotals).to receive(:count).with('first').and_return(14)
+    end
+
+    it 'returns total failures count for specified queue' do
+      expect(queue_info.failures_count_for_queue('first')).to eq 14
+    end
+  end
+
+  describe 'configuration merging' do
+    let(:first_queue_name) { 'first' }
+    let(:third_queue_name) { 'third' }
+
+    it 'merges configs for queue in order of appearance' do
+      expect(queue_info.threshold_age(first_queue_name)).to eq 20
+      expect(queue_info.threshold_size(first_queue_name)).to eq 100
+      expect(queue_info.threshold_failures_count(first_queue_name, '5m')).to eq 15
+      expect(queue_info.threshold_failures_count(first_queue_name, '1h')).to eq 90
+
+      expect(queue_info.threshold_age(third_queue_name)).to eq 30
+      expect(queue_info.threshold_size(third_queue_name)).to eq 100
+      expect(queue_info.threshold_failures_count(third_queue_name, '5m')).to eq 15
+      expect(queue_info.threshold_failures_count(third_queue_name, '1h')).to eq 70
+    end
+  end
+
+  describe '#data' do
+    it do
+      expect(queue_info.data).to eq [
+        {
+          '{#QUEUE}' => 'first',
+
+          '{#CHANNEL}' => 'first',
+          '{#THRESHOLD_AGE}' => 20,
+          '{#THRESHOLD_SIZE}' => 100,
+          '{#THRESHOLD_FAILURES_PER_5M}' => 15,
+          '{#THRESHOLD_FAILURES_PER_1H}' => 90,
+
+          '{#WARNING_CHANNEL}' => 'first_warnings',
+          '{#WARNING_AGE}' => 10,
+          '{#WARNING_SIZE}' => 50,
+          '{#WARNING_FAILURES_PER_5M}' => 7,
+          '{#WARNING_FAILURES_PER_1H}' => 45
+        },
+        {
+          '{#QUEUE}' => 'third',
+
+          '{#CHANNEL}' => 'first',
+          '{#THRESHOLD_AGE}' => 30,
+          '{#THRESHOLD_SIZE}' => 100,
+          '{#THRESHOLD_FAILURES_PER_5M}' => 15,
+          '{#THRESHOLD_FAILURES_PER_1H}' => 70,
+
+          '{#WARNING_CHANNEL}' => 'first_warnings',
+          '{#WARNING_AGE}' => 15,
+          '{#WARNING_SIZE}' => 50,
+          '{#WARNING_FAILURES_PER_5M}' => 7,
+          '{#WARNING_FAILURES_PER_1H}' => 35
+        },
+        {
+          '{#QUEUE}' => 'second_queue',
+
+          '{#CHANNEL}' => 'first second',
+          '{#THRESHOLD_AGE}' => 10,
+          '{#THRESHOLD_SIZE}' => 10,
+          '{#THRESHOLD_FAILURES_PER_5M}' => 5,
+          '{#THRESHOLD_FAILURES_PER_1H}' => 60,
+
+          '{#WARNING_CHANNEL}' => 'first_warnings second_warnings',
+          '{#WARNING_AGE}' => 8,
+          '{#WARNING_SIZE}' => 7,
+          '{#WARNING_FAILURES_PER_5M}' => 3,
+          '{#WARNING_FAILURES_PER_1H}' => 30
+        },
+        {
+          '{#QUEUE}' => 'without_channel',
+
+          '{#CHANNEL}' => 'default',
+          '{#THRESHOLD_AGE}' => 10,
+          '{#THRESHOLD_SIZE}' => 10,
+          '{#THRESHOLD_FAILURES_PER_5M}' => 5,
+          '{#THRESHOLD_FAILURES_PER_1H}' => 60,
+
+          '{#WARNING_CHANNEL}' => 'default_warnings',
+          '{#WARNING_AGE}' => 8,
+          '{#WARNING_SIZE}' => 7,
+          '{#WARNING_FAILURES_PER_5M}' => 3,
+          '{#WARNING_FAILURES_PER_1H}' => 30
+        }
+      ]
+    end
+  end
+end

data/spec/resque/integration/unique_spec.rb

@@ -0,0 +1,184 @@
+require 'spec_helper'
+
+describe Resque::Integration::Unique, '#meta_id, #lock' do
+  let(:job) { Class.new }
+  before { job.extend Resque::Integration::Unique }
+
+  context 'when unique arguments are not set' do
+    it 'returns same results when equal arguments given' do
+      job.meta_id('a', 'b').should eq job.meta_id('a', 'b')
+      job.lock_id('a', 'b').should eq job.lock_id('a', 'b')
+    end
+
+    it 'returns different results when different arguments given' do
+      job.meta_id('a', 'b').should_not eq job.meta_id('a', 'c')
+      job.lock_id('a', 'b').should_not eq job.lock_id('a', 'c')
+    end
+  end
+
+  context 'when unique arguments are set' do
+    # mark second argument as unique
+    before { job.lock_on &->(a, b) { b } }
+
+    it 'returns same results when equal arguments given' do
+      job.meta_id('a', 'b').should eq job.meta_id('a', 'b')
+      job.lock_id('a', 'b').should eq job.lock_id('a', 'b')
+      job.lock_id('a', foo: 1).should eq job.lock_id('a', foo: 1)
+
+      job.meta_id('a', 'b').should eq job.meta_id('c', 'b')
+      job.lock_id('a', 'b').should eq job.lock_id('c', 'b')
+      job.lock_id('a', foo: 1).should eq job.lock_id('c', foo: 1)
+    end
+
+    it 'returns different results when different arguments given' do
+      job.meta_id('a', 'b').should_not eq job.meta_id('a', 'c')
+      job.lock_id('a', 'b').should_not eq job.lock_id('a', 'c')
+    end
+  end
+end
+
+describe Resque::Integration::Unique, '#enqueue, #enqueued?' do
+  class JobEnqueueTest
+    extend Resque::Integration::Unique
+
+    @queue = :queue1
+  end
+
+  it 'returns false when job is not enqueued' do
+    JobEnqueueTest.should_not be_enqueued(0)
+  end
+
+  it 'returns new meta when job is enqueued' do
+    meta = JobEnqueueTest.enqueue(1)
+    meta.should be_a Resque::Plugins::Meta::Metadata
+
+    JobEnqueueTest.should be_enqueued(1)
+  end
+
+  it 'returns new meta when job is enqueued to specific queue' do
+    meta = JobEnqueueTest.enqueue_to(:fast, 1)
+    meta.should be_a Resque::Plugins::Meta::Metadata
+
+    expect(JobEnqueueTest.enqueued?(1)).to_not be_nil
+    expect(Resque.size(:fast)).to eq 1
+  end
+
+  it 'returns new meta when job is scheduled' do
+    meta = JobEnqueueTest.scheduled(:fast, 'JobEnqueueTest', 1)
+    meta.should be_a Resque::Plugins::Meta::Metadata
+
+    expect(JobEnqueueTest.enqueued?(1)).to_not be_nil
+    expect(Resque.size(:fast)).to eq 1
+  end
+
+  it 'returns the same meta if job already in queue' do
+    meta1 = JobEnqueueTest.enqueue(2)
+    meta2 = JobEnqueueTest.enqueue(2)
+
+    meta1.meta_id.should eq meta2.meta_id
+    meta1.enqueued_at.should eq meta2.enqueued_at
+
+    JobEnqueueTest.should be_enqueued(2)
+  end
+end
+
+describe Resque::Integration::Unique, '#dequeue' do
+  class JobDequeueTest
+    extend Resque::Integration::Unique
+
+    @queue = :queue2
+
+    def self.execute(x)
+      sleep 0.1
+    end
+  end
+
+  it 'dequeues and unlocks job if job is not in work now' do
+    JobDequeueTest.enqueue(1)
+    JobDequeueTest.should be_enqueued(1)
+
+    JobDequeueTest.dequeue(1)
+    JobDequeueTest.should_not be_enqueued(1)
+    JobDequeueTest.should_not be_locked(1)
+
+    meta = JobDequeueTest.get_meta(JobDequeueTest.meta_id(1))
+    meta.should be_failed
+  end
+
+  it 'does not dequeue jobs in progress' do
+    meta = JobDequeueTest.enqueue(1)
+
+    job = Resque.reserve(:queue2)
+
+    worker = Thread.new {
+      job.perform
+    }
+
+    sleep 0.01 # give a worker some time to start
+    meta.reload!
+    meta.should be_working
+
+    JobDequeueTest.dequeue(1)
+    JobDequeueTest.should be_locked(1)
+    JobDequeueTest.should be_enqueued(1)
+
+    worker.join
+  end
+end
+
+describe Resque::Integration::Unique, '#on_failure_retry' do
+  class JobUniqueWithRetry
+    include Resque::Integration
+    extend Resque::Plugins::Retry
+
+    @retry_limit = 2
+    @retry_delay = 1
+    @retry_exceptions = [IOError]
+    @expire_retry_key_after = 300
+
+    unique do |foo_var, params|
+      params[:foo]
+    end
+
+    queue :default
+
+    def self.execute(foo_var, params)
+      sleep 0.2
+      Resque.logger.info 'Hello, world'
+    end
+  end
+
+  class JobOnlyUnique
+    include Resque::Integration
+
+    unique
+
+    def self.execute
+      raise ArgumentError.new('Some exception in Job')
+    end
+  end
+
+  let(:worker) { Resque::Worker.new(:default) }
+
+  context 'when unique with retry' do
+    let(:job) { Resque::Job.new(:default, 'class' => 'JobUniqueWithRetry', 'args' => ['abcd', 1, {foo: 'bar'}]) }
+
+    before { worker.working_on(job) }
+
+    it do
+      expect { worker.unregister_worker }.not_to raise_error
+
+      expect(Resque::Failure.count).to eq 1
+      expect(Resque::Failure.all['exception']).to eq 'Resque::DirtyExit'
+    end
+  end
+
+  context 'when only unique' do
+    let(:job) { Resque::Job.new(:default, 'class' => 'JobOnlyUnique', 'args' => ['abcd']) }
+
+    it do
+      expect { job.perform }.not_to raise_error(RuntimeError, /no superclass method `on_failure_retry'/)
+      expect { job.perform }.to raise_error(ArgumentError)
+    end
+  end
+end

data/spec/resque/integration_spec.rb

@@ -0,0 +1,105 @@
+# coding: utf-8
+require 'spec_helper'
+
+RSpec.describe Resque::Integration do
+  describe '#unique?' do
+    let(:job) { Class.new }
+    before { job.send :include, Resque::Integration }
+
+    subject { job }
+
+    context 'when #unique is not called' do
+      it { should_not be_unique }
+    end
+
+    context 'when #unique is called' do
+      before { job.unique }
+
+      it { should be_unique }
+    end
+  end
+
+  describe 'enqueue' do
+    context 'when job is uniq' do
+      class DummyService
+        def self.call
+          # no-op
+        end
+      end
+
+      class UniqueJob
+        include Resque::Integration
+
+        queue :test
+        unique
+
+        def self.execute(id, params)
+          DummyService.call
+        end
+      end
+
+      it 'enqueues only one job' do
+        UniqueJob.enqueue(1, param: 'one')
+
+        Timecop.travel(10.hours.since) do
+          UniqueJob.enqueue(1, param: 'one')
+
+          expect(Resque.peek(:test, 0, 100).size).to eq(1)
+        end
+      end
+
+      it 'enqueues two jobs with differ args' do
+        UniqueJob.enqueue(1, param: 'one')
+
+        Timecop.travel(10.hours.since) do
+          UniqueJob.enqueue(1, param: 'two')
+
+          expect(Resque.peek(:test, 0, 100).size).to eq(2)
+        end
+      end
+
+      it 'enqueues two jobs after expire lock timeout' do
+        UniqueJob.enqueue(1, param: 'one')
+
+        Timecop.travel(4.days.since) do
+          UniqueJob.enqueue(1, param: 'one')
+
+          expect(Resque.peek(:test, 0, 100).size).to eq(2)
+        end
+      end
+
+      describe 'unlock' do
+        include_context 'resque inline'
+
+        class UniqueJobWithBlock
+          include Resque::Integration
+
+          queue :test_with_block
+          unique { |id, params| [id, params[:one], params[:two]] }
+
+          def self.execute(id, params)
+            DummyService.call
+          end
+        end
+
+        it 'unlocks uniq job with args and without block' do
+          expect(DummyService).to receive(:call).twice
+
+          UniqueJob.enqueue(1, one: 1, two: 2)
+          UniqueJob.enqueue(1, one: 1, two: 2)
+
+          expect(UniqueJob.locked?(1, one: 1, two: 2)).to eq(false)
+        end
+
+        it 'unlocks uniq job with args and block' do
+          expect(DummyService).to receive(:call).twice
+
+          UniqueJobWithBlock.enqueue(1, one: 1, two: 2)
+          UniqueJobWithBlock.enqueue(1, one: 1, two: 2)
+
+          expect(UniqueJobWithBlock.locked?(1, one: 1, two: 2)).to eq(false)
+        end
+      end
+    end
+  end
+end

data/spec/shared/resque_inline.rb

@@ -0,0 +1,10 @@
+RSpec.shared_context 'resque inline' do
+  around do |example|
+    inline = Resque.inline
+    Resque.inline = true
+
+    example.run
+
+    Resque.inline = inline
+  end
+end