queue_classic_plus 1.0.0.alpha2 → 4.0.0.alpha8

data/spec/base_spec.rb

@@ -12,7 +12,19 @@ describe QueueClassicPlus::Base do
       it "does not allow multiple enqueues" do
         subject.do
         subject.do
-        subject.should have_queue_size_of(1)
+        expect(subject).to have_queue_size_of(1)
+      end
+
+      it "checks for an existing job using the same serializing as job enqueuing" do
+        # simulate a case where obj#to_json and JSON.dump(obj) do not match
+        require 'active_support/core_ext/date_time'
+        require 'active_support/json'
+        ActiveSupport::JSON::Encoding.use_standard_json_time_format = false
+
+        date = DateTime.new(2020, 11, 3)
+        subject.do(date)
+        subject.do(date)
+        expect(subject).to have_queue_size_of(1)
       end
 
       it "does allow multiple enqueues if something got locked for too long" do
@@ -20,7 +32,7 @@ describe QueueClassicPlus::Base do
         one_day_ago = Time.now - 60*60*24
         execute "UPDATE queue_classic_jobs SET locked_at = '#{one_day_ago}' WHERE q_name = 'test'"
         subject.do
-        subject.should have_queue_size_of(2)
+        expect(subject).to have_queue_size_of(2)
       end
     end
 
@@ -39,13 +51,15 @@ describe QueueClassicPlus::Base do
       end
 
       it "calls perform in a transaction" do
-        QueueClassicPlus::Base.should_receive(:transaction).and_call_original
+        expect(QueueClassicPlus::Base).to receive(:transaction).and_call_original
+
         subject._perform
       end
 
       it "measures the time" do
-        QueueClassicPlus::Metrics.should_receive(:timing).with("qu_perform_time", {source: "funky.name"}).and_call_original
-        subject._perform 
+        expect(QueueClassicPlus::Metrics).to receive(:timing).with("qu_perform_time", {source: "funky.name"}).and_call_original
+
+        subject._perform
       end
     end
 
@@ -61,7 +75,8 @@ describe QueueClassicPlus::Base do
       end
 
       it "calls perform outside of a transaction" do
-        QueueClassicPlus::Base.should_not_receive(:transaction)
+        expect(QueueClassicPlus::Base).to_not receive(:transaction)
+
         subject._perform
       end
     end
@@ -79,15 +94,15 @@ describe QueueClassicPlus::Base do
       end
 
       it "retries on specified exception" do
-        subject.retries_on?(SomeException.new).should be(true)
+        expect(subject.retries_on?(SomeException.new)).to be(true)
       end
 
       it "does not retry on unspecified exceptions" do
-        subject.retries_on?(RuntimeError).should be(false)
+        expect(subject.retries_on?(RuntimeError)).to be(false)
       end
 
       it "sets max retries" do
-        subject.max_retries.should == 5
+        expect(subject.max_retries).to eq(5)
       end
     end
 
@@ -104,16 +119,16 @@ describe QueueClassicPlus::Base do
       end
 
       it "retries on all specified exceptions" do
-        subject.retries_on?(SomeException.new).should be(true)
-        subject.retries_on?(SomeOtherException.new).should be(true)
+        expect(subject.retries_on?(SomeException.new)).to be(true)
+        expect(subject.retries_on?(SomeOtherException.new)).to be(true)
       end
 
       it "does not retry on unspecified exceptions" do
-        subject.retries_on?(RuntimeError).should be(false)
+        expect(subject.retries_on?(RuntimeError)).to be(false)
       end
 
       it "sets max retries" do
-        subject.max_retries.should == 5
+        expect(subject.max_retries).to eq(5)
       end
     end
 
@@ -133,22 +148,49 @@ describe QueueClassicPlus::Base do
       end
 
       it "retries on a subclass of a specified exception" do
-        subject.retries_on?(ServiceReallyUnavailable.new).should be(true)
+        expect(subject.retries_on?(ServiceReallyUnavailable.new)).to be(true)
       end
 
       it "does not retry on unspecified exceptions" do
-        subject.retries_on?(RuntimeError).should be(false)
+        expect(subject.retries_on?(RuntimeError)).to be(false)
       end
 
       it "sets max retries" do
-        subject.max_retries.should == 5
+        expect(subject.max_retries).to eq(5)
+      end
+    end
+
+    context "with Rails defined" do
+      require 'active_job/arguments'
+
+      before { stub_const('Rails', true) }
+
+      subject do
+        Class.new(QueueClassicPlus::Base) do
+          @queue = :test
+
+          def self.perform(foo, bar)
+          end
+        end
+      end
+
+      it "serializes parameters when enqueuing a job" do
+        expect(ActiveJob::Arguments).to receive(:serialize).with([42, true])
+
+        subject.do(42, true)
+      end
+
+      it "deserializes parameters when performing an enqueued job" do
+        expect(ActiveJob::Arguments).to receive(:deserialize).with([42, true]) { [42, true] }
+
+        subject._perform(42, true)
       end
     end
   end
 
   describe ".librato_key" do
     it "removes unsupported caracter from the classname" do
-      Jobs::Tests::TestJob.librato_key.should == 'jobs.tests.test_job'
+      expect(Jobs::Tests::TestJob.librato_key).to eq('jobs.tests.test_job')
     end
   end
 end

data/spec/datadog_spec.rb

@@ -0,0 +1,18 @@
+describe 'requiring queue_classic_plus/new_relic' do
+  class FunkyName < QueueClassicPlus::Base
+    @queue = :test
+
+    def self.perform
+    end
+  end
+
+  subject { FunkyName._perform }
+
+  it 'adds Datadog profiling support' do
+    require 'queue_classic_plus/datadog'
+    expect(Datadog.tracer).to receive(:trace).with(
+      'qc.job', service_name: 'qc.job', resource: 'FunkyName#perform'
+    )
+    subject
+  end
+end

data/spec/helpers.rb

@@ -4,6 +4,6 @@ module QcHelpers
   end
 
   def find_job(id)
-    execute("SELECT * FROM #{QC::TABLE_NAME} WHERE id = $1", id)
+    execute("SELECT * FROM #{QC.table_name} WHERE id = $1", id)
   end
 end

data/spec/new_relic_spec.rb

@@ -0,0 +1,26 @@
+describe 'requiring queue_classic_plus/new_relic' do
+  subject do
+    Class.new(QueueClassicPlus::Base) do
+      @queue = :test
+
+      def self.perform
+      end
+
+      def self.name
+        'Funky::Name'
+      end
+    end
+  end
+
+  it 'adds NewRelic profiling support' do
+    expect(subject).to receive(:perform_action_with_newrelic_trace).once.with({
+      name: 'perform',
+      class_name: 'Funky::Name',
+      category: 'OtherTransaction/QueueClassicPlus',
+    })
+
+    subject._perform
+    require 'queue_classic_plus/new_relic'
+    subject._perform
+  end
+end

data/spec/queue_classic/queue_spec.rb

@@ -0,0 +1,43 @@
+require 'spec_helper'
+require 'active_record'
+
+describe QC do
+  describe ".lock" do
+    context "with a connection from ActiveRecord that casts return types" do
+      before do
+        @old_conn_adapter = QC.default_conn_adapter
+        @activerecord_conn = ActiveRecord::Base.establish_connection(ENV["DATABASE_URL"])
+        QC.default_conn_adapter = QC::ConnAdapter.new(
+          connection: ActiveRecord::Base.connection.raw_connection
+        )
+      end
+
+      after do
+        @activerecord_conn.disconnect!
+        QC.default_conn_adapter = @old_conn_adapter
+      end
+
+      it "locks the job with remaining_retries" do
+        QC.enqueue_retry_in(1, "puts", 5, 2)
+        sleep 1
+        job = QC.lock
+
+        expect(job[:q_name]).to eq("default")
+        expect(job[:method]).to eq("puts")
+        expect(job[:args][0]).to be(2)
+        expect(job[:remaining_retries]).to eq("5")
+      end
+    end
+
+    it "locks the job with remaining_retries" do
+      QC.enqueue_retry_in(1, "puts", 5, 2)
+      sleep 1
+      job = QC.lock
+
+      expect(job[:q_name]).to eq("default")
+      expect(job[:method]).to eq("puts")
+      expect(job[:args][0]).to be(2)
+      expect(job[:remaining_retries]).to eq("5")
+    end
+  end
+end

data/spec/sample_jobs.rb

@@ -1,3 +1,5 @@
+require 'pg'
+
 class SomeException < RuntimeError
 end
 
@@ -12,7 +14,7 @@ module Jobs
       @queue = :low
       retry! on: SomeException, max: 5
 
-      def self.perform should_raise
+      def self.perform(should_raise)
         raise SomeException if should_raise
       end
     end
@@ -21,10 +23,11 @@ module Jobs
     class TestJobNoRetry < QueueClassicPlus::Base
       class Custom < RuntimeError
       end
+      disable_retries!
 
       @queue = :low
 
-      def self.perform should_raise
+      def self.perform(should_raise)
         raise Custom if should_raise
       end
     end
@@ -32,12 +35,36 @@ module Jobs
 
     class TestJob < QueueClassicPlus::Base
       @queue = :low
-      retry! on: SomeException, max: 5
+      retry! on: SomeException, max: 1
 
-      def self.perform should_raise
+      def self.perform(should_raise)
         raise SomeException if should_raise
       end
     end
 
+    class Exception < RuntimeError
+      attr_reader :original_exception
+
+      def initialize(e)
+        @original_exception = e
+      end
+    end
+
+    class ConnectionReapedTestJob < QueueClassicPlus::Base
+      @queue = :low
+      retry! on: Exception, max: 5
+
+      def self.perform
+        raise Exception.new(PG::UnableToSend.new)
+      end
+    end
+
+    class UniqueViolationTestJob < QueueClassicPlus::Base
+      @queue = :low
+
+      def self.perform
+        raise Exception.new(PG::UniqueViolation.new)
+      end
+    end
   end
 end

data/spec/spec_helper.rb

@@ -4,8 +4,11 @@ require 'timecop'
 require 'queue_classic_matchers'
 require_relative './sample_jobs'
 require_relative './helpers'
+require 'byebug'
 require 'pry'
+require 'ddtrace'
 
+ENV["QC_RAILS_DATABASE"] ||= "false" # test on QC::ConnAdapter by default
 ENV["DATABASE_URL"] ||= "postgres:///queue_classic_plus_test"
 
 RSpec.configure do |config|
@@ -20,5 +23,7 @@ RSpec.configure do |config|
 
   config.before(:each) do
     QC.default_conn_adapter.execute "TRUNCATE queue_classic_jobs;"
+    # Reset the default (memoized) queue instance between specs
+    QC.default_queue = nil
   end
 end

data/spec/worker_spec.rb

@@ -9,22 +9,22 @@ describe QueueClassicPlus::CustomWorker do
 
     it "record failures in the failed queue" do
       queue.enqueue("Kerklfadsjflaksj", 1, 2, 3)
-      failed_queue.count.should == 0
+      expect(failed_queue.count).to eq(0)
       worker.work
-      failed_queue.count.should == 1
+      expect(failed_queue.count).to eq(1)
       job = failed_queue.lock
-      job[:method].should == "Kerklfadsjflaksj"
-      job[:args].should == [1, 2, 3]
+      expect(job[:method]).to eq("Kerklfadsjflaksj")
+      expect(job[:args]).to eq([1, 2, 3])
       full_job = find_job(job[:id])
 
-      full_job['last_error'].should_not be_nil
+      expect(full_job['last_error']).to_not be_nil
     end
 
     it "records normal errors" do
       queue.enqueue("Jobs::Tests::TestJobNoRetry.perform", true)
-      failed_queue.count.should == 0
+      expect(failed_queue.count).to eq(0)
       worker.work
-      failed_queue.count.should == 1
+      expect(failed_queue.count).to eq(1)
     end
   end
 
@@ -42,47 +42,141 @@ describe QueueClassicPlus::CustomWorker do
         job_type.enqueue_perform(true)
       end.to change_queue_size_of(job_type).by(1)
 
-      Jobs::Tests::LockedTestJob.should have_queue_size_of(1)
-      failed_queue.count.should == 0
-      QueueClassicMatchers::QueueClassicRspec.find_by_args('low', 'Jobs::Tests::LockedTestJob._perform', [true]).first['remaining_retries'].should be_nil
+      expect(Jobs::Tests::LockedTestJob).to have_queue_size_of(1)
+      expect(failed_queue.count).to eq(0)
+      expect(QueueClassicMatchers::QueueClassicRspec.find_by_args('low', 'Jobs::Tests::LockedTestJob._perform', [true]).first['remaining_retries']).to be_nil
+
+      expect(QueueClassicPlus::Metrics).to receive(:increment).with('qc.retry', source: nil )
 
       Timecop.freeze do
         worker.work
 
-        failed_queue.count.should == 0 # not enqueued on Failed
-        QueueClassicMatchers::QueueClassicRspec.find_by_args('low', 'Jobs::Tests::LockedTestJob._perform', [true]).first['remaining_retries'].should eq "4"
-        Jobs::Tests::LockedTestJob.should have_scheduled(true).at(Time.now + described_class::BACKOFF_WIDTH) # should have scheduled a retry for later
+        expect(failed_queue.count).to eq(0) # not enqueued on Failed
+        expect(QueueClassicMatchers::QueueClassicRspec.find_by_args('low', 'Jobs::Tests::LockedTestJob._perform', [true]).first['remaining_retries']).to eq "4"
+        expect(Jobs::Tests::LockedTestJob).to have_scheduled(true).at(Time.now + described_class::BACKOFF_WIDTH) # should have scheduled a retry for later
       end
 
       Timecop.freeze(Time.now + (described_class::BACKOFF_WIDTH * 2)) do
         # the job should be re-enqueued with a decremented retry count
         jobs = QueueClassicMatchers::QueueClassicRspec.find_by_args('low', 'Jobs::Tests::LockedTestJob._perform', [true])
-        jobs.size.should == 1
+        expect(jobs.size).to eq(1)
         job = jobs.first
-        job['remaining_retries'].to_i.should == job_type.max_retries - 1
-        job['locked_by'].should be_nil
-        job['locked_at'].should be_nil
+        expect(job['remaining_retries'].to_i).to eq(job_type.max_retries - 1)
+        expect(job['locked_by']).to be_nil
+        expect(job['locked_at']).to be_nil
       end
     end
 
-    it "enqueues in the failed queue when retries have been exhausted" do
-      job_type.max_retries = 0
-      expect do
-        job_type.enqueue_perform(true)
-      end.to change_queue_size_of(job_type).by(1)
+    context 'when Rails is defined' do
+      require 'active_job'
+      require 'active_job/arguments'
 
-      Jobs::Tests::LockedTestJob.should have_queue_size_of(1)
-      failed_queue.count.should == 0
-      QueueClassicMatchers::QueueClassicRspec.find_by_args('low', 'Jobs::Tests::LockedTestJob._perform', [true]).first['remaining_retries'].should be_nil
+      before { stub_const('Rails', Struct.new(:logger).new(Logger.new(STDOUT))) }
+
+      it 'retries' do
+        expect do
+          job_type.enqueue_perform(:foo)
+        end.to change_queue_size_of(job_type).by(1)
+
+        expect(Jobs::Tests::LockedTestJob).to have_queue_size_of(1)
+        expect(failed_queue.count).to eq(0)
+        expect(QueueClassicMatchers::QueueClassicRspec.find_by_args('low', 'Jobs::Tests::LockedTestJob._perform', [:foo]).first['remaining_retries']).to be_nil
+
+        expect(QueueClassicPlus::Metrics).to receive(:increment).with('qc.retry', source: nil).twice
+
+        Timecop.freeze do
+          worker.work
+
+          expect(failed_queue.count).to eq(0) # not enqueued on Failed
+          expect(QueueClassicMatchers::QueueClassicRspec.find_by_args('low', 'Jobs::Tests::LockedTestJob._perform', [:foo]).first['remaining_retries']).to eq "4"
+          expect(Jobs::Tests::LockedTestJob).to have_scheduled(:foo).at(Time.now + described_class::BACKOFF_WIDTH) # should have scheduled a retry for later
+        end
+
+        Timecop.freeze(Time.now + (described_class::BACKOFF_WIDTH * 2)) do
+          # the job should be re-enqueued with a decremented retry count
+          jobs = QueueClassicMatchers::QueueClassicRspec.find_by_args('low', 'Jobs::Tests::LockedTestJob._perform', [:foo])
+          expect(jobs.size).to eq(1)
+          job = jobs.first
+          expect(job['remaining_retries'].to_i).to eq(job_type.max_retries - 1)
+          expect(job['locked_by']).to be_nil
+          expect(job['locked_at']).to be_nil
+        end
 
-      Timecop.freeze do
         worker.work
+      end
+    end
+
+    context 'when PG connection reaped during a job' do
+      before { Jobs::Tests::ConnectionReapedTestJob.enqueue_perform }
 
-        QueueClassicMatchers::QueueClassicRspec.find_by_args('failed_jobs', 'Jobs::Tests::LockedTestJob._perform', [true]).first['remaining_retries'].should be_nil
-        failed_queue.count.should == 1 # not enqueued on Failed
-        Jobs::Tests::LockedTestJob.should_not have_scheduled(true).at(Time.now + described_class::BACKOFF_WIDTH) # should have scheduled a retry for later
+      it 'retries' do
+        expect(QueueClassicPlus::Metrics).to receive(:increment).with('qc.force_retry', source: nil )
+        Timecop.freeze do
+          worker.work
+          expect(failed_queue.count).to eq 0
+          expect(QueueClassicMatchers::QueueClassicRspec.find_by_args('low', 'Jobs::Tests::ConnectionReapedTestJob._perform', []).first['remaining_retries']).to eq "4"
+        end
+      end
+
+      it 'ensures to rollback' do
+        allow(QC.default_conn_adapter).to receive(:execute).and_call_original
+        expect(QC.default_conn_adapter).to receive(:execute).with('ROLLBACK')
+        Timecop.freeze do
+          worker.work
+        end
+      end
+    end
+
+    context 'with a custom exception having max: 1 retry' do
+      before { Jobs::Tests::TestJob.enqueue_perform(true) }
+
+      it 'retries' do
+        expect(QueueClassicPlus::Metrics).to receive(:increment).with('qc.retry', source: nil )
+        Timecop.freeze do
+          worker.work
+          expect(failed_queue.count).to eq 0
+          expect(QueueClassicMatchers::QueueClassicRspec.find_by_args('low', 'Jobs::Tests::TestJob._perform', [true]).first['remaining_retries']).to eq "0"
+        end
+      end
+    end
+
+    context 'with non-connection based PG jobs' do
+      before { Jobs::Tests::UniqueViolationTestJob.enqueue_perform }
+
+      it 'sends the job to the failed jobs queue' do
+        Timecop.freeze do
+          worker.work
+        end
+        expect(failed_queue.count).to eq 1
+      end
+    end
+
+    context 'when retries have been exhausted' do
+      before do
+        job_type.max_retries = 0
+      end
+
+      after do
+        job_type.max_retries = 5
+      end
+
+      it 'enqueues in the failed queue' do
+        expect do
+          job_type.enqueue_perform(true)
+        end.to change_queue_size_of(job_type).by(1)
+
+        expect(Jobs::Tests::LockedTestJob).to have_queue_size_of(1)
+        expect(failed_queue.count).to eq(0)
+        expect(QueueClassicMatchers::QueueClassicRspec.find_by_args('low', 'Jobs::Tests::LockedTestJob._perform', [true]).first['remaining_retries']).to be_nil
+
+        Timecop.freeze do
+          worker.work
+
+          expect(QueueClassicMatchers::QueueClassicRspec.find_by_args('failed_jobs', 'Jobs::Tests::LockedTestJob._perform', [true]).first['remaining_retries']).to be_nil
+          expect(failed_queue.count).to eq(1) # not enqueued on Failed
+          expect(Jobs::Tests::LockedTestJob).to_not have_scheduled(true).at(Time.now + described_class::BACKOFF_WIDTH) # should have scheduled a retry for later
+        end
       end
     end
   end
 end
-

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: queue_classic_plus
 version: !ruby/object:Gem::Version
-  version: 1.0.0.alpha2
+  version: 4.0.0.alpha8
 platform: ruby
 authors:
 - Simon Mathieu
@@ -10,36 +10,36 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-21 00:00:00.000000000 Z
+date: 2021-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: queue_classic
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: 3.1.0
+        version: 4.0.0.pre.alpha1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: 3.1.0
+        version: 4.0.0.pre.alpha1
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -54,6 +54,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+- !ruby/object:Gem::Dependency
+  name: activejob
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: ''
 email:
 - simon.math@gmail.com
@@ -63,10 +91,11 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
+- ".github/dependabot.yml"
 - ".gitignore"
 - ".rspec"
 - ".rvmrc"
-- ".travis.yml"
 - Gemfile
 - Guardfile
 - LICENSE.txt
@@ -74,6 +103,7 @@ files:
 - Rakefile
 - lib/queue_classic_plus.rb
 - lib/queue_classic_plus/base.rb
+- lib/queue_classic_plus/datadog.rb
 - lib/queue_classic_plus/inflector.rb
 - lib/queue_classic_plus/inheritable_attr.rb
 - lib/queue_classic_plus/metrics.rb
@@ -89,8 +119,11 @@ files:
 - lib/rails/generators/qc_plus_job/templates/job_spec.rb.erb
 - queue_classic_plus.gemspec
 - spec/base_spec.rb
+- spec/datadog_spec.rb
 - spec/helpers.rb
 - spec/inflector_spec.rb
+- spec/new_relic_spec.rb
+- spec/queue_classic/queue_spec.rb
 - spec/sample_jobs.rb
 - spec/spec_helper.rb
 - spec/update_metrics_spec.rb
@@ -114,15 +147,17 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Useful extras for Queue Classic
 test_files:
 - spec/base_spec.rb
+- spec/datadog_spec.rb
 - spec/helpers.rb
 - spec/inflector_spec.rb
+- spec/new_relic_spec.rb
+- spec/queue_classic/queue_spec.rb
 - spec/sample_jobs.rb
 - spec/spec_helper.rb
 - spec/update_metrics_spec.rb

data/.travis.yml

@@ -1,8 +0,0 @@
-language: ruby
-before_script:
-  - psql -c 'create database queue_classic_plus_test;' -U postgres
-
-rvm:
-  - 2.0.0
-  - 2.1.5
-  - 2.2.0