queue_classic_plus 1.0.0.alpha2 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f5b1934a03303e576a999e71f978a68da13f0298
-  data.tar.gz: d1de5250aa7b2ddfe4114043d87e5a95986145f8
+  metadata.gz: 7d499c51f34d828e8abf2c5e405d82ea3ae49b19
+  data.tar.gz: 335713b0d79d1b54b9dc342a36c88c2008fccab2
 SHA512:
-  metadata.gz: ebadc9325c0307789f6a6216047a2c2cfe240d4364b613ee9eefc9a479224a9196813146b90ed251124cd5ad7a4966556c8fca8e1b48a490cfd806bf95124ff3
-  data.tar.gz: 619c736ed7b74f94841328acfa27926e2799fecc6589974d9b79de236c1f396ef126d1e80205df223e626536b69c1c0a834014e09e190b000635e571c2a8e677
+  metadata.gz: a529d8ca6f1a179f186f6b57bc90543bec9607a2c87e700db996a87301bea7073b353880ade9796d93d733573112f5545ef1e22440c29156127bd11edeb87212
+  data.tar.gz: c23fde5d11939355a2c8a90a92001f336da6ecf3c422aca1bd11ebed5e55f019b0d8fc5720495ad65439a95b79c11f77cd00f4433533a8d7baf38b19f807fcc7

data/.gitignore

@@ -21,3 +21,4 @@ tmp
 *.a
 mkmf.log
 tags
+.project

data/.travis.yml

@@ -1,4 +1,7 @@
 language: ruby
+before_install:
+  - gem update bundler
+install: bundle install --without development
 before_script:
   - psql -c 'create database queue_classic_plus_test;' -U postgres
 
@@ -6,3 +9,4 @@ rvm:
   - 2.0.0
   - 2.1.5
   - 2.2.0
+  - 2.3.1

data/Gemfile

@@ -12,6 +12,7 @@ group :development do
 end
 
 group :test do
+  gem 'rake'
   gem 'rspec'
   gem 'timecop'
 end

data/README.md

@@ -2,11 +2,11 @@
 
 [![Build Status](https://travis-ci.org/rainforestapp/queue_classic_plus.svg?branch=master)](https://travis-ci.org/rainforestapp/queue_classic_plus)
 
-[QueueClassic](https://github.com/QueueClassic/queue_classic) is a simple Postgresql back DB queue. However, it's a little too simple to use it as the main queueing system of a medium to large app.
+[queue_classic](https://github.com/QueueClassic/queue_classic) is a simple Postgresql backed DB queue. However, it's a little too simple to use it as the main queueing system of a medium to large app. This was developed at [Rainforest QA](https://www.rainforestqa.com/).
 
 QueueClassicPlus adds many lacking features to QueueClassic.
 
-- Standard job format
+- Standardized job format
 - Retry on specific exceptions
 - Singleton jobs
 - Metrics
@@ -101,11 +101,21 @@ Jobs::UpdateMetrics.do 'type_a' # does not enqueues job since it's already queue
 Jobs::UpdateMetrics.do 'type_b' # enqueues job as the arguments are different.
 ```
 
+#### Transactions
+
+By default, all QueueClassicPlus jobs are executed in a PostgreSQL
+transaction. This decision was made because most jobs are usually
+pretty small and it's preferable to have all the benefits of the
+transaction. You can optionally specify a postgres statement timeout
+(in seconds) for all transactions with the environment variable
+`POSTGRES_STATEMENT_TIMEOUT`.
+
+You can disable this feature on a per job basis in the following way:
+
 ```ruby
 class Jobs::NoTransaction < QueueClassicPlus::Base
   # Don't run the perform method in a transaction
   skip_transaction!
-
   @queue = :low
 
   def self.perform(user_id)
@@ -114,12 +124,6 @@ class Jobs::NoTransaction < QueueClassicPlus::Base
 end
 ```
 
-#### Transaction
-
-By default, all QueueClassicPlus jobs are executed in a PostgreSQL transaction. This decision was made because most jobs are usually pretty small and it's preferable to have all the benefits of the transaction.
-
-You can disable this feature on a per job basis in the follwing way:
-
 ## Advanced configuration
 
 If you want to log exceptions in your favorite exception tracker. You can configured it like sso:

data/lib/queue_classic_plus.rb

@@ -26,7 +26,7 @@ module QueueClassicPlus
   end
 
   def self.exception_handler
-    @exception_handler ||= -> (exception, job) { nil }
+    @exception_handler ||= ->(exception, job) { nil }
   end
 
   def self.exception_handler=(handler)

data/lib/queue_classic_plus/base.rb

@@ -10,11 +10,16 @@ module QueueClassicPlus
     inheritable_attr :skip_transaction
     inheritable_attr :retries_on
     inheritable_attr :max_retries
+    inheritable_attr :disable_retries
 
-    self.max_retries = 0
+    self.max_retries = 5
     self.retries_on = {}
+    self.disable_retries = false
 
     def self.retry!(on: RuntimeError, max: 5)
+      if self.disable_retries
+        raise 'retry! should not be used in conjuction with disable_retries!'
+      end
       Array(on).each {|e| self.retries_on[e] = true}
       self.max_retries = max
     end
@@ -23,6 +28,14 @@ module QueueClassicPlus
       self.retries_on[exception.class] || self.retries_on.keys.any? {|klass| exception.is_a? klass}
     end
 
+    def self.disable_retries!
+      unless self.retries_on.empty?
+        raise 'disable_retries! should not be enabled in conjunction with retry!'
+      end
+
+      self.disable_retries = true
+    end
+
     def self.lock!
       self.locked = true
     end
@@ -89,10 +102,13 @@ module QueueClassicPlus
     def self._perform(*args)
       Metrics.timing("qu_perform_time", source: librato_key) do
         if skip_transaction
-          perform *args
+          perform(*args)
         else
           transaction do
-            perform *args
+            # .to_i defaults to 0, which means no timeout in postgres
+            timeout = ENV['POSTGRES_STATEMENT_TIMEOUT'].to_i * 1000
+            execute "SET LOCAL statement_timeout = #{timeout}"
+            perform(*args)
           end
         end
       end

data/lib/queue_classic_plus/inheritable_attr.rb

@@ -28,7 +28,9 @@ module QueueClassicPlus
       end
 
       def self.uncloneable
-        [Symbol, TrueClass, FalseClass, NilClass]
+        tmp = [Symbol, TrueClass, FalseClass, NilClass]
+        tmp += [Fixnum, Bignum] if RUBY_VERSION < '2.4.0'
+        tmp
       end
     end
   end

data/lib/queue_classic_plus/metrics.rb

@@ -7,7 +7,7 @@ module QueueClassicPlus
 
   class Metrics
     def self.timing(*args, &block)
-      provider.timing *args, &block
+      provider.timing(*args, &block)
     end
 
     def self.increment(*args)

data/lib/queue_classic_plus/queue_classic/queue.rb

@@ -1,11 +1,34 @@
 module QC
   class Queue
+
     def enqueue_retry_in(seconds, method, remaining_retries, *args)
       QC.log_yield(:measure => 'queue.enqueue') do
         s = "INSERT INTO #{TABLE_NAME} (q_name, method, args, scheduled_at, remaining_retries)
              VALUES ($1, $2, $3, now() + interval '#{seconds.to_i} seconds', $4)"
-        res = conn_adapter.execute(s, name, method, JSON.dump(args), remaining_retries)
+
+        conn_adapter.execute(s, name, method, JSON.dump(args), remaining_retries)
       end
     end
+
+    def lock
+      QC.log_yield(:measure => 'queue.lock') do
+        s = "SELECT * FROM lock_head($1, $2)"
+        if r = conn_adapter.execute(s, name, top_bound)
+          {}.tap do |job|
+            job[:id] = r["id"]
+            job[:q_name] = r["q_name"]
+            job[:method] = r["method"]
+            job[:args] = JSON.parse(r["args"])
+            job[:remaining_retries] = r["remaining_retries"]
+            if r["scheduled_at"]
+              job[:scheduled_at] = Time.parse(r["scheduled_at"])
+              ttl = Integer((Time.now - job[:scheduled_at]) * 1000)
+              QC.measure("time-to-lock=#{ttl}ms source=#{name}")
+            end
+          end
+        end
+      end
+    end
+
   end
-end
+end

data/lib/queue_classic_plus/tasks/work.rake

@@ -2,13 +2,21 @@ namespace :qc_plus do
   desc "Start a new worker for the (default or $QUEUE) queue"
   task :work  => :environment do
     puts "Starting up worker for queue #{ENV['QUEUE']}"
+
+    if defined? Raven
+      Raven.configure do |config|
+        # ActiveRecord::RecordNotFound is ignored by Raven by default,
+        # which shouldn't happen in background jobs.
+        config.excluded_exceptions = []
+      end
+    end
+
     @worker = QueueClassicPlus::CustomWorker.new
 
     trap('INT') do
       $stderr.puts("Received INT. Shutting down.")
       if !@worker.running
-        $stderr.puts("Worker has stopped running. Exit.")
-        exit(1)
+        $stderr.puts("Worker has already stopped running.")
       end
       @worker.stop
     end
@@ -19,5 +27,6 @@ namespace :qc_plus do
     end
 
     @worker.start
+    $stderr.puts 'Shut down successfully'
   end
 end

data/lib/queue_classic_plus/version.rb

@@ -1,3 +1,3 @@
 module QueueClassicPlus
-  VERSION = "1.0.0.alpha2"
+  VERSION = "1.0.0"
 end

data/lib/queue_classic_plus/worker.rb

@@ -1,5 +1,9 @@
+require 'pg'
+require 'queue_classic'
+
 module QueueClassicPlus
   class CustomWorker < QC::Worker
+    CONNECTION_ERRORS = [PG::UnableToSend, PG::ConnectionBad].freeze
     BACKOFF_WIDTH = 10
     FailedQueue = QC::Queue.new("failed_jobs")
 
@@ -14,20 +18,34 @@ module QueueClassicPlus
 
     def handle_failure(job, e)
       QueueClassicPlus.logger.info "Handling exception #{e.message} for job #{job[:id]}"
+
+      force_retry = false
+      if connection_error?(e)
+        # If we've got here, unfortunately ActiveRecord's rollback mechanism may
+        # not have kicked in yet and we might be in a failed transaction. To be
+        # *absolutely* sure the retry/failure gets enqueued, we do a rollback
+        # just in case (and if we're not in a transaction it will be a no-op).
+        QueueClassicPlus.logger.info "Reset connection for job #{job[:id]}"
+        @conn_adapter.connection.reset
+        @conn_adapter.execute 'ROLLBACK'
+
+        # We definitely want to retry because the connection was lost mid-task.
+        force_retry = true
+      end
+
       klass = job_klass(job)
 
+      if force_retry && !(klass.respond_to?(:disable_retries) && klass.disable_retries)
+        Metrics.increment("qc.force_retry", source: @q_name)
+
+        retry_with_remaining(klass, job, e, 0)
       # The mailers doesn't have a retries_on?
-      if klass && klass.respond_to?(:retries_on?) && klass.retries_on?(e)
-        remaining_retries = job[:remaining_retries] || klass.max_retries
-        remaining_retries -= 1
-
-        if remaining_retries > 0
-          klass.restart_in((klass.max_retries - remaining_retries) * BACKOFF_WIDTH,
-                           remaining_retries,
-                           *job[:args])
-        else
-          enqueue_failed(job, e)
-        end
+      elsif klass && klass.respond_to?(:retries_on?) && klass.retries_on?(e)
+        Metrics.increment("qc.retry", source: @q_name)
+
+        backoff = (max_retries(klass) - remaining_retries(klass, job)) * BACKOFF_WIDTH
+
+        retry_with_remaining(klass, job, e, backoff)
       else
         enqueue_failed(job, e)
       end
@@ -36,6 +54,25 @@ module QueueClassicPlus
     end
 
     private
+
+    def retry_with_remaining(klass, job, e, backoff)
+      @remaining_retries = remaining_retries(klass, job)
+
+      if @remaining_retries > 0
+        klass.restart_in(backoff, @remaining_retries, *job[:args])
+      else
+        enqueue_failed(job, e)
+      end
+    end
+
+    def max_retries(klass)
+      klass.respond_to?(:max_retries) ? klass.max_retries : 5
+    end
+
+    def remaining_retries(klass, job)
+      @remaining_retries ? @remaining_retries - 1 : (job[:remaining_retries] || max_retries(klass)).to_i - 1
+    end
+
     def job_klass(job)
       begin
         Object.const_get(job[:method].split('.')[0])
@@ -43,5 +80,11 @@ module QueueClassicPlus
         nil
       end
     end
+
+    def connection_error?(e)
+      CONNECTION_ERRORS.any? { |klass| e.kind_of? klass } ||
+        (e.respond_to?(:original_exception) &&
+         CONNECTION_ERRORS.any? { |klass| e.original_exception.kind_of? klass })
+    end
   end
 end

data/spec/queue_classic/queue_spec.rb

@@ -0,0 +1,23 @@
+require 'spec_helper'
+
+describe QC do
+
+  describe ".lock" do
+
+    context "lock" do
+
+      it "should lock the job with remaining_retries" do
+        QC.enqueue_retry_in(1, "puts", 5, 2)
+        sleep 1
+        job = QC.lock
+
+        expect(job[:q_name]).to eq("default")
+        expect(job[:method]).to eq("puts")
+        expect(job[:args][0]).to be(2)
+        expect(job[:remaining_retries]).to eq("5")
+      end
+    end
+
+  end
+
+end

data/spec/sample_jobs.rb

@@ -1,3 +1,5 @@
+require 'pg'
+
 class SomeException < RuntimeError
 end
 
@@ -21,6 +23,7 @@ module Jobs
     class TestJobNoRetry < QueueClassicPlus::Base
       class Custom < RuntimeError
       end
+      disable_retries!
 
       @queue = :low
 
@@ -39,5 +42,29 @@ module Jobs
       end
     end
 
+    class Exception < RuntimeError
+      attr_reader :original_exception
+
+      def initialize(e)
+        @original_exception = e
+      end
+    end
+
+    class ConnectionReapedTestJob < QueueClassicPlus::Base
+      @queue = :low
+      retry! on: Exception, max: 5
+
+      def self.perform
+        raise Exception.new(PG::UnableToSend.new)
+      end
+    end
+
+    class UniqueViolationTestJob < QueueClassicPlus::Base
+      @queue = :low
+
+      def self.perform
+        raise Exception.new(PG::UniqueViolation.new)
+      end
+    end
   end
 end

data/spec/worker_spec.rb

@@ -46,6 +46,8 @@ describe QueueClassicPlus::CustomWorker do
       failed_queue.count.should == 0
       QueueClassicMatchers::QueueClassicRspec.find_by_args('low', 'Jobs::Tests::LockedTestJob._perform', [true]).first['remaining_retries'].should be_nil
 
+      QueueClassicPlus::Metrics.should_receive(:increment).with('qc.retry', source: nil )
+
       Timecop.freeze do
         worker.work
 
@@ -65,24 +67,64 @@ describe QueueClassicPlus::CustomWorker do
       end
     end
 
-    it "enqueues in the failed queue when retries have been exhausted" do
-      job_type.max_retries = 0
-      expect do
-        job_type.enqueue_perform(true)
-      end.to change_queue_size_of(job_type).by(1)
+    context 'when PG connection reaped during a job' do
+      before { Jobs::Tests::ConnectionReapedTestJob.enqueue_perform }
 
-      Jobs::Tests::LockedTestJob.should have_queue_size_of(1)
-      failed_queue.count.should == 0
-      QueueClassicMatchers::QueueClassicRspec.find_by_args('low', 'Jobs::Tests::LockedTestJob._perform', [true]).first['remaining_retries'].should be_nil
+      it 'retries' do
+        QueueClassicPlus::Metrics.should_receive(:increment).with('qc.force_retry', source: nil )
+        Timecop.freeze do
+          worker.work
+          expect(failed_queue.count).to eq 0
+          QueueClassicMatchers::QueueClassicRspec.find_by_args('low', 'Jobs::Tests::ConnectionReapedTestJob._perform', []).first['remaining_retries'].should eq "4"
+        end
+      end
 
-      Timecop.freeze do
-        worker.work
+      it 'ensures to rollback' do
+        allow(QC.default_conn_adapter).to receive(:execute).and_call_original
+        expect(QC.default_conn_adapter).to receive(:execute).with('ROLLBACK')
+        Timecop.freeze do
+          worker.work
+        end
+      end
+    end
 
-        QueueClassicMatchers::QueueClassicRspec.find_by_args('failed_jobs', 'Jobs::Tests::LockedTestJob._perform', [true]).first['remaining_retries'].should be_nil
-        failed_queue.count.should == 1 # not enqueued on Failed
-        Jobs::Tests::LockedTestJob.should_not have_scheduled(true).at(Time.now + described_class::BACKOFF_WIDTH) # should have scheduled a retry for later
+    context 'with non-connection based PG jobs' do
+      before { Jobs::Tests::UniqueViolationTestJob.enqueue_perform }
+
+      it 'sends the job to the failed jobs queue' do
+        Timecop.freeze do
+          worker.work
+        end
+        expect(failed_queue.count).to eq 1
+      end
+    end
+
+    context 'when retries have been exhausted' do
+      before do
+        job_type.max_retries = 0
+      end
+
+      after do
+        job_type.max_retries = 5
+      end
+
+      it 'enqueues in the failed queue' do
+        expect do
+          job_type.enqueue_perform(true)
+        end.to change_queue_size_of(job_type).by(1)
+
+        Jobs::Tests::LockedTestJob.should have_queue_size_of(1)
+        failed_queue.count.should == 0
+        QueueClassicMatchers::QueueClassicRspec.find_by_args('low', 'Jobs::Tests::LockedTestJob._perform', [true]).first['remaining_retries'].should be_nil
+
+        Timecop.freeze do
+          worker.work
+
+          QueueClassicMatchers::QueueClassicRspec.find_by_args('failed_jobs', 'Jobs::Tests::LockedTestJob._perform', [true]).first['remaining_retries'].should be_nil
+          failed_queue.count.should == 1 # not enqueued on Failed
+          Jobs::Tests::LockedTestJob.should_not have_scheduled(true).at(Time.now + described_class::BACKOFF_WIDTH) # should have scheduled a retry for later
+        end
       end
     end
   end
 end
-

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: queue_classic_plus
 version: !ruby/object:Gem::Version
-  version: 1.0.0.alpha2
+  version: 1.0.0
 platform: ruby
 authors:
 - Simon Mathieu
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-21 00:00:00.000000000 Z
+date: 2018-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: queue_classic
@@ -91,6 +91,7 @@ files:
 - spec/base_spec.rb
 - spec/helpers.rb
 - spec/inflector_spec.rb
+- spec/queue_classic/queue_spec.rb
 - spec/sample_jobs.rb
 - spec/spec_helper.rb
 - spec/update_metrics_spec.rb
@@ -110,12 +111,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Useful extras for Queue Classic
@@ -123,6 +124,7 @@ test_files:
 - spec/base_spec.rb
 - spec/helpers.rb
 - spec/inflector_spec.rb
+- spec/queue_classic/queue_spec.rb
 - spec/sample_jobs.rb
 - spec/spec_helper.rb
 - spec/update_metrics_spec.rb