queue_classic_plus 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8acdaf6cd41362f50d7ddb9c50fa87555c00adc9
+  data.tar.gz: 58c26d31a05c2390504975b6a7513179f0397ede
+SHA512:
+  metadata.gz: 453a051c64c4c539f488f9ea7f0c01da9ac326de11891746b6b0abea14bc4103bb347afd4020bedb0b37c0262d1c1a983659f39c0b35909521db6af2951dcacb
+  data.tar.gz: 37263c41d6929a351e74920a8abe5d2d4d3bbc3c5c9bdb941f4cd2e6758ef2443ddf580dd16298124809ca0232ce7df02b3533fac75fef8fa911f21c45ecb597

data/.gitignore

@@ -0,0 +1,23 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+tags

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.rvmrc

@@ -0,0 +1 @@
+rvm use 2.1.3@queue_classic_plus --create

data/Gemfile

@@ -0,0 +1,11 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in queue_classic_plus.gemspec
+gemspec
+
+
+gem 'rspec'
+gem 'pg'
+gem 'timecop'
+gem 'queue_classic-later', github: 'jipiboily/queue_classic-later', branch: "add-qc-3-to-custom-columns" # This is until the 3.0 work is merged into original repo
+gem 'queue_classic_matchers'

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 CLDRDR Inc.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,97 @@
+# QueueClassicPlus
+
+QueueClassic is a simple Postgresql back DB queue. However, it's a little too simple to use it as the main queueing system of a medium to large app.
+
+QueueClassicPlus adds many lacking features to QueueClassic.
+
+- Standard job format
+- Retry on specific exceptions
+- Singleton jobs
+- Metrics
+- Error logging / handling
+- Transactions
+
+## Installation
+
+Add these line to your application's Gemfile:
+
+    gem 'queue_classic_plus'
+    gem "queue_classic-later", github: "dpiddy/queue_classic-later"
+
+And then execute:
+
+    $ bundle
+
+Run the migration
+
+```ruby
+  QueueClassicPlus.migrate
+```
+
+## Usage
+
+### Create a new job
+
+```bash
+rails g my_job test_job
+```
+
+```ruby
+# /app/jobs/my_job.rb
+class Jobs::MyJob < QueueClassicPlus::Base
+  # Specified the queue name
+  @queue = :low
+
+  # Extry up to 5 times when SomeException is raised
+  retry! on: SomeException, max: 5
+
+  def self.perform(a, b)
+    # ...
+  end
+end
+
+# In your code, you can enqueue this task like so:
+Jobs::MyJob.do(1, "foo")
+
+# You can also schedule a job in the future by doing
+
+Jobs::MyJob.enqueue_perform_in(1.hour, 1, "foo")
+```
+
+### Run the QueueClassicPlus worker
+
+```
+QUEUE=low bundle exec qc_plus:work
+```
+
+## Advance configuration
+
+If you want to log exceptions in your favorite exception tracker. You can configured it like sso:
+
+```ruby
+QueueClassicPlus.exception_handler = -> (exception, job) do
+  Raven.capture_exception(exception, extra: {job: job, env: ENV})
+end
+```
+
+If you use Librato, we push useful metrics directly to them.
+
+Push metrics to your metric provider (only Librato is supported for now).
+
+```ruby
+QueueClassicPlus.update_metrics
+```
+
+Call this is a cron job or something similar.
+
+## TODO
+
+- Remove dep on ActiveRecord
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/queue_classic_plus/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/lib/queue_classic_plus/base.rb

@@ -0,0 +1,127 @@
+module QueueClassicPlus
+  class Base
+    def self.queue
+      QC::Queue.new(@queue)
+    end
+
+    class_attribute :locked
+    class_attribute :skip_transaction
+    class_attribute :retries_on
+    class_attribute :max_retries
+
+    self.max_retries = 0
+    self.retries_on = {}
+
+    def self.retry!(on: RuntimeError, max: 5)
+      Array(on).each {|e| self.retries_on[e] = true}
+      self.max_retries = max
+    end
+
+    def self.retries_on? exception
+      self.retries_on[exception.class] || self.retries_on.keys.any? {|klass| exception.is_a? klass}
+    end
+
+    def self.lock!
+      self.locked = true
+    end
+
+    def self.skip_transaction!
+      self.skip_transaction = true
+    end
+
+    def self.locked?
+      !!self.locked
+    end
+
+    def self.logger
+      QueueClassicPlus.logger
+    end
+
+    def self.can_enqueue?(method, *args)
+      if locked?
+        lock_key = [@queue, method, args].to_json
+        max_lock_time = ENV.fetch("QUEUE_CLASSIC_MAX_LOCK_TIME", 10 * 60).to_i
+
+        ActiveRecord::Base.with_advisory_lock(lock_key, 0.1) do
+          q = "SELECT COUNT(1) AS count
+               FROM
+               (
+                 (
+                 SELECT 1
+                 FROM queue_classic_jobs
+                 WHERE q_name = $1 AND method = $2 AND args::text = $3::text
+                   AND (locked_at IS NULL OR locked_at > current_timestamp - interval '#{max_lock_time} seconds')
+                 LIMIT 1
+                 )
+                 
+                 UNION
+                 
+                 (
+                 SELECT 1
+                 FROM queue_classic_later_jobs
+                 WHERE q_name = $4 AND method = $5 AND args = $6::text
+                 LIMIT 1
+                 )
+               )
+               AS x"
+
+          result = QC.default_conn_adapter.execute(q, @queue, method, args.to_json, @queue, method, args.to_json)
+          result['count'].to_i == 0
+        end
+      else
+        true
+      end
+    end
+
+    def self.enqueue(method, *args)
+      if can_enqueue?(method, *args)
+        queue.enqueue(method, *args)
+      end
+    end
+
+    def self.enqueue_perform(*args)
+      enqueue("#{self.to_s}._perform", *args)
+    end
+
+    def self.enqueue_perform_in(time, *args)
+      raise "Can't enqueue in the future for locked jobs" if locked?
+      queue.enqueue_in(time, "#{self.to_s}._perform", *args)
+    end
+
+    def self.restart_in(time, remaining_retries, *args)
+      queue.enqueue_in_with_custom(time, "#{self.to_s}._perform", {'remaining_retries' => remaining_retries}, *args)
+    end
+
+    def self.do(*args)
+      Metrics.timing("qc_enqueue_time", source: librato_key) do
+        enqueue_perform(*args)
+      end
+    end
+
+    def self._perform(*args)
+      Metrics.timing("qu_perform_time", source: librato_key) do
+        if skip_transaction
+          perform *args
+        else
+          transaction do
+            perform *args
+          end
+        end
+      end
+    end
+
+    def self.librato_key
+      (self.name || "").underscore.gsub(/\//, ".")
+    end
+
+    def self.transaction(options = {}, &block)
+      ActiveRecord::Base.transaction(options, &block)
+    end
+
+    # Debugging
+    def self.list
+      q = "SELECT * FROM queue_classic_jobs WHERE q_name = '#{@queue}'"
+      ActiveRecord::Base.connection.execute(q).to_a
+    end
+  end
+end

data/lib/queue_classic_plus/metrics.rb

@@ -0,0 +1,29 @@
+module QueueClassicPlus
+  class Empty
+    def self.method_missing(*)
+      yield if block_given?
+    end
+  end
+
+  class Metrics
+    def self.timing(*args, &block)
+      provider.timing *args, &block
+    end
+
+    def self.increment(*args)
+      provider.increment(*args)
+    end
+
+    def self.measure(*args)
+      provider.measure(*args)
+    end
+
+    def self.provider
+      if defined?(Librato)
+        Librato
+      else
+        Empty
+      end
+    end
+  end
+end

data/lib/queue_classic_plus/railtie.rb

@@ -0,0 +1,12 @@
+require 'rails'
+
+module MyPlugin
+  class Railtie < Rails::Railtie
+    railtie_name :queue_classic_plus
+
+    rake_tasks do
+      load "queue_classic_plus/tasks/work.rake"
+    end
+  end
+end
+

data/lib/queue_classic_plus/tasks/work.rake

@@ -0,0 +1,23 @@
+namespace :qc_plus do
+  desc "Start a new worker for the (default or $QUEUE) queue"
+  task :work  => :environment do
+    puts "Starting up worker for queue #{ENV['QUEUE']}"
+    @worker = QueueClassicPlus::CustomWorker.new
+
+    trap('INT') do
+      $stderr.puts("Received INT. Shutting down.")
+      if !@worker.running
+        $stderr.puts("Worker has stopped running. Exit.")
+        exit(1)
+      end
+      @worker.stop
+    end
+
+    trap('TERM') do
+      $stderr.puts("Received Term. Shutting down.")
+      @worker.stop
+    end
+
+    @worker.start
+  end
+end

data/lib/queue_classic_plus/update_metrics.rb

@@ -0,0 +1,62 @@
+module QueueClassicPlus
+  module UpdateMetrics
+    def self.update
+      {
+        "queued" => "queue_classic_jobs",
+        "scheduled" => "queue_classic_later_jobs",
+      }.each do |type, table|
+        next unless ActiveRecord::Base.connection.table_exists?(table)
+        q = "SELECT q_name, COUNT(1) FROM #{table} GROUP BY q_name"
+        results = execute(q)
+
+        # Log individual queue sizes
+        results.each do |h|
+          Metrics.measure("qc.jobs_#{type}", h.fetch('count').to_i, source: h.fetch('q_name'))
+        end
+      end
+
+      # Log oldest locked_at and created_at
+      ['locked_at', 'created_at'].each do |column|
+        age = max_age(column)
+        Metrics.measure("qc.max_#{column}", age)
+      end
+
+      # Log oldes unlocked jobs
+      age = max_age("created_at", "locked_at IS NULL")
+      Metrics.measure("qc.max_created_at.unlocked", age)
+
+      if ActiveRecord::Base.connection.table_exists?('queue_classic_later_jobs')
+        lag = execute("SELECT MAX(EXTRACT(EPOCH FROM now() - not_before)) AS lag 
+                FROM queue_classic_later_jobs").first
+                lag = lag ? lag['lag'] : 0
+
+        Metrics.measure("qc.jobs_delayed.lag", lag.to_f)
+
+        nb_late = execute("SELECT COUNT(1) 
+           FROM queue_classic_later_jobs 
+           WHERE not_before < NOW()").first
+        nb_late = nb_late ? nb_late['count'] : 0
+
+        Metrics.measure("qc.jobs_delayed.late_count", nb_late.to_i)
+      end
+    end
+
+    private
+    def self.max_age(column, *conditions)
+      conditions.unshift "q_name != '#{::QueueClassicPlus::CustomWorker::FailedQueue.name}'"
+
+      q = "SELECT EXTRACT(EPOCH FROM now() - #{column}) AS age_in_seconds 
+           FROM queue_classic_jobs
+           WHERE #{conditions.join(" AND ")}
+           ORDER BY age_in_seconds DESC
+           "
+       age_info = execute(q).first
+
+       age_info ? age_info['age_in_seconds'].to_i : 0
+    end
+
+    def self.execute(q)
+      ActiveRecord::Base.connection.execute(q)
+    end
+  end
+end

data/lib/queue_classic_plus/version.rb

@@ -0,0 +1,3 @@
+module QueueClassicPlus
+  VERSION = "0.0.2"
+end

data/lib/queue_classic_plus/worker.rb

@@ -0,0 +1,54 @@
+module QueueClassicPlus
+  class CustomWorker < QC::Worker
+    class QueueClassicJob < ActiveRecord::Base
+    end
+
+    BACKOFF_WIDTH = 10
+    FailedQueue = QC::Queue.new("failed_jobs")
+
+    def enqueue_failed(job, e)
+      ActiveRecord::Base.transaction do
+        FailedQueue.enqueue(job[:method], *job[:args])
+        new_job = QueueClassicJob.order(:id).where(q_name: 'failed_jobs').last
+        new_job.last_error = if e.backtrace then ([e.message] + e.backtrace ).join("\n") else e.message end
+        new_job.save!
+      end
+
+      QueueClassicPlus.exception_handler.call(e, job)
+      Metrics.increment("qc.errors", source: @q_name)
+    end
+
+    def handle_failure(job, e)
+      QueueClassicPlus.logger.info "Handling exception #{e.message} for job #{job[:id]}"
+      klass = job_klass(job)
+
+      model = QueueClassicJob.find(job[:id])
+
+      # The mailers doesn't have a retries_on?
+      if klass && klass.respond_to?(:retries_on?) && klass.retries_on?(e)
+        remaining_retries = model.remaining_retries || klass.max_retries
+        remaining_retries -= 1
+
+        if remaining_retries > 0
+          klass.restart_in((klass.max_retries - remaining_retries) * BACKOFF_WIDTH,
+                           remaining_retries,
+                           *job[:args])
+        else
+          enqueue_failed(job, e)
+        end
+      else
+        enqueue_failed(job, e)
+      end
+      model.destroy
+    end
+
+    private
+    def job_klass(job)
+      begin
+        Object.const_get(job[:method].split('.')[0])
+      rescue NameError
+        nil
+      end
+    end
+  end
+end

data/lib/queue_classic_plus.rb

@@ -0,0 +1,49 @@
+require "queue_classic"
+require "queue_classic/later"
+require "active_record"
+require "with_advisory_lock"
+require "active_support/core_ext/class/attribute"
+
+require "queue_classic_plus/version"
+require "queue_classic_plus/metrics"
+require "queue_classic_plus/update_metrics"
+require "queue_classic_plus/base"
+require "queue_classic_plus/worker"
+
+module QueueClassicPlus
+  require 'queue_classic_plus/railtie' if defined?(Rails)
+
+  def self.migrate(c = QC::default_conn_adapter.connection)
+    conn = QC::ConnAdapter.new(c)
+    conn.execute("ALTER TABLE queue_classic_jobs ADD COLUMN last_error TEXT")
+    conn.execute("ALTER TABLE queue_classic_jobs ADD COLUMN remaining_retries INTEGER")
+    conn.execute("ALTER TABLE queue_classic_later_jobs ADD COLUMN remaining_retries INTEGER")
+  end
+
+  def self.demigrate(c = QC::default_conn_adapter.connection)
+    conn = QC::ConnAdapter.new(c)
+    conn.execute("ALTER TABLE queue_classic_jobs DROP COLUMN last_error")
+    conn.execute("ALTER TABLE queue_classic_jobs DROP COLUMN remaining_retries")
+    conn.execute("ALTER TABLE queue_classic_later_jobs DROP COLUMN remaining_retries")
+  end
+
+  def self.exception_handler
+    @exception_handler ||= -> (exception, job) { nil }
+  end
+
+  def self.exception_handler=(handler)
+    @exception_handler = handler
+  end
+
+  def self.update_metrics
+    UpdateMetrics.update
+  end
+
+  def self.logger
+    @logger ||= defined?(Rails) ? Rails.logger : Logger.new(STDOUT)
+  end
+
+  def self.logger=(l)
+    @logger = l
+  end
+end

data/lib/rails/generators/qc_plus_job/qc_plus_job_generator.rb

@@ -0,0 +1,11 @@
+class QcPlusJobGenerator < Rails::Generators::NamedBase
+  source_root File.expand_path('../templates', __FILE__)
+
+  def generate_lib
+    template "job.rb.erb", File.join("app", "jobs", class_path, "#{file_name}.rb")
+    if defined?(RSpec)
+      template "job_spec.rb.erb", File.join("spec", "jobs", class_path, "#{file_name}_spec.rb")
+    end
+  end
+end
+

data/lib/rails/generators/qc_plus_job/templates/job.rb.erb

@@ -0,0 +1,8 @@
+<% module_namespacing do -%>
+class Jobs::<%= class_name %> < QueueClassicPlus::Base
+  @queue = :low
+
+  def self.perform
+  end
+end
+<% end -%>

data/lib/rails/generators/qc_plus_job/templates/job_spec.rb.erb

@@ -0,0 +1,13 @@
+<% module_namespacing do -%>
+describe Jobs::<%= class_name %>, type: :job do
+  it_should_behave_like "a queueable class"
+
+  describe ".perform" do
+    it "does not die" do
+      expect do
+        described_class.perform
+      end.to_not raise_error
+    end
+  end
+end
+<% end -%>

data/queue_classic_plus.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'queue_classic_plus/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "queue_classic_plus"
+  spec.version       = QueueClassicPlus::VERSION
+  spec.authors       = ["Simon Mathieu", "Russell Smith", "Jean-Philippe Boily"]
+  spec.email         = ["simon.math@gmail.com", "russ@rainforestqa.com", "j@jipi.ca"]
+  spec.summary       = %q{Useful extras for Queue Classic}
+  spec.description   = %q{}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "queue_classic"
+  spec.add_dependency "queue_classic-later"
+  spec.add_dependency "activerecord", "> 3.0"
+  spec.add_dependency "activesupport", "> 3.0"
+  spec.add_dependency "with_advisory_lock"
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+end

data/spec/base_spec.rb

@@ -0,0 +1,156 @@
+
+describe QueueClassicPlus::Base do
+  context "A child of QueueClassicPlus::Base" do
+    context "that is locked" do
+      subject do
+        Class.new(QueueClassicPlus::Base) do
+          @queue = :test
+          lock!
+        end
+      end
+
+      it "does not allow multiple enqueues" do
+        subject.do
+        subject.do
+        subject.should have_queue_size_of(1)
+      end
+
+      it "does allow multiple enqueues if something got locked for too long" do
+        subject.do
+        ActiveRecord::Base.connection.execute "
+          UPDATE queue_classic_jobs SET locked_at = '#{1.day.ago.to_s}' WHERE q_name = 'test'
+        "
+        subject.do
+        subject.should have_queue_size_of(2)
+      end
+    end
+
+    context "with default settings" do
+      subject do
+        Class.new(QueueClassicPlus::Base) do
+          @queue = :test
+
+          def self.perform
+          end
+
+          def self.name
+            "Funky::Name"
+          end
+        end
+      end
+
+      it "calls perform in a transaction" do
+        ActiveRecord::Base.should_receive(:transaction).and_call_original
+        subject._perform 
+      end
+
+      it "measures the time" do
+        QueueClassicPlus::Metrics.should_receive(:timing).with("qu_perform_time", {source: "funky.name"}).and_call_original
+        subject._perform 
+      end
+    end
+
+    context "skips transaction" do
+      subject do
+        Class.new(QueueClassicPlus::Base) do
+          @queue = :test
+          skip_transaction!
+
+          def self.perform
+          end
+        end
+      end
+
+      it "calls perform outside of a transaction" do
+        ActiveRecord::Base.should_not_receive(:transaction)
+        subject._perform 
+      end
+    end
+
+    context "retries on single exception" do
+      subject do
+        Class.new(QueueClassicPlus::Base) do
+          @queue = :test
+          retry! on: SomeException, max: 5
+          skip_transaction!
+
+          def self.perform
+          end
+        end
+      end
+
+      it "retries on specified exception" do
+        subject.retries_on?(SomeException.new).should be(true)
+      end
+
+      it "does not retry on unspecified exceptions" do
+        subject.retries_on?(RuntimeError).should be(false)
+      end
+
+      it "sets max retries" do
+        subject.max_retries.should == 5
+      end
+    end
+
+    context "retries on multiple exceptions" do
+      subject do
+        Class.new(QueueClassicPlus::Base) do
+          @queue = :test
+          retry! on: [SomeException, SomeOtherException], max: 5
+          skip_transaction!
+
+          def self.perform
+          end
+        end
+      end
+
+      it "retries on all specified exceptions" do
+        subject.retries_on?(SomeException.new).should be(true)
+        subject.retries_on?(SomeOtherException.new).should be(true)
+      end
+
+      it "does not retry on unspecified exceptions" do
+        subject.retries_on?(RuntimeError).should be(false)
+      end
+
+      it "sets max retries" do
+        subject.max_retries.should == 5
+      end
+    end
+
+    context "handles exception subclasses" do
+      class ServiceReallyUnavailable < SomeException
+      end
+
+      subject do
+        Class.new(QueueClassicPlus::Base) do
+          @queue = :test
+          retry! on: SomeException, max: 5
+          skip_transaction!
+
+          def self.perform
+          end
+        end
+      end
+
+      it "retries on a subclass of a specified exception" do
+        subject.retries_on?(ServiceReallyUnavailable.new).should be(true)
+      end
+
+      it "does not retry on unspecified exceptions" do
+        subject.retries_on?(RuntimeError).should be(false)
+      end
+
+      it "sets max retries" do
+        subject.max_retries.should == 5
+      end
+    end
+  end
+
+  describe ".librato_key" do
+    it "removes unsupported caracter from the classname" do
+      Jobs::Tests::TestJob.librato_key.should == 'jobs.tests.test_job'
+    end
+  end
+end
+

data/spec/sample_jobs.rb

@@ -0,0 +1,43 @@
+class SomeException < RuntimeError
+end
+
+class SomeOtherException < RuntimeError
+end
+
+module Jobs
+  module Tests
+    class LockedTestJob < QueueClassicPlus::Base
+      lock!
+
+      @queue = :low
+      retry! on: SomeException, max: 5
+
+      def self.perform should_raise
+        raise SomeException if should_raise
+      end
+    end
+
+
+    class TestJobNoRetry < QueueClassicPlus::Base
+      class Custom < RuntimeError
+      end
+
+      @queue = :low
+
+      def self.perform should_raise
+        raise Custom if should_raise
+      end
+    end
+
+
+    class TestJob < QueueClassicPlus::Base
+      @queue = :low
+      retry! on: SomeException, max: 5
+
+      def self.perform should_raise
+        raise SomeException if should_raise
+      end
+    end
+
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,31 @@
+require 'queue_classic_plus'
+require 'pg'
+require 'timecop'
+require 'queue_classic_matchers'
+require_relative './sample_jobs'
+
+RSpec.configure do |config|
+  config.before(:suite) do
+    ActiveRecord::Base.establish_connection(
+      :adapter  => "postgresql",
+      :username => "postgres",
+      :database => "queue_classic_plus_test",
+      :host => 'localhost',
+    )
+
+    ActiveRecord::Base.connection.execute "drop schema public cascade; create schema public;"
+
+    QC.default_conn_adapter = QC::ConnAdapter.new(ActiveRecord::Base.connection.raw_connection)
+    QC::Setup.create
+    QC::Later::Setup.create
+    QueueClassicPlus.migrate
+  end
+
+  config.before(:each) do
+    tables = ActiveRecord::Base.connection.tables.select do |table|
+      table != "schema_migrations"
+    end
+    ActiveRecord::Base.connection.execute("TRUNCATE #{tables.join(', ')} CASCADE") unless tables.empty?
+
+  end
+end

data/spec/worker_spec.rb

@@ -0,0 +1,129 @@
+require 'spec_helper'
+
+describe QueueClassicPlus::CustomWorker do
+  class QueueClassicLaterJob < ActiveRecord::Base
+  end
+
+  class QueueClassicJob < ActiveRecord::Base
+  end
+
+  let(:failed_queue) { described_class::FailedQueue }
+
+  context "failure" do
+    let(:queue) { QC::Queue.new("test") }
+    let(:worker) { described_class.new q_name: queue.name }
+
+    it "record failures in the failed queue" do
+      queue.enqueue("Kerklfadsjflaksj", 1, 2, 3)
+      failed_queue.count.should == 0
+      worker.work
+      failed_queue.count.should == 1
+      job = failed_queue.lock
+      job[:method].should == "Kerklfadsjflaksj"
+      job[:args].should == [1, 2, 3]
+      QueueClassicJob.last.last_error.should be_present
+    end
+
+    it "records normal errors" do
+      queue.enqueue("Jobs::Tests::TestJobNoRetry.perform", true)
+      failed_queue.count.should == 0
+      worker.work
+      failed_queue.count.should == 1
+    end
+  end
+
+  context "retry" do
+    let(:job_type) { Jobs::Tests::LockedTestJob }
+    let(:worker) { described_class.new q_name: job_type.queue.name }
+    let(:enqueue_expected_ts) { described_class::BACKOFF_WIDTH.seconds.from_now }
+
+    before do
+      job_type.skip_transaction!
+    end
+
+    it "retries" do
+      expect do
+        job_type.enqueue_perform(true)
+      end.to change_queue_size_of(job_type).by(1)
+
+      Jobs::Tests::LockedTestJob.should have_queue_size_of(1)
+      failed_queue.count.should == 0
+      QueueClassicMatchers::QueueClassicRspec.find_by_args('low', 'Jobs::Tests::LockedTestJob._perform', [true]).first['remaining_retries'].should be_nil
+
+      Timecop.freeze do
+        expect do
+          worker.work
+        end.to change_queue_size_of(job_type).by(-1)
+
+        failed_queue.count.should == 0 # not enqueued on Failed
+        Jobs::Tests::LockedTestJob.should have_scheduled(true).at(Time.now + described_class::BACKOFF_WIDTH.seconds.to_i) # should have scheduled a retry for later
+      end
+
+      Timecop.freeze(Time.now + (described_class::BACKOFF_WIDTH.seconds.to_i * 2)) do
+        QC::Later.tick(true)
+        # the job should be re-enqueued with a decremented retry count
+        jobs = QueueClassicMatchers::QueueClassicRspec.find_by_args('low', 'Jobs::Tests::LockedTestJob._perform', [true])
+        jobs.size.should == 1
+        job = jobs.first
+        job['remaining_retries'].to_i.should == job_type.max_retries - 1
+        job['locked_by'].should be_nil
+        job['locked_at'].should be_nil
+      end
+    end
+
+    it "enqueues in the failed queue when retries have been exhausted" do
+      job_type.max_retries = 0
+      expect do
+        job_type.enqueue_perform(true)
+      end.to change_queue_size_of(job_type).by(1)
+
+      Jobs::Tests::LockedTestJob.should have_queue_size_of(1)
+      failed_queue.count.should == 0
+      QueueClassicMatchers::QueueClassicRspec.find_by_args('low', 'Jobs::Tests::LockedTestJob._perform', [true]).first['remaining_retries'].should be_nil
+
+      Timecop.freeze do
+        expect do
+          worker.work
+        end.to change_queue_size_of(job_type).by(-1)
+
+        failed_queue.count.should == 1 # not enqueued on Failed
+        Jobs::Tests::LockedTestJob.should_not have_scheduled(true).at(Time.now + described_class::BACKOFF_WIDTH.seconds.to_i) # should have scheduled a retry for later
+      end
+    end
+  end
+
+  context "enqueuing during a retry" do
+    let(:job_type) { Jobs::Tests::LockedTestJob }
+    let(:worker) { described_class.new q_name: job_type.queue.name }
+    let(:enqueue_expected_ts) { described_class::BACKOFF_WIDTH.seconds.from_now }
+
+    before do
+      job_type.max_retries = 5
+      job_type.skip_transaction!
+    end
+
+    it "does not enqueue in main queue while retrying" do
+      expect do
+        job_type.enqueue_perform(true)
+      end.to change_queue_size_of(job_type).by(1)
+
+      Jobs::Tests::LockedTestJob.should have_queue_size_of(1)
+      failed_queue.count.should == 0
+      QueueClassicMatchers::QueueClassicRspec.find_by_args('low', 'Jobs::Tests::LockedTestJob._perform', [true]).first['remaining_retries'].should be_nil
+
+      Timecop.freeze do
+        expect do
+          worker.work
+        end.to change_queue_size_of(job_type).by(-1)
+
+        failed_queue.count.should == 0 # not enqueued on Failed
+        Jobs::Tests::LockedTestJob.should have_scheduled(true).at(Time.now + described_class::BACKOFF_WIDTH.seconds.to_i) # should have scheduled a retry for later
+
+        expect do
+          job_type.enqueue_perform(true)
+        end.to change_queue_size_of(job_type).by(0)
+      end
+    end
+  end
+end
+

metadata

@@ -0,0 +1,173 @@
+--- !ruby/object:Gem::Specification
+name: queue_classic_plus
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Simon Mathieu
+- Russell Smith
+- Jean-Philippe Boily
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-10-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: queue_classic
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: queue_classic-later
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: with_advisory_lock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ''
+email:
+- simon.math@gmail.com
+- russ@rainforestqa.com
+- j@jipi.ca
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rvmrc"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/queue_classic_plus.rb
+- lib/queue_classic_plus/base.rb
+- lib/queue_classic_plus/metrics.rb
+- lib/queue_classic_plus/railtie.rb
+- lib/queue_classic_plus/tasks/work.rake
+- lib/queue_classic_plus/update_metrics.rb
+- lib/queue_classic_plus/version.rb
+- lib/queue_classic_plus/worker.rb
+- lib/rails/generators/qc_plus_job/qc_plus_job_generator.rb
+- lib/rails/generators/qc_plus_job/templates/job.rb.erb
+- lib/rails/generators/qc_plus_job/templates/job_spec.rb.erb
+- queue_classic_plus.gemspec
+- spec/base_spec.rb
+- spec/sample_jobs.rb
+- spec/spec_helper.rb
+- spec/worker_spec.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Useful extras for Queue Classic
+test_files:
+- spec/base_spec.rb
+- spec/sample_jobs.rb
+- spec/spec_helper.rb
+- spec/worker_spec.rb